diff --git a/CLAUDE.md b/CLAUDE.md index 3ae280206d..53ea4e0cd2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -150,8 +150,8 @@ from a live database by `/internal/goldeneye`, a nested module, and its tests verify the committed files against one byte for byte. The same module checks the `analyze_*` cases under `/internal/endtoend/testdata/` against what the database itself reports for them, so a `fixture.sql` next to a case's schema -gives the queries rows to run against. Engines whose database is not -available skip. +gives the queries rows to run against. ClickHouse and SQLite have the check +today; engines whose database is not available skip. ```bash cd internal/goldeneye diff --git a/internal/core/analyzer/expr.go b/internal/core/analyzer/expr.go index d034432662..904993e703 100644 --- a/internal/core/analyzer/expr.go +++ b/internal/core/analyzer/expr.go @@ -285,6 +285,12 @@ func (a *analyzer) typeAExpr(e *ast.A_Expr) (exprType, error) { if err != nil { return exprType{}, err } + // The postfix null tests SQLite has — x ISNULL, x NOTNULL, x NOT NULL — + // arrive as operators with no right operand, and are predicates that + // are never NULL. + if e.Rexpr == nil && isNullTest(opName) { + return a.boolType(false) + } rightT, err := a.typeExpr(e.Rexpr) if err != nil { return exprType{}, err @@ -305,12 +311,25 @@ func (a *analyzer) typeAExpr(e *ast.A_Expr) (exprType, error) { if err != nil { return exprType{}, err } + // An operator's result is NULL when an operand is, except for IS and + // IS NOT, which test for NULL rather than propagate it: x IS NULL and + // x IS y are never NULL, whatever x and y are. return exprType{ typeOID: overload.ResultTypeOID, - nullable: leftT.nullable || rightT.nullable, + nullable: (leftT.nullable || rightT.nullable) && !isNullTest(opName), }, nil } +// isNullTest reports whether an operator compares with NULL as a value +// rather than propagating it, so that its result is never NULL. +func isNullTest(opName string) bool { + switch opName { + case "IS", "IS NOT", "ISNULL", "NOTNULL", "NOT NULL": + return true + } + return false +} + // typeQuantifiedExpr types "x = ANY($1)" and "x > ALL(...)": the right side // holds values of the left side's type, and the result is a predicate. func (a *analyzer) typeQuantifiedExpr(e *ast.A_Expr) (exprType, error) { diff --git a/internal/endtoend/testdata/analyze_expressions/sqlite/exec.json b/internal/endtoend/testdata/analyze_expressions/sqlite/exec.json new file mode 100644 index 0000000000..aa77909cb2 --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/sqlite/exec.json @@ -0,0 +1,5 @@ +{ + "command": "analyze", + "args": ["--dialect", "sqlite", "--schema", "schema.sql", "query.sql"], + "contexts": ["base"] +} diff --git a/internal/endtoend/testdata/analyze_expressions/sqlite/fixture.sql b/internal/endtoend/testdata/analyze_expressions/sqlite/fixture.sql new file mode 100644 index 0000000000..cda60b0180 --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/sqlite/fixture.sql @@ -0,0 +1,4 @@ +INSERT INTO users (id, name, bio, score) VALUES (1, 'ann', NULL, 1.5); +INSERT INTO users (id, name, bio, score) VALUES (2, 'bob', 'hello', 2.5); +INSERT INTO posts (id, user_id, title, created) VALUES (1, 1, 'first', '2024-01-01'); +INSERT INTO posts (id, user_id, title, created) VALUES (2, 1, NULL, '2024-01-02'); diff --git a/internal/endtoend/testdata/analyze_expressions/sqlite/query.sql b/internal/endtoend/testdata/analyze_expressions/sqlite/query.sql new file mode 100644 index 0000000000..04fdc2f8a0 --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/sqlite/query.sql @@ -0,0 +1,23 @@ +-- name: PostStats :one +SELECT count(*) AS total, max(id) AS latest, min(created) AS first +FROM posts WHERE user_id = ?; + +-- name: UserScores :one +SELECT avg(score) AS mean, sum(score) AS sum, sum(id) AS ids, group_concat(name) AS names +FROM users; + +-- name: ListUsers :many +SELECT id, lower(name) AS lname, id + 1 AS next +FROM users WHERE id IN (?, ?); + +-- name: ListPosts :many +SELECT id, title FROM posts ORDER BY created LIMIT ? OFFSET ?; + +-- name: UserPosts :many +SELECT u.name, p.title, p.created +FROM users u LEFT JOIN posts p ON p.user_id = u.id +WHERE u.name LIKE ? || '%'; + +-- name: CreatePost :one +INSERT INTO posts (user_id, title, created) VALUES (?, ?, datetime('now')) +RETURNING id, created; diff --git a/internal/endtoend/testdata/analyze_expressions/sqlite/schema.sql b/internal/endtoend/testdata/analyze_expressions/sqlite/schema.sql new file mode 100644 index 0000000000..72c6a57080 --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/sqlite/schema.sql @@ -0,0 +1,15 @@ +CREATE TABLE users ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + bio TEXT, + score REAL NOT NULL DEFAULT 0 +); + +CREATE TABLE posts ( + id INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + title TEXT, + created TEXT NOT NULL +); + +CREATE INDEX posts_user ON posts(user_id); diff --git a/internal/endtoend/testdata/analyze_expressions/sqlite/stdout.json b/internal/endtoend/testdata/analyze_expressions/sqlite/stdout.json new file mode 100644 index 0000000000..24326932cd --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/sqlite/stdout.json @@ -0,0 +1,245 @@ +[ + { + "name": "PostStats", + "cmd": ":one", + "columns": [ + { + "name": "total", + "type": { + "name": "integer" + } + }, + { + "name": "latest", + "type": { + "name": "integer", + "nullable": true + } + }, + { + "name": "first", + "type": { + "name": "text", + "nullable": true + } + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "user_id", + "type": { + "name": "integer" + }, + "table": "posts" + } + } + ] + }, + { + "name": "UserScores", + "cmd": ":one", + "columns": [ + { + "name": "mean", + "type": { + "name": "real", + "nullable": true + } + }, + { + "name": "sum", + "type": { + "name": "real", + "nullable": true + } + }, + { + "name": "ids", + "type": { + "name": "integer", + "nullable": true + } + }, + { + "name": "names", + "type": { + "name": "text", + "nullable": true + } + } + ], + "params": [] + }, + { + "name": "ListUsers", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "integer" + }, + "table": "users" + }, + { + "name": "lname", + "type": { + "name": "text" + } + }, + { + "name": "next", + "type": { + "name": "integer" + } + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "id", + "type": { + "name": "integer" + }, + "table": "users" + } + }, + { + "number": 2, + "column": { + "name": "id", + "type": { + "name": "integer" + }, + "table": "users" + } + } + ] + }, + { + "name": "ListPosts", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "integer" + }, + "table": "posts" + }, + { + "name": "title", + "type": { + "name": "text", + "nullable": true + }, + "table": "posts" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "", + "type": { + "name": "integer" + } + } + }, + { + "number": 2, + "column": { + "name": "", + "type": { + "name": "integer" + } + } + } + ] + }, + { + "name": "UserPosts", + "cmd": ":many", + "columns": [ + { + "name": "name", + "type": { + "name": "text" + }, + "table": "users" + }, + { + "name": "title", + "type": { + "name": "text", + "nullable": true + }, + "table": "posts" + }, + { + "name": "created", + "type": { + "name": "text" + }, + "table": "posts" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "", + "type": { + "name": "text" + } + } + } + ] + }, + { + "name": "CreatePost", + "cmd": ":one", + "columns": [ + { + "name": "id", + "type": { + "name": "integer" + }, + "table": "posts" + }, + { + "name": "created", + "type": { + "name": "text" + }, + "table": "posts" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "user_id", + "type": { + "name": "integer" + }, + "table": "posts" + } + }, + { + "number": 2, + "column": { + "name": "title", + "type": { + "name": "text", + "nullable": true + }, + "table": "posts" + } + } + ] + } +] diff --git a/internal/engine/sqlite/dialect/dialect.json b/internal/engine/sqlite/dialect/dialect.json index ba49e48300..36129df83a 100644 --- a/internal/engine/sqlite/dialect/dialect.json +++ b/internal/engine/sqlite/dialect/dialect.json @@ -7,7 +7,7 @@ "bool": "boolean" }, "bool": "boolean", - "comparison": ["=", "==", "<>", "!=", "<", "<=", ">", ">=", "IS", "IS NOT"], + "comparison": ["=", "==", "<>", "!=", "<", "<=", ">", ">=", "IS", "IS NOT", "LIKE", "NOT LIKE", "GLOB", "NOT GLOB", "REGEXP", "NOT REGEXP", "MATCH", "NOT MATCH"], "comparison_categories": "BNSDU", "arithmetic": ["+", "-", "*", "/", "%"], "arithmetic_categories": "N", diff --git a/internal/engine/sqlite/dialect/functions.jsonl b/internal/engine/sqlite/dialect/functions.jsonl index da25d4d581..18012549de 100644 --- a/internal/engine/sqlite/dialect/functions.jsonl +++ b/internal/engine/sqlite/dialect/functions.jsonl @@ -1,6 +1,19 @@ {"name":"-\u003e","args":[{"type":"any"},{"type":"any"}],"returns":"text","nullable":true} {"name":"-\u003e\u003e","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} {"name":"abs","args":[{"type":"any"}],"returns":"real"} +{"name":"abs","args":[{"type":"integer"}],"returns":"integer"} +{"name":"abs","args":[{"type":"int"}],"returns":"integer"} +{"name":"abs","args":[{"type":"tinyint"}],"returns":"integer"} +{"name":"abs","args":[{"type":"smallint"}],"returns":"integer"} +{"name":"abs","args":[{"type":"mediumint"}],"returns":"integer"} +{"name":"abs","args":[{"type":"bigint"}],"returns":"integer"} +{"name":"abs","args":[{"type":"unsigned big int"}],"returns":"integer"} +{"name":"abs","args":[{"type":"int2"}],"returns":"integer"} +{"name":"abs","args":[{"type":"int8"}],"returns":"integer"} +{"name":"abs","args":[{"type":"real"}],"returns":"real"} +{"name":"abs","args":[{"type":"double"}],"returns":"real"} +{"name":"abs","args":[{"type":"double precision"}],"returns":"real"} +{"name":"abs","args":[{"type":"float"}],"returns":"real"} {"name":"acos","args":[{"type":"real"}],"returns":"real"} {"name":"acosh","args":[{"type":"real"}],"returns":"real"} {"name":"asin","args":[{"type":"real"}],"returns":"real"} @@ -10,7 +23,33 @@ {"name":"atanh","args":[{"type":"real"}],"returns":"real"} {"name":"avg","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} {"name":"ceil","args":[{"type":"any"}],"returns":"real"} +{"name":"ceil","args":[{"type":"integer"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"int"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"tinyint"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"smallint"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"mediumint"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"bigint"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"unsigned big int"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"int2"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"int8"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"real"}],"returns":"real"} +{"name":"ceil","args":[{"type":"double"}],"returns":"real"} +{"name":"ceil","args":[{"type":"double precision"}],"returns":"real"} +{"name":"ceil","args":[{"type":"float"}],"returns":"real"} {"name":"ceiling","args":[{"type":"any"}],"returns":"real"} +{"name":"ceiling","args":[{"type":"integer"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"int"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"tinyint"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"smallint"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"mediumint"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"bigint"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"unsigned big int"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"int2"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"int8"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"real"}],"returns":"real"} +{"name":"ceiling","args":[{"type":"double"}],"returns":"real"} +{"name":"ceiling","args":[{"type":"double precision"}],"returns":"real"} +{"name":"ceiling","args":[{"type":"float"}],"returns":"real"} {"name":"changes","returns":"integer"} {"name":"char","args":[{"type":"integer","mode":"v"}],"returns":"text"} {"name":"coalesce","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any"} @@ -31,6 +70,19 @@ {"name":"exp","args":[{"type":"real"}],"returns":"real"} {"name":"first_value","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} {"name":"floor","args":[{"type":"any"}],"returns":"real"} +{"name":"floor","args":[{"type":"integer"}],"returns":"integer"} +{"name":"floor","args":[{"type":"int"}],"returns":"integer"} +{"name":"floor","args":[{"type":"tinyint"}],"returns":"integer"} +{"name":"floor","args":[{"type":"smallint"}],"returns":"integer"} +{"name":"floor","args":[{"type":"mediumint"}],"returns":"integer"} +{"name":"floor","args":[{"type":"bigint"}],"returns":"integer"} +{"name":"floor","args":[{"type":"unsigned big int"}],"returns":"integer"} +{"name":"floor","args":[{"type":"int2"}],"returns":"integer"} +{"name":"floor","args":[{"type":"int8"}],"returns":"integer"} +{"name":"floor","args":[{"type":"real"}],"returns":"real"} +{"name":"floor","args":[{"type":"double"}],"returns":"real"} +{"name":"floor","args":[{"type":"double precision"}],"returns":"real"} +{"name":"floor","args":[{"type":"float"}],"returns":"real"} {"name":"format","args":[{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} {"name":"glob","args":[{"type":"text"},{"type":"text"}],"returns":"integer"} {"name":"group_concat","kind":"a","args":[{"type":"text"}],"returns":"text","nullable":true} @@ -137,6 +189,19 @@ {"name":"substring","args":[{"type":"any"},{"type":"integer"},{"type":"integer"}],"returns":"text"} {"name":"subtype","args":[{"type":"any"}],"returns":"integer"} {"name":"sum","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"integer"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"int"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"tinyint"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"smallint"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"mediumint"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"bigint"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"unsigned big int"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"int2"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"int8"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"real"}],"returns":"real","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"double"}],"returns":"real","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"double precision"}],"returns":"real","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"float"}],"returns":"real","nullable":true} {"name":"tan","args":[{"type":"real"}],"returns":"real"} {"name":"tanh","args":[{"type":"real"}],"returns":"real"} {"name":"time","args":[{"type":"any","mode":"v"}],"returns":"text","nullable":true} @@ -146,6 +211,19 @@ {"name":"trim","args":[{"type":"text"}],"returns":"text"} {"name":"trim","args":[{"type":"text"},{"type":"text"}],"returns":"text"} {"name":"trunc","args":[{"type":"any"}],"returns":"real"} +{"name":"trunc","args":[{"type":"integer"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"int"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"tinyint"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"smallint"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"mediumint"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"bigint"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"unsigned big int"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"int2"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"int8"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"real"}],"returns":"real"} +{"name":"trunc","args":[{"type":"double"}],"returns":"real"} +{"name":"trunc","args":[{"type":"double precision"}],"returns":"real"} +{"name":"trunc","args":[{"type":"float"}],"returns":"real"} {"name":"typeof","args":[{"type":"any"}],"returns":"text"} {"name":"unhex","args":[{"type":"text"}],"returns":"blob","nullable":true} {"name":"unhex","args":[{"type":"text"},{"type":"text"}],"returns":"blob","nullable":true} diff --git a/internal/engine/sqlite/dialect/operators.jsonl b/internal/engine/sqlite/dialect/operators.jsonl index 1312de4e53..9ed8beaf64 100644 --- a/internal/engine/sqlite/dialect/operators.jsonl +++ b/internal/engine/sqlite/dialect/operators.jsonl @@ -1,3 +1 @@ {"name": "||", "left": "text", "right": "text", "result": "text"} -{"name": "~~", "left": "text", "right": "text", "result": "boolean"} -{"name": "!~~", "left": "text", "right": "text", "result": "boolean"} diff --git a/internal/goldeneye/README.md b/internal/goldeneye/README.md index 76807265ee..7139a16ac9 100644 --- a/internal/goldeneye/README.md +++ b/internal/goldeneye/README.md @@ -80,15 +80,24 @@ the hand-written files alone, and the checks do not look at them. `sqlite/signatures.go`, since a SQLite function returns NULL as often by setting no result as by saying so. The pinned release is the one the main module's driver embeds. SQLite has no catalog of types or operators, so - `types.jsonl` and `operators.jsonl` are hand-written. + `types.jsonl` and `operators.jsonl` are hand-written. A function that + returns an integer for an integer and a real for a real — `abs`, `ceil`, + `floor`, `trunc`, `sum` — is written once over `any`, returning the real + that a text or blob argument gets, and once more per spelling + `types.jsonl` gives integer and real, returning that type, the way + PostgreSQL's catalog has a `sum` per numeric type; the overload over `any` + comes first, since the legacy compiler resolves by arity alone and takes + it. `install` builds one more shell, for the analysis check below; + nothing is generated from it. ## Layout - `dialect/` — the record types the files are made of, mirrored from `internal/core/seed`, and the helpers that write a generated set of files into an engine directory or diff it against what is committed. -- `endtoend/` — finds the analyze cases and compares an engine's answer with - a case's committed output. +- `endtoend/` — finds the analyze cases, splits their query files, and + compares an engine's answer with a case's committed output. +- `analysis/` — the shape of that answer: the JSON `sqlc analyze` prints. - `postgresql/`, `duckdb/`, `clickhouse/`, `sqlite/` — one package per engine, each exposing `Locate`, `Version` and `Generate`, `Analyze` where the engine has an analysis check, and tests that run the checks. @@ -111,5 +120,32 @@ asks for `--ast` is skipped, since only sqlc can print that. `EXPLAIN QUERY TREE`, and parameters from sentinel constants substituted for `?`, `sqlc.arg()` and `sqlc.narg()`, since ClickHouse itself never sees a placeholder; `INSERT ... VALUES` parameters map onto `DESCRIBE TABLE`. +- **`sqlite`** runs each case through one more shell `install` builds, under + `analysis/`: every extension option at once, so that any case's schema + loads, and `SQLITE_ENABLE_COLUMN_METADATA`, which lets the shell's `.stats + stmt` say which table column each result column of a statement is read + from. That column's declared type is the result column's, and its NOT + NULL decides nullability, the rowid counting as NOT NULL. SQLite types + values rather than expressions, so a column the library has no origin for + — an aggregate, an arithmetic result — is typed by the storage class of + the value it returns, which is why a case wants a `fixture.sql`: the query + is run over the fixture, with each parameter bound to a value of the + column it stands in for, and again over no rows, and a column is nullable + when either run returns a NULL for it — an aggregate over nothing, the far + side of an outer join. The library reports nothing about a parameter but + its number, so parameters are found in the bytecode `EXPLAIN` prints, the + way ClickHouse's are found in its query tree: each is followed from the + register its `Variable` loads, through copies and the expressions it is an + argument of, to the first opcode that uses it against something the + catalog can name — the other operand of a comparison, the row a seek lands + on, the position in the record an `Insert` writes, the column of an IN + list's ephemeral table it comes back out of. One that reaches nothing + nameable is described by what the program requires of it, when it + requires anything: `MustBeInt` makes LIMIT's an integer. `sqlc.arg(name)` + becomes `?N`, numbered as sqlc numbers them, so a repeated name is one + parameter. Two things the check reports that sqlc does not: a bare column + selected alongside an aggregate is NULL over no rows, and so nullable, and + a comparison such as `x IS NULL` is an integer, since that is what SQLite + returns. The other engines have no analysis check yet. diff --git a/internal/goldeneye/analysis/analysis.go b/internal/goldeneye/analysis/analysis.go new file mode 100644 index 0000000000..fe907766a3 --- /dev/null +++ b/internal/goldeneye/analysis/analysis.go @@ -0,0 +1,61 @@ +// Package analysis is the shape of an engine's answer about a query: the +// JSON `sqlc analyze` prints, so that a case's committed stdout.json can be +// compared with what the database itself reports, byte for byte. +package analysis + +import ( + "bytes" + "encoding/json" +) + +// Query is what was found out about one query. +type Query struct { + Name string `json:"name"` + Cmd string `json:"cmd"` + Columns []Column `json:"columns"` + Params []Param `json:"params"` +} + +// Column describes a result column, or the column a parameter stands in +// for. +type Column struct { + Name string `json:"name"` + Type *TypeExpr `json:"type,omitempty"` + Table string `json:"table,omitempty"` +} + +// Param is one parameter and what it is compared with or assigned to. +type Param struct { + Number int `json:"number"` + Column Column `json:"column"` +} + +// TypeExpr is a type as a call expression: a lowercased name applied to an +// ordered argument list, each argument another type, an integer, a boolean +// or a quoted string, optionally labelled. Nullability is an attribute of +// the type rather than a wrapper. +type TypeExpr struct { + Name string `json:"name"` + Nullable bool `json:"nullable,omitempty"` + Args []TypeArg `json:"args,omitempty"` +} + +// TypeArg is one argument of a TypeExpr. +type TypeArg struct { + Label string `json:"label,omitempty"` + Type *TypeExpr `json:"type,omitempty"` + Int *int64 `json:"int,omitempty"` + Bool *bool `json:"bool,omitempty"` + String *string `json:"string,omitempty"` +} + +// Encode prints the answer the way sqlc analyze does. +func Encode(queries []Query) ([]byte, error) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetIndent("", " ") + if err := enc.Encode(queries); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/internal/goldeneye/clickhouse/analyze.go b/internal/goldeneye/clickhouse/analyze.go index ded6201b51..ff2148fdf5 100644 --- a/internal/goldeneye/clickhouse/analyze.go +++ b/internal/goldeneye/clickhouse/analyze.go @@ -7,33 +7,15 @@ import ( "regexp" "strconv" "strings" -) - -// The output is the JSON `sqlc analyze` prints, so a case's committed -// stdout.json can be compared with it byte for byte. - -type analyzedQuery struct { - Name string `json:"name"` - Cmd string `json:"cmd"` - Columns []analyzedColumn `json:"columns"` - Params []analyzedParam `json:"params"` -} -type analyzedColumn struct { - Name string `json:"name"` - Type *typeExpr `json:"type,omitempty"` - Table string `json:"table,omitempty"` -} - -type analyzedParam struct { - Number int `json:"number"` - Column analyzedColumn `json:"column"` -} + "github.com/sqlc-dev/sqlc/internal/goldeneye/analysis" + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" +) // analyze runs every query against the schema and fixture and records what // ClickHouse reports about each. -func analyze(ctx context.Context, l local, schema, fixture string, queries []query) ([]analyzedQuery, error) { - out := make([]analyzedQuery, 0, len(queries)) +func analyze(ctx context.Context, l local, schema, fixture string, queries []endtoend.Query) ([]analysis.Query, error) { + out := make([]analysis.Query, 0, len(queries)) for _, q := range queries { aq, err := analyzeQuery(ctx, l, schema, fixture, q) if err != nil { @@ -44,7 +26,7 @@ func analyze(ctx context.Context, l local, schema, fixture string, queries []que return out, nil } -func analyzeQuery(ctx context.Context, l local, schema, fixture string, q query) (analyzedQuery, error) { +func analyzeQuery(ctx context.Context, l local, schema, fixture string, q endtoend.Query) (analysis.Query, error) { sql, phs := bindPlaceholders(q.SQL) explain := returnsRows(sql) @@ -62,33 +44,33 @@ func analyzeQuery(ctx context.Context, l local, schema, fixture string, q query) results, err := l.run(ctx, script.String()) if err != nil { - return analyzedQuery{}, err + return analysis.Query{}, err } - aq := analyzedQuery{ + aq := analysis.Query{ Name: q.Name, Cmd: q.Cmd, - Columns: []analyzedColumn{}, - Params: []analyzedParam{}, + Columns: []analysis.Column{}, + Params: []analysis.Param{}, } if !explain { return analyzeExec(ctx, l, script.String(), sql, phs, aq) } if len(results) != 2 { - return analyzedQuery{}, fmt.Errorf("expected the query tree and one result set, got %d results", len(results)) + return analysis.Query{}, fmt.Errorf("expected the query tree and one result set, got %d results", len(results)) } var lines []string for _, row := range results[0].Data { var line string if err := json.Unmarshal(row["explain"], &line); err != nil { - return analyzedQuery{}, fmt.Errorf("reading query tree: %w", err) + return analysis.Query{}, fmt.Errorf("reading query tree: %w", err) } lines = append(lines, line) } tree, err := parseQueryTree(lines) if err != nil { - return analyzedQuery{}, err + return analysis.Query{}, err } // Names and types come from the block header of the executed query, the @@ -104,23 +86,23 @@ func analyzeQuery(ctx context.Context, l local, schema, fixture string, q query) sentinels := tree.sentinels() for i, ph := range phs { - ac := analyzedColumn{} + ac := analysis.Column{} if sentinel := sentinels[i+1]; sentinel != nil { ac = tree.paramColumn(sentinel) } if ph.Name != "" { ac.Name = ph.Name } - aq.Params = append(aq.Params, analyzedParam{Number: ph.Number, Column: ac}) + aq.Params = append(aq.Params, analysis.Param{Number: ph.Number, Column: ac}) } return aq, nil } -func column(name, typ string) analyzedColumn { +func column(name, typ string) analysis.Column { if typ == "" { - return analyzedColumn{Name: name} + return analysis.Column{Name: name} } - return analyzedColumn{Name: name, Type: parseType(typ)} + return analysis.Column{Name: name, Type: parseType(typ)} } // returnsRows reports whether a statement produces a result set and so can @@ -185,7 +167,7 @@ func sentinelOrdinal(c *treeNode) (int, bool) { // paramColumn describes what a placeholder is compared with or assigned to: // the other operand of the function it is an argument of, preferring a // column over an expression, or the projected column it stands for. -func (t *queryTree) paramColumn(sentinel *treeNode) analyzedColumn { +func (t *queryTree) paramColumn(sentinel *treeNode) analysis.Column { list := sentinel.parent if list != nil && list.kind == "LIST" && list.parent != nil { switch owner := list.parent; { @@ -218,7 +200,7 @@ func (t *queryTree) paramColumn(sentinel *treeNode) analyzedColumn { } // describe turns a tree expression into a column description. -func (t *queryTree) describe(n *treeNode) analyzedColumn { +func (t *queryTree) describe(n *treeNode) analysis.Column { switch n.kind { case "COLUMN": ac := column(n.attrs["column_name"], n.attrs["result_type"]) @@ -236,7 +218,7 @@ func (t *queryTree) describe(n *treeNode) analyzedColumn { } return column(name, n.attrs["constant_value_type"]) } - return analyzedColumn{} + return analysis.Column{} } var insertValuesRe = regexp.MustCompile(`(?is)^insert\s+into\s+(?:table\s+)?([\w.` + "`" + `"]+)\s*(?:\(([^)]*)\))?\s*(?:format\s+)?values\b`) @@ -244,20 +226,20 @@ var insertValuesRe = regexp.MustCompile(`(?is)^insert\s+into\s+(?:table\s+)?([\w // analyzeExec runs a statement that returns no rows. The only parameters it // can describe are those of an INSERT ... VALUES, which map positionally // onto the target columns reported by DESCRIBE TABLE. -func analyzeExec(ctx context.Context, l local, script, sql string, phs []placeholder, aq analyzedQuery) (analyzedQuery, error) { +func analyzeExec(ctx context.Context, l local, script, sql string, phs []placeholder, aq analysis.Query) (analysis.Query, error) { m := insertValuesRe.FindStringSubmatch(sql) if m != nil { script += "DESCRIBE TABLE " + m[1] + ";\n" } results, err := l.run(ctx, script) if err != nil { - return analyzedQuery{}, err + return analysis.Query{}, err } - var targets []analyzedColumn + var targets []analysis.Column if m != nil && len(results) == 1 { - byName := map[string]analyzedColumn{} - var all []analyzedColumn + byName := map[string]analysis.Column{} + var all []analysis.Column table := strings.Trim(m[1][strings.LastIndexByte(m[1], '.')+1:], "`\"") for _, row := range results[0].Data { var name, typ string @@ -277,14 +259,14 @@ func analyzeExec(ctx context.Context, l local, script, sql string, phs []placeho } } for i, ph := range phs { - ac := analyzedColumn{} + ac := analysis.Column{} if len(targets) > 0 { ac = targets[i%len(targets)] } if ph.Name != "" { ac.Name = ph.Name } - aq.Params = append(aq.Params, analyzedParam{Number: ph.Number, Column: ac}) + aq.Params = append(aq.Params, analysis.Param{Number: ph.Number, Column: ac}) } return aq, nil } diff --git a/internal/goldeneye/clickhouse/check.go b/internal/goldeneye/clickhouse/check.go index 1ad8d9cb93..fab53b8a08 100644 --- a/internal/goldeneye/clickhouse/check.go +++ b/internal/goldeneye/clickhouse/check.go @@ -1,12 +1,10 @@ package clickhouse import ( - "bytes" "context" - "encoding/json" - "fmt" "os" + "github.com/sqlc-dev/sqlc/internal/goldeneye/analysis" "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" ) @@ -23,25 +21,15 @@ func Analyze(ctx context.Context, binary string, c endtoend.Case) ([]byte, error return nil, err } } - src, err := os.ReadFile(c.Query) + queries, err := c.Queries() if err != nil { return nil, err } - queries, err := parseQueries(string(src)) - if err != nil { - return nil, fmt.Errorf("%s: %w", c.Query, err) - } out, err := analyze(ctx, local{binary: binary}, string(schema), string(fixture), queries) if err != nil { return nil, err } - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - enc.SetIndent("", " ") - if err := enc.Encode(out); err != nil { - return nil, err - } - return buf.Bytes(), nil + return analysis.Encode(out) } // Check compares what ClickHouse reports for a case with the output the diff --git a/internal/goldeneye/clickhouse/queries.go b/internal/goldeneye/clickhouse/queries.go index 69e3b1ac65..a9f77877b7 100644 --- a/internal/goldeneye/clickhouse/queries.go +++ b/internal/goldeneye/clickhouse/queries.go @@ -2,59 +2,10 @@ package clickhouse import ( "fmt" - "regexp" "strings" - "unicode" -) - -// query is one entry of a sqlc query file. -type query struct { - Name string - Cmd string - SQL string -} -var headerRe = regexp.MustCompile(`^\s*--\s*name:\s*(\S+)\s+(:\S+)\s*$`) - -// parseQueries splits a sqlc query file on its `-- name: X :cmd` headers. -func parseQueries(src string) ([]query, error) { - var queries []query - var cur *query - var body []string - flush := func() error { - if cur == nil { - return nil - } - sql := strings.TrimSpace(strings.Join(body, "\n")) - sql = strings.TrimRight(sql, "; \t\r\n") - if sql == "" { - return fmt.Errorf("query %s has no body", cur.Name) - } - cur.SQL = sql - queries = append(queries, *cur) - return nil - } - for _, line := range strings.Split(src, "\n") { - if m := headerRe.FindStringSubmatch(line); m != nil { - if err := flush(); err != nil { - return nil, err - } - cur = &query{Name: m[1], Cmd: m[2]} - body = body[:0] - continue - } - if cur != nil { - body = append(body, line) - } - } - if err := flush(); err != nil { - return nil, err - } - if len(queries) == 0 { - return nil, fmt.Errorf("no queries found: expected `-- name: Name :cmd` headers") - } - return queries, nil -} + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" +) // placeholder is one parameter reference in a query, in order of appearance. type placeholder struct { @@ -79,74 +30,17 @@ func sentinelFor(lastWord string, ordinal int) string { return fmt.Sprintf("(NULL + %d)", ordinal) } -var namedArgRe = regexp.MustCompile(`^sqlc\.(n?arg)\(\s*'?([A-Za-z_][A-Za-z0-9_]*)'?\s*\)`) - // bindPlaceholders rewrites sqlc's parameter syntax (?, sqlc.arg(name), -// sqlc.narg(name)) into constants ClickHouse can analyze, skipping string -// literals, quoted identifiers and comments. ClickHouse binds every ? -// positionally, so each placeholder is its own parameter even when a name -// repeats, which is how sqlc numbers them too. +// sqlc.narg(name)) into constants ClickHouse can analyze. ClickHouse binds +// every ? positionally, so each placeholder is its own parameter even when +// a name repeats, which is how sqlc numbers them too. func bindPlaceholders(sql string) (string, []placeholder) { - var ( - out strings.Builder - phs []placeholder - lastWord string - i = 0 - ) - number := func(string) int { return len(phs) + 1 } - for i < len(sql) { - c := sql[i] - switch { - case c == '\'' || c == '"' || c == '`': - end := skipQuoted(sql, i) - out.WriteString(sql[i:end]) - i = end - case strings.HasPrefix(sql[i:], "--"): - end := strings.IndexByte(sql[i:], '\n') - if end < 0 { - end = len(sql) - } else { - end += i - } - out.WriteString(sql[i:end]) - i = end - case strings.HasPrefix(sql[i:], "/*"): - end := strings.Index(sql[i:], "*/") - if end < 0 { - end = len(sql) - } else { - end += i + 2 - } - out.WriteString(sql[i:end]) - i = end - case c == '?': - out.WriteString(sentinelFor(lastWord, len(phs)+1)) - phs = append(phs, placeholder{Number: number("")}) - lastWord = "" - i++ - case c == 's' && namedArgRe.MatchString(sql[i:]): - m := namedArgRe.FindStringSubmatch(sql[i:]) - out.WriteString(sentinelFor(lastWord, len(phs)+1)) - phs = append(phs, placeholder{Number: number(m[2]), Name: m[2]}) - lastWord = "" - i += len(m[0]) - case isWordByte(c): - end := i - for end < len(sql) && isWordByte(sql[end]) { - end++ - } - lastWord = sql[i:end] - out.WriteString(lastWord) - i = end - default: - if !unicode.IsSpace(rune(c)) && c != ',' && c != '(' { - lastWord = "" - } - out.WriteByte(c) - i++ - } - } - return out.String(), phs + var phs []placeholder + out := endtoend.Rewrite(sql, func(name, lastWord string) string { + phs = append(phs, placeholder{Number: len(phs) + 1, Name: name}) + return sentinelFor(lastWord, len(phs)) + }) + return out, phs } func isWordByte(c byte) bool { diff --git a/internal/goldeneye/clickhouse/types.go b/internal/goldeneye/clickhouse/types.go index 77ed8d8c19..57f6caf731 100644 --- a/internal/goldeneye/clickhouse/types.go +++ b/internal/goldeneye/clickhouse/types.go @@ -3,6 +3,8 @@ package clickhouse import ( "strconv" "strings" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/analysis" ) // A type is a call expression, the way ClickHouse itself models one: a @@ -35,22 +37,8 @@ import ( // String) is a type with no arguments. Resolving names against the catalog // is the reader's job; the output only records what was said. -type typeExpr struct { - Name string `json:"name"` - Nullable bool `json:"nullable,omitempty"` - Args []typeArg `json:"args,omitempty"` -} - -type typeArg struct { - Label string `json:"label,omitempty"` - Type *typeExpr `json:"type,omitempty"` - Int *int64 `json:"int,omitempty"` - Bool *bool `json:"bool,omitempty"` - String *string `json:"string,omitempty"` -} - // parseType turns a ClickHouse type string into its expression. -func parseType(t string) *typeExpr { +func parseType(t string) *analysis.TypeExpr { name, args := splitType(t) name = strings.ToLower(strings.TrimSpace(name)) if name == "nullable" && len(args) == 1 { @@ -61,7 +49,7 @@ func parseType(t string) *typeExpr { if name == "" { name = "nothing" } - expr := &typeExpr{Name: name} + expr := &analysis.TypeExpr{Name: name} for _, a := range args { expr.Args = append(expr.Args, parseArg(a)) } @@ -71,7 +59,7 @@ func parseType(t string) *typeExpr { // parseArg parses one argument: a quoted string, an integer, a boolean, a // labelled argument (`lat Float64` in a Tuple, `'a' = 1` in an Enum), or a // type. -func parseArg(a string) typeArg { +func parseArg(a string) analysis.TypeArg { a = strings.TrimSpace(a) if strings.HasPrefix(a, "'") { end := skipQuoted(a, 0) @@ -81,22 +69,22 @@ func parseArg(a string) typeArg { arg.Label = lit return arg } - return typeArg{String: &lit} + return analysis.TypeArg{String: &lit} } if n, err := strconv.ParseInt(a, 10, 64); err == nil { - return typeArg{Int: &n} + return analysis.TypeArg{Int: &n} } switch strings.ToLower(a) { case "true", "false": b := strings.EqualFold(a, "true") - return typeArg{Bool: &b} + return analysis.TypeArg{Bool: &b} } if i := labelEnd(a); i > 0 { arg := parseArg(a[i+1:]) arg.Label = a[:i] return arg } - return typeArg{Type: parseType(a)} + return analysis.TypeArg{Type: parseType(a)} } // labelEnd returns the index of the space separating a label from the type diff --git a/internal/goldeneye/cmd/goldeneye/main.go b/internal/goldeneye/cmd/goldeneye/main.go index 34abbf7c7d..13fa087f44 100644 --- a/internal/goldeneye/cmd/goldeneye/main.go +++ b/internal/goldeneye/cmd/goldeneye/main.go @@ -70,7 +70,7 @@ var engines = []engine{ {clickhouse.Engine, clickhouse.Locate, clickhouse.Version, clickhouse.Generate, clickhouse.Analyze}, {duckdb.Engine, duckdb.Locate, duckdb.Version, duckdb.Generate, nil}, {postgresql.Engine, postgresql.Locate, postgresql.Version, postgresql.Generate, nil}, - {sqlite.Engine, sqlite.Locate, sqlite.Version, sqlite.Generate, nil}, + {sqlite.Engine, sqlite.Locate, sqlite.Version, sqlite.Generate, sqlite.Analyze}, } // installer puts the binary an engine is read through in place, for the diff --git a/internal/goldeneye/dialect/dialect.go b/internal/goldeneye/dialect/dialect.go index 3db8aab463..6085ec9b2f 100644 --- a/internal/goldeneye/dialect/dialect.go +++ b/internal/goldeneye/dialect/dialect.go @@ -118,6 +118,25 @@ func JSONL[T any](records []T) ([]byte, error) { return buf.Bytes(), nil } +// ReadTypes reads the hand-written types.jsonl of a dialect directory, for +// a generator that writes a function once per spelling of a type. +func ReadTypes(dir string) ([]Type, error) { + blob, err := os.ReadFile(filepath.Join(dir, TypesFile)) + if err != nil { + return nil, err + } + var types []Type + dec := json.NewDecoder(bytes.NewReader(blob)) + for dec.More() { + var t Type + if err := dec.Decode(&t); err != nil { + return nil, fmt.Errorf("%s: %w", filepath.Join(dir, TypesFile), err) + } + types = append(types, t) + } + return types, nil +} + // Dir returns the dialect directory of an engine, // internal/engine//dialect, found relative to this source file so // the working directory does not matter. diff --git a/internal/goldeneye/endtoend/query.go b/internal/goldeneye/endtoend/query.go new file mode 100644 index 0000000000..465595eda7 --- /dev/null +++ b/internal/goldeneye/endtoend/query.go @@ -0,0 +1,163 @@ +package endtoend + +import ( + "fmt" + "os" + "regexp" + "strings" + "unicode" +) + +// Query is one entry of a sqlc query file. +type Query struct { + Name string + Cmd string + SQL string +} + +// Queries reads the case's query file and splits it into its queries. +func (c Case) Queries() ([]Query, error) { + src, err := os.ReadFile(c.Query) + if err != nil { + return nil, err + } + queries, err := ParseQueries(string(src)) + if err != nil { + return nil, fmt.Errorf("%s: %w", c.Query, err) + } + return queries, nil +} + +var headerRe = regexp.MustCompile(`^\s*--\s*name:\s*(\S+)\s+(:\S+)\s*$`) + +// ParseQueries splits a sqlc query file on its `-- name: X :cmd` headers. +func ParseQueries(src string) ([]Query, error) { + var queries []Query + var cur *Query + var body []string + flush := func() error { + if cur == nil { + return nil + } + sql := strings.TrimSpace(strings.Join(body, "\n")) + sql = strings.TrimRight(sql, "; \t\r\n") + if sql == "" { + return fmt.Errorf("query %s has no body", cur.Name) + } + cur.SQL = sql + queries = append(queries, *cur) + return nil + } + for _, line := range strings.Split(src, "\n") { + if m := headerRe.FindStringSubmatch(line); m != nil { + if err := flush(); err != nil { + return nil, err + } + cur = &Query{Name: m[1], Cmd: m[2]} + body = body[:0] + continue + } + if cur != nil { + body = append(body, line) + } + } + if err := flush(); err != nil { + return nil, err + } + if len(queries) == 0 { + return nil, fmt.Errorf("no queries found: expected `-- name: Name :cmd` headers") + } + return queries, nil +} + +var namedArgRe = regexp.MustCompile(`^sqlc\.(n?arg)\(\s*'?([A-Za-z_][A-Za-z0-9_]*)'?\s*\)`) + +// Rewrite replaces every parameter reference in a query — ?, sqlc.arg(name) +// and sqlc.narg(name) — with what bind returns for it, in order of +// appearance, skipping string literals, quoted identifiers and comments. +// bind is handed the name, empty for a ?, and the word before the +// reference, so that a LIMIT or OFFSET can be bound differently from a +// value. Each engine decides what a reference becomes and how the +// references are numbered. +func Rewrite(sql string, bind func(name, lastWord string) string) string { + var ( + out strings.Builder + lastWord string + i = 0 + ) + for i < len(sql) { + c := sql[i] + switch { + case c == '\'' || c == '"' || c == '`': + end := skipQuoted(sql, i) + out.WriteString(sql[i:end]) + i = end + case strings.HasPrefix(sql[i:], "--"): + end := strings.IndexByte(sql[i:], '\n') + if end < 0 { + end = len(sql) + } else { + end += i + } + out.WriteString(sql[i:end]) + i = end + case strings.HasPrefix(sql[i:], "/*"): + end := strings.Index(sql[i:], "*/") + if end < 0 { + end = len(sql) + } else { + end += i + 2 + } + out.WriteString(sql[i:end]) + i = end + case c == '?': + out.WriteString(bind("", lastWord)) + lastWord = "" + i++ + case c == 's' && namedArgRe.MatchString(sql[i:]): + m := namedArgRe.FindStringSubmatch(sql[i:]) + out.WriteString(bind(m[2], lastWord)) + lastWord = "" + i += len(m[0]) + case isWordByte(c): + end := i + for end < len(sql) && isWordByte(sql[end]) { + end++ + } + lastWord = sql[i:end] + out.WriteString(lastWord) + i = end + default: + if !unicode.IsSpace(rune(c)) && c != ',' && c != '(' { + lastWord = "" + } + out.WriteByte(c) + i++ + } + } + return out.String() +} + +func isWordByte(c byte) bool { + return c == '_' || c == '.' || c >= '0' && c <= '9' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' +} + +// skipQuoted returns the index just past the quoted token starting at i, +// honouring backslash escapes and doubled quotes. +func skipQuoted(s string, i int) int { + q := s[i] + j := i + 1 + for j < len(s) { + switch { + case s[j] == '\\' && j+1 < len(s): + j += 2 + case s[j] == q && j+1 < len(s) && s[j+1] == q: + j += 2 + case s[j] == q: + return j + 1 + default: + j++ + } + } + return len(s) +} diff --git a/internal/goldeneye/sqlite/analyze.go b/internal/goldeneye/sqlite/analyze.go new file mode 100644 index 0000000000..b1a8003dff --- /dev/null +++ b/internal/goldeneye/sqlite/analyze.go @@ -0,0 +1,269 @@ +package sqlite + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/analysis" + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" +) + +// The analyze cases are checked against the analysis shell, which is asked +// three things about each query. What the library reports about the +// prepared statement, through `.stats stmt`: each result column's name and, +// for one read straight from a table, its declared type and which table +// column it is, with the column's NOT NULL looked up in the catalog. What +// the query returns, over the case's fixture and again over no rows, for +// the columns the library has no answer for: an expression's type is the +// storage class of its value, since SQLite types values rather than +// expressions, and any column is nullable when a NULL comes back — an +// aggregate over no rows, the far side of an outer join. And the bytecode +// the statement compiles to, which is where the parameters are found, +// since the library reports nothing about them but their number. + +// placeholder is one parameter of a query as sqlc numbers them: each ? in +// turn, and each sqlc.arg name once, at its first appearance. SQLite +// numbers ?NNN the same way, so the query is rewritten with those. +type placeholder struct { + Number int + Name string +} + +func bind(sql string) (string, []placeholder) { + var phs []placeholder + numbers := map[string]int{} + out := endtoend.Rewrite(sql, func(name, _ string) string { + n, ok := numbers[name] + if name == "" || !ok { + n = len(phs) + 1 + phs = append(phs, placeholder{Number: n, Name: name}) + if name != "" { + numbers[name] = n + } + } + return fmt.Sprintf("?%d", n) + }) + return out, phs +} + +// Analyze runs a case's queries through the analysis shell and returns what +// SQLite reports in the JSON shape sqlc analyze prints. +func Analyze(ctx context.Context, dir string, c endtoend.Case) ([]byte, error) { + if err := checkOptions(ctx, dir, analysisShell); err != nil { + return nil, err + } + binary := analysisShell.binary(dir) + schema, err := os.ReadFile(c.Schema) + if err != nil { + return nil, err + } + var fixture []byte + if c.Fixture != "" { + if fixture, err = os.ReadFile(c.Fixture); err != nil { + return nil, err + } + } + queries, err := c.Queries() + if err != nil { + return nil, err + } + out := make([]analysis.Query, 0, len(queries)) + for _, q := range queries { + aq, err := analyzeQuery(ctx, binary, string(schema), string(fixture), q) + if err != nil { + return nil, fmt.Errorf("%s: %w", q.Name, err) + } + out = append(out, aq) + } + return analysis.Encode(out) +} + +// Check compares what SQLite reports for a case with the output the case +// committed, returning a diff when they differ. +func Check(ctx context.Context, dir string, c endtoend.Case) (string, error) { + got, err := Analyze(ctx, dir, c) + if err != nil { + return "", err + } + return c.Compare(got) +} + +func analyzeQuery(ctx context.Context, binary, schema, fixture string, q endtoend.Query) (analysis.Query, error) { + sql, phs := bind(q.SQL) + + // What the library says: the catalog, the bytecode, and the statement + // prepared, with nothing bound to its parameters. + s := newScript() + s.sql(schema) + s.sql(fixture) + s.add(".mode json") + s.add(".explain off") + for _, cq := range catalogQueries { + s.section(cq.section) + s.sql(cq.sql) + } + s.section("explain") + s.sql("EXPLAIN " + sql) + s.section("query") + s.add(".stats stmt") + line := s.sql(sql) + s.section("end") + out, err := run(ctx, binary, s) + if err != nil { + return analysis.Query{}, err + } + if errs := out.errorsBefore(line); len(errs) > 0 { + return analysis.Query{}, errors.New(strings.Join(errs, "\n")) + } + query := out.sections["query"] + if query == nil || !query.prepared { + msg := strings.TrimSpace(out.stderr) + if msg == "" { + msg = "the statement was not prepared" + } + return analysis.Query{}, errors.New(msg) + } + cat, err := readCatalog(out) + if err != nil { + return analysis.Query{}, err + } + var prog []instr + if explain := out.sections["explain"]; explain != nil && len(explain.blocks) > 0 { + if err := explain.decode(0, &prog); err != nil { + return analysis.Query{}, fmt.Errorf("reading the bytecode: %w", err) + } + } + var names []string + for _, m := range query.columns { + names = append(names, m.Name) + } + t := newTracer(cat, names) + t.run(prog) + params := make([]analysis.Column, len(phs)) + for i, ph := range phs { + params[i] = t.param(ph.Number) + } + + // What the statement returns: over the fixture, with each parameter + // bound to a value of the column it stands in for, so that the rows + // come through the WHERE clause, and over no rows. + s = newScript() + s.sql(schema) + s.sql(fixture) + s.add(".parameter init") + for i, ph := range phs { + if v := sample(params[i]); v != "" { + s.sql(fmt.Sprintf("REPLACE INTO temp.sqlite_parameters(key, value) VALUES ('?%d', %s)", ph.Number, v)) + } + } + s.add(".mode quote") + s.section("query") + s.sql(sql) + s.section("end") + bound, err := run(ctx, binary, s) + if err != nil { + return analysis.Query{}, err + } + s = newScript() + s.sql(schema) + s.add(".mode quote") + s.section("query") + s.sql(sql) + s.section("end") + empty, err := run(ctx, binary, s) + if err != nil { + return analysis.Query{}, err + } + var rows []string + for _, o := range []*output{bound, empty} { + if q := o.sections["query"]; q != nil { + rows = append(rows, q.rows...) + } + } + + aq := analysis.Query{ + Name: q.Name, + Cmd: q.Cmd, + Columns: []analysis.Column{}, + Params: []analysis.Param{}, + } + for i, m := range query.columns { + aq.Columns = append(aq.Columns, describeColumn(cat, m, classes(rows, i))) + } + for i, ph := range phs { + ac := params[i] + if ph.Name != "" { + ac.Name = ph.Name + } + aq.Params = append(aq.Params, analysis.Param{Number: ph.Number, Column: ac}) + } + return aq, nil +} + +// sample is an expression for a value to bind to a parameter: one of its +// column's values in the fixture, or a value of its type when all that is +// known is the type. Empty when nothing is known. +func sample(ac analysis.Column) string { + if ac.Table != "" && ac.Name != "" { + col, tbl := quoteIdent(ac.Name), quoteIdent(ac.Table) + return fmt.Sprintf("(SELECT %s FROM %s WHERE %s IS NOT NULL LIMIT 1)", col, tbl, col) + } + if ac.Type == nil { + return "" + } + switch ac.Type.Name { + case "integer", "numeric": + return "1" + case "real": + return "1.0" + case "text": + return "'a'" + case "blob": + return "x'00'" + } + return "" +} + +func quoteIdent(s string) string { + return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` +} + +// classes lists the storage classes of a column's values across rows. +func classes(rows []string, i int) []string { + var out []string + for _, row := range rows { + if c := cells(row); i < len(c) { + out = append(out, storageClass(c[i])) + } + } + return out +} + +// describeColumn describes a result column: as the table column it is read +// from when it is one, otherwise by the storage class of its values, and +// nullable when any of its values was NULL. +func describeColumn(cat *catalog, m columnMeta, classes []string) analysis.Column { + ac := analysis.Column{Name: m.Name} + if col := cat.lookup(m.Table, m.Origin); col != nil { + d := col.describe() + ac.Type, ac.Table = d.Type, d.Table + } else { + for _, class := range classes { + if class != "null" { + ac.Type = &analysis.TypeExpr{Name: class} + break + } + } + } + if ac.Type != nil { + for _, class := range classes { + if class == "null" { + ac.Type.Nullable = true + } + } + } + return ac +} diff --git a/internal/goldeneye/sqlite/bytecode.go b/internal/goldeneye/sqlite/bytecode.go new file mode 100644 index 0000000000..2a0e0dd281 --- /dev/null +++ b/internal/goldeneye/sqlite/bytecode.go @@ -0,0 +1,456 @@ +package sqlite + +import ( + "strconv" + "strings" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/analysis" +) + +// SQLite has nothing to say about a parameter's type: a bound value is +// whatever it is, and the column a parameter stands in for appears nowhere +// in what the library reports about a prepared statement. What it does +// report is the program it compiled the statement into. EXPLAIN prints the +// bytecode, in which a parameter is a Variable loading a register, and the +// column it stands in for is the other operand of the comparison the +// register reaches, the row the seek it drives lands on, or the position +// in the record that Insert writes. So each parameter is followed from the +// register it loads, through copies and through the expressions it is an +// argument of, to the first opcode that uses it against something the +// catalog can name, the way the ClickHouse check follows a placeholder +// through the query tree to the column it is compared with. +// +// Registers hold what the last opcode to write them wrote, and remember +// which parameters have been loaded into them, which a later constant does +// not clear: coalesce(?, 'x') overwrites the parameter's register with the +// fallback on the branch where the parameter is NULL, and the comparison +// that follows is still the parameter's. A parameter with no such use is +// described by what the program does to it before using it, when it does +// anything: MustBeInt says it has to be an integer, as LIMIT's does, and +// Affinity says which affinity it is coerced to. + +// instr is one line of EXPLAIN output. +type instr struct { + Addr int `json:"addr"` + Opcode string `json:"opcode"` + P1 int `json:"p1"` + P2 int `json:"p2"` + P3 int `json:"p3"` + P4 *string `json:"p4"` + P5 int `json:"p5"` +} + +func (in instr) p4() string { + if in.P4 == nil { + return "" + } + return *in.P4 +} + +type valueKind int + +const ( + vNone valueKind = iota + vParam // a Variable + vColumn // Column: a stored column of a cursor + vRowid // Rowid, IdxRowid: the rowid of a cursor + vConstant // a literal, with its storage class + vFunction // the result of a function call, with its name + vExpr // the result of an operator + vRecord // MakeRecord: the registers packed into a record +) + +// value is what a register holds. +type value struct { + kind valueKind + cursor int + index int + class string // vConstant + fn string // vFunction + regs []int // vRecord +} + +type register struct { + desc value + params []int // every parameter loaded into the register +} + +// tracer follows the parameters of one statement through its bytecode. +type tracer struct { + cat *catalog + names []string // the statement's result columns + // cursors is what each open cursor is on; ephemeral holds, for the + // cursors on ephemeral tables and sorters, the parameters stored in + // each column, so that a parameter put into an IN list's table comes + // back out with the Column that reads it. + cursors map[int]object + ephemeral map[int]map[int][]int + regs map[int]*register + // found is the column each parameter was found to stand in for, and + // hint what the program said about a parameter's type. + found map[int]analysis.Column + hint map[int]string +} + +func newTracer(cat *catalog, names []string) *tracer { + return &tracer{ + cat: cat, + names: names, + cursors: map[int]object{}, + ephemeral: map[int]map[int][]int{}, + regs: map[int]*register{}, + found: map[int]analysis.Column{}, + hint: map[int]string{}, + } +} + +// run walks the program in the order it runs: the prologue Init jumps to, +// where the Variables of a query are loaded, and then the body. +func (t *tracer) run(prog []instr) { + order := prog + if len(prog) > 0 && prog[0].Opcode == "Init" && prog[0].P2 > 0 && prog[0].P2 < len(prog) { + order = append(append([]instr{}, prog[prog[0].P2:]...), prog[1:prog[0].P2]...) + } + for _, in := range order { + t.step(in) + } +} + +// param describes what the parameter was found to stand in for. +func (t *tracer) param(number int) analysis.Column { + if ac, ok := t.found[number]; ok { + return ac + } + if typ := t.hint[number]; typ != "" { + return analysis.Column{Type: &analysis.TypeExpr{Name: typ}} + } + return analysis.Column{} +} + +func (t *tracer) reg(n int) *register { + r := t.regs[n] + if r == nil { + r = ®ister{} + t.regs[n] = r + } + return r +} + +func (t *tracer) params(n int) []int { + if r := t.regs[n]; r != nil { + return r.params + } + return nil +} + +// set records a write to a register. A Variable is the parameter and +// nothing else; a column read replaces whatever the register held, since +// the compiler reuses registers for the columns it reads; a constant keeps +// the parameters, for the coalesce case above; anything else carries the +// parameters it was computed from. +func (t *tracer) set(n int, v value, params []int) { + r := t.reg(n) + r.desc = v + switch v.kind { + case vConstant: + default: + r.params = params + } +} + +func (t *tracer) copy(from, to int) { + src := t.reg(from) + dst := t.reg(to) + dst.desc = src.desc + dst.params = append([]int{}, src.params...) +} + +// assoc records the column the parameters in a register stand in for, +// keeping the first found for each. +func (t *tracer) assoc(params []int, ac analysis.Column) { + for _, p := range params { + if _, ok := t.found[p]; !ok { + t.found[p] = ac + } + } +} + +// typeHint records what the program requires of a register's parameters. +func (t *tracer) typeHint(n int, typ string) { + if typ == "" { + return + } + for _, p := range t.params(n) { + if _, ok := t.hint[p]; !ok { + t.hint[p] = typ + } + } +} + +// describe turns what a register holds into the column a parameter +// compared with it stands in for, when it can be named: a stored column +// or rowid, a constant's storage class, or a function's name. +func (t *tracer) describe(v value) (analysis.Column, bool) { + switch v.kind { + case vColumn: + return t.stored(v.cursor, v.index) + case vRowid: + if owner := t.cursors[v.cursor].owner(); owner != nil { + return owner.rowid(), true + } + case vConstant: + if v.class != "null" && v.class != "" { + return analysis.Column{Type: &analysis.TypeExpr{Name: v.class}}, true + } + case vFunction: + return analysis.Column{Name: v.fn}, true + } + return analysis.Column{}, false +} + +// stored describes the ith stored column of a cursor. +func (t *tracer) stored(cursor, i int) (analysis.Column, bool) { + col, owner, ok := t.cursors[cursor].column(i) + switch { + case !ok: + return analysis.Column{}, false + case col != nil: + return col.describe(), true + default: + return owner.rowid(), true + } +} + +// rank orders what a parameter's fellow operands may be described by: a +// column first, then a function, then a constant. +func rank(v value) int { + switch v.kind { + case vColumn, vRowid: + return 0 + case vFunction: + return 1 + case vConstant: + if v.class != "null" && v.class != "" { + return 2 + } + } + return -1 +} + +// expression handles an opcode computing a result from operand registers: +// each parameter among the operands is described by the best other +// operand, and one with no other operand to be described by is carried +// into the result, for whatever the result is then compared with. +func (t *tracer) expression(args []int, dest int, result value) { + var carried []int + for _, a := range args { + params := t.params(a) + if len(params) == 0 { + continue + } + best := -1 + for _, b := range args { + if b == a { + continue + } + if r := rank(t.reg(b).desc); r >= 0 && (best < 0 || r < rank(t.reg(best).desc)) { + best = b + } + } + if best >= 0 { + if ac, ok := t.describe(t.reg(best).desc); ok { + t.assoc(params, ac) + continue + } + } + carried = append(carried, params...) + } + t.set(dest, result, carried) +} + +// compare handles a comparison of two registers: the parameters of each +// stand in for what the other holds. +func (t *tracer) compare(a, b int) { + if ac, ok := t.describe(t.reg(b).desc); ok { + t.assoc(t.params(a), ac) + } + if ac, ok := t.describe(t.reg(a).desc); ok { + t.assoc(t.params(b), ac) + } +} + +// store handles a register written as the ith column of a cursor's row: +// an ephemeral table remembers the parameters for the reads to come, and +// a real one is the column the parameters stand in for. +func (t *tracer) store(cursor, i, reg int) { + params := t.params(reg) + if len(params) == 0 { + return + } + if cols, ok := t.ephemeral[cursor]; ok { + cols[i] = append(cols[i], params...) + return + } + if ac, ok := t.stored(cursor, i); ok { + t.assoc(params, ac) + } +} + +// key handles registers used as a key into a cursor: n registers from +// start, or the record in start when n is zero. +func (t *tracer) key(cursor, start, n int) { + regs := t.span(start, n) + if n == 0 { + if r := t.reg(start); r.desc.kind == vRecord { + regs = r.desc.regs + } + } + for i, reg := range regs { + t.store(cursor, i, reg) + } +} + +func (t *tracer) span(start, n int) []int { + regs := make([]int, 0, max(n, 0)) + for i := 0; i < n; i++ { + regs = append(regs, start+i) + } + return regs +} + +// affinity names the type an affinity character coerces to. +func affinity(c byte) string { + switch c { + case 'A': + return "blob" + case 'B': + return "text" + case 'C': + return "numeric" + case 'D': + return "integer" + case 'E': + return "real" + } + return "" +} + +func (t *tracer) step(in instr) { + switch in.Opcode { + case "OpenRead", "OpenWrite", "ReopenIdx": + t.cursors[in.P1] = t.cat.root(in.P3, in.P2) + case "OpenDup": + t.cursors[in.P1] = t.cursors[in.P2] + if cols, ok := t.ephemeral[in.P2]; ok { + t.ephemeral[in.P1] = cols + } + case "OpenEphemeral", "OpenAutoindex", "SorterOpen", "OpenPseudo": + t.cursors[in.P1] = object{} + t.ephemeral[in.P1] = map[int][]int{} + case "Variable": + t.set(in.P2, value{kind: vParam}, []int{in.P1}) + case "Column", "VColumn": + t.set(in.P3, value{kind: vColumn, cursor: in.P1, index: in.P2}, t.ephemeral[in.P1][in.P2]) + case "Rowid", "IdxRowid": + t.set(in.P2, value{kind: vRowid, cursor: in.P1}, nil) + case "Copy": + for i := 0; i <= in.P3; i++ { + t.copy(in.P1+i, in.P2+i) + } + case "SCopy", "IntCopy": + t.copy(in.P1, in.P2) + case "Null", "SoftNull": + end := max(in.P2, in.P3) + if in.Opcode == "SoftNull" { + end = in.P1 + in.P2 = in.P1 + } + for r := in.P2; r <= end; r++ { + t.set(r, value{kind: vConstant, class: "null"}, nil) + } + case "Integer", "Int64": + t.set(in.P2, value{kind: vConstant, class: "integer"}, nil) + case "Real": + t.set(in.P2, value{kind: vConstant, class: "real"}, nil) + case "String8", "String": + t.set(in.P2, value{kind: vConstant, class: "text"}, nil) + case "Blob": + t.set(in.P2, value{kind: vConstant, class: "blob"}, nil) + case "Function", "PureFunc": + // P4 prints the function's arity, which for a variadic function + // is negative and says nothing about this call. Its arguments + // are the registers from P2 written so far, allocated together + // for the call. + name, n := parseCall(in.p4()) + if n < 0 { + n = 0 + for t.regs[in.P2+n] != nil { + n++ + } + } + t.expression(t.span(in.P2, n), in.P3, value{kind: vFunction, fn: name}) + case "Add", "Subtract", "Multiply", "Divide", "Remainder", "Concat", "BitAnd", "BitOr", "ShiftLeft", "ShiftRight": + t.expression([]int{in.P2, in.P1}, in.P3, value{kind: vExpr}) + case "MakeRecord": + t.set(in.P3, value{kind: vRecord, regs: t.span(in.P1, in.P2)}, nil) + case "Insert": + if r := t.reg(in.P2); r.desc.kind == vRecord { + for i, reg := range r.desc.regs { + t.store(in.P1, i, reg) + } + } + if owner := t.cursors[in.P1].owner(); owner != nil { + t.assoc(t.params(in.P3), owner.rowid()) + } + case "IdxInsert", "SorterInsert": + n, _ := strconv.Atoi(in.p4()) + if n > 0 { + t.key(in.P1, in.P3, n) + } else { + t.key(in.P1, in.P2, 0) + } + case "Eq", "Ne", "Lt", "Le", "Gt", "Ge": + t.compare(in.P1, in.P3) + case "SeekRowid", "NotExists": + if owner := t.cursors[in.P1].owner(); owner != nil { + t.assoc(t.params(in.P3), owner.rowid()) + } + case "SeekGE", "SeekGT", "SeekLE", "SeekLT": + // On a table with a rowid the seek is by the rowid in P3; on an + // index it is by the P4 registers from P3. + if o := t.cursors[in.P1]; o.table != nil && !o.table.withoutRowid { + t.assoc(t.params(in.P3), o.table.rowid()) + break + } + n, _ := strconv.Atoi(in.p4()) + t.key(in.P1, in.P3, n) + case "IdxGE", "IdxGT", "IdxLE", "IdxLT", "Found", "NotFound", "NoConflict": + n, _ := strconv.Atoi(in.p4()) + t.key(in.P1, in.P3, n) + case "MustBeInt": + t.typeHint(in.P1, "integer") + case "Affinity": + for i, c := range []byte(in.p4()) { + if i < in.P2 { + t.typeHint(in.P1+i, affinity(c)) + } + } + case "ResultRow": + for i := 0; i < in.P2; i++ { + if i < len(t.names) { + t.assoc(t.params(in.P1+i), analysis.Column{Name: t.names[i]}) + } + } + } +} + +// parseCall splits the `name(n)` a Function's P4 prints into the function's +// name and the number of arguments it is called with. +func parseCall(p4 string) (string, int) { + open := strings.LastIndexByte(p4, '(') + if open < 0 || !strings.HasSuffix(p4, ")") { + return p4, 0 + } + n, _ := strconv.Atoi(p4[open+1 : len(p4)-1]) + return p4[:open], n +} diff --git a/internal/goldeneye/sqlite/catalog.go b/internal/goldeneye/sqlite/catalog.go new file mode 100644 index 0000000000..8077543325 --- /dev/null +++ b/internal/goldeneye/sqlite/catalog.go @@ -0,0 +1,294 @@ +package sqlite + +import ( + "fmt" + "strconv" + "strings" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/analysis" +) + +// catalog is what the database says about a schema: its tables and their +// columns, its indexes, and the b-tree each is stored in, which is how the +// bytecode names them. +type catalog struct { + tables map[string]*table + roots map[int]object // by root page +} + +type table struct { + name string + withoutRowid bool + cols []*tableColumn // by cid + // stored lists the columns in the order a row's record holds them, + // which is cid order less the generated columns that are not stored. + stored []*tableColumn + // pk is the PRIMARY KEY index of a WITHOUT ROWID table, whose order is + // the order the table's own b-tree stores a row in. + pk *index +} + +type tableColumn struct { + table *table + cid int + name string + declType string + notNull bool + pk int // position in the primary key, 0 for none +} + +type index struct { + name string + table *table + cids []int // by seqno: -1 for the rowid, -2 for an expression +} + +// object is what a cursor is open on: a table, an index, or, with neither +// set, something the catalog does not describe — an ephemeral table, a +// sorter, a virtual table. +type object struct { + table *table + index *index +} + +// The catalog is read with these, each in a section of its own, in json +// mode. A query that finds nothing prints nothing, so a section may be +// empty. +var catalogQueries = []struct{ section, sql string }{ + {"tables", `SELECT name, type, wr FROM pragma_table_list WHERE schema = 'main' ORDER BY name`}, + {"columns", `SELECT t.name AS tbl, c.cid, c.name, c.type, c."notnull" AS "notnull", c.pk, c.hidden + FROM pragma_table_list t, pragma_table_xinfo(t.name) c + WHERE t.schema = 'main' AND t.type IN ('table', 'virtual') ORDER BY t.name, c.cid`}, + {"indexes", `SELECT t.name AS tbl, i.name AS idx, i.origin, x.seqno, x.cid + FROM pragma_table_list t, pragma_index_list(t.name) i, pragma_index_xinfo(i.name) x + WHERE t.schema = 'main' ORDER BY t.name, i.name, x.seqno`}, + {"roots", `SELECT type, name, tbl_name, rootpage FROM sqlite_schema WHERE rootpage > 0`}, +} + +// readCatalog builds the catalog from the sections catalogQueries printed. +func readCatalog(out *output) (*catalog, error) { + var tables []struct { + Name string `json:"name"` + Type string `json:"type"` + WR int `json:"wr"` + } + var columns []struct { + Table string `json:"tbl"` + CID int `json:"cid"` + Name string `json:"name"` + Type string `json:"type"` + NotNull int `json:"notnull"` + PK int `json:"pk"` + Hidden int `json:"hidden"` + } + var indexes []struct { + Table string `json:"tbl"` + Index string `json:"idx"` + Origin string `json:"origin"` + Seqno int `json:"seqno"` + CID int `json:"cid"` + } + var roots []struct { + Type string `json:"type"` + Name string `json:"name"` + Table string `json:"tbl_name"` + Rootpage int `json:"rootpage"` + } + for name, v := range map[string]any{"tables": &tables, "columns": &columns, "indexes": &indexes, "roots": &roots} { + s := out.sections[name] + if s == nil { + return nil, fmt.Errorf("reading the catalog: no %s section in the output", name) + } + if len(s.blocks) > 0 { + if err := s.decode(0, v); err != nil { + return nil, fmt.Errorf("reading the catalog's %s: %w", name, err) + } + } + } + + c := &catalog{tables: map[string]*table{}, roots: map[int]object{}} + for _, t := range tables { + if t.Type == "table" || t.Type == "virtual" { + c.tables[t.Name] = &table{name: t.Name, withoutRowid: t.WR == 1} + } + } + for _, col := range columns { + t := c.tables[col.Table] + if t == nil { + continue + } + tc := &tableColumn{table: t, cid: col.CID, name: col.Name, declType: col.Type, notNull: col.NotNull == 1, pk: col.PK} + for len(t.cols) <= col.CID { + t.cols = append(t.cols, nil) + } + t.cols[col.CID] = tc + // A generated column that is not stored — hidden is 2 — has no + // place in the record. + if col.Hidden != 2 { + t.stored = append(t.stored, tc) + } + } + byName := map[string]*index{} + for _, ic := range indexes { + t := c.tables[ic.Table] + if t == nil { + continue + } + ix := byName[ic.Index] + if ix == nil { + ix = &index{name: ic.Index, table: t} + byName[ic.Index] = ix + if ic.Origin == "pk" && t.withoutRowid { + t.pk = ix + } + } + for len(ix.cids) <= ic.Seqno { + ix.cids = append(ix.cids, -2) + } + ix.cids[ic.Seqno] = ic.CID + } + for _, r := range roots { + switch r.Type { + case "table": + if t := c.tables[r.Name]; t != nil { + c.roots[r.Rootpage] = object{table: t} + } + case "index": + if ix := byName[r.Name]; ix != nil { + c.roots[r.Rootpage] = object{index: ix} + } + } + } + return c, nil +} + +// root returns what is stored in a b-tree of the main database. +func (c *catalog) root(db, page int) object { + if db != 0 { + return object{} + } + return c.roots[page] +} + +// lookup finds a table column by the names `.stats stmt` prints. +func (c *catalog) lookup(tbl, col string) *tableColumn { + t := c.tables[tbl] + if t == nil { + return nil + } + for _, tc := range t.cols { + if tc != nil && tc.name == col { + return tc + } + } + return nil +} + +// rowidAlias returns the column that is another name for the rowid: the +// lone INTEGER PRIMARY KEY of a table that has a rowid. +func (t *table) rowidAlias() *tableColumn { + if t.withoutRowid { + return nil + } + var alias *tableColumn + for _, tc := range t.cols { + if tc == nil || tc.pk == 0 { + continue + } + if alias != nil || !strings.EqualFold(tc.declType, "INTEGER") { + return nil + } + alias = tc + } + return alias +} + +// describe is the column as sqlc analyze describes one: its declared type, +// nullable unless declared NOT NULL or the rowid, which is never NULL. +func (tc *tableColumn) describe() analysis.Column { + typ := parseType(tc.declType) + typ.Nullable = !tc.notNull && tc.table.rowidAlias() != tc + return analysis.Column{Name: tc.name, Type: typ, Table: tc.table.name} +} + +// rowid describes the table's rowid: the column that aliases it, or the +// rowid itself. +func (t *table) rowid() analysis.Column { + if alias := t.rowidAlias(); alias != nil { + return alias.describe() + } + return analysis.Column{Name: "rowid", Type: &analysis.TypeExpr{Name: "integer"}, Table: t.name} +} + +// column returns what the object's ith stored column is: a table column, +// or the rowid of a table. An index stores the columns it is on and then +// the rowid; a WITHOUT ROWID table is stored in the order of its primary +// key; any other table is stored in column order. +func (o object) column(i int) (*tableColumn, *table, bool) { + ix := o.index + if o.table != nil && o.table.withoutRowid { + ix = o.table.pk + } + switch { + case ix != nil: + if i < 0 || i >= len(ix.cids) { + return nil, nil, false + } + cid := ix.cids[i] + switch { + case cid == -1: + return nil, ix.table, true + case cid >= 0 && cid < len(ix.table.cols) && ix.table.cols[cid] != nil: + return ix.table.cols[cid], nil, true + } + case o.table != nil: + if i >= 0 && i < len(o.table.stored) { + return o.table.stored[i], nil, true + } + } + return nil, nil, false +} + +// owner is the table the object stores rows of. +func (o object) owner() *table { + if o.index != nil { + return o.index.table + } + return o.table +} + +// parseType reads a declared type the way sqlc's catalog does: the name +// lowercased, with whatever is in parentheses after it as arguments. A +// column declared with no type at all can hold anything. +func parseType(decl string) *analysis.TypeExpr { + decl = strings.TrimSpace(decl) + if decl == "" { + return &analysis.TypeExpr{Name: "any"} + } + name, args := decl, "" + if open := strings.IndexByte(decl, '('); open >= 0 && strings.HasSuffix(decl, ")") { + name, args = decl[:open], decl[open+1:len(decl)-1] + } + t := &analysis.TypeExpr{Name: strings.ToLower(strings.TrimSpace(name))} + if strings.TrimSpace(args) == "" { + return t + } + for _, a := range strings.Split(args, ",") { + a = strings.TrimSpace(a) + switch { + case strings.HasPrefix(a, "'") && strings.HasSuffix(a, "'") && len(a) >= 2: + s := strings.ReplaceAll(a[1:len(a)-1], "''", "'") + t.Args = append(t.Args, analysis.TypeArg{String: &s}) + case strings.EqualFold(a, "true") || strings.EqualFold(a, "false"): + b := strings.EqualFold(a, "true") + t.Args = append(t.Args, analysis.TypeArg{Bool: &b}) + default: + if n, err := strconv.ParseInt(a, 10, 64); err == nil { + t.Args = append(t.Args, analysis.TypeArg{Int: &n}) + } else { + t.Args = append(t.Args, analysis.TypeArg{Type: &analysis.TypeExpr{Name: strings.ToLower(a)}}) + } + } + } + return t +} diff --git a/internal/goldeneye/sqlite/install.go b/internal/goldeneye/sqlite/install.go index 0a0ed1d674..d09a38013a 100644 --- a/internal/goldeneye/sqlite/install.go +++ b/internal/goldeneye/sqlite/install.go @@ -13,6 +13,7 @@ import ( "os/exec" "path/filepath" "runtime" + "slices" "strings" "sync" ) @@ -50,6 +51,26 @@ var extensions = []build{ {"enable_rtree", []string{"SQLITE_ENABLE_RTREE"}}, } +// analysisShell is the shell the analyze cases run through, which is no +// dialect build: nothing is generated from it. It is built with column +// metadata, so that `.stats stmt` can say which table column each result +// column of a statement comes from, and with every extension option at +// once, so that whatever a case's schema asks for is there. +var analysisShell = build{"analysis", append([]string{"SQLITE_ENABLE_COLUMN_METADATA"}, extensionOptions()...)} + +// extensionOptions is every option an extension build turns on, once each. +func extensionOptions() []string { + var opts []string + for _, b := range extensions { + for _, opt := range b.options { + if !slices.Contains(opts, opt) { + opts = append(opts, opt) + } + } + } + return opts +} + // asset is one downloadable amalgamation of SQLite: the zip published on // sqlite.org holding sqlite3.c, sqlite3.h and the shell's shell.c. type asset struct { @@ -87,7 +108,8 @@ func (a asset) url() string { } // cacheDir is where Install puts a version: the sources under src/, and one -// shell per build under default/ and under each option's extension name. +// shell per build under default/, under each option's extension name and +// under analysis/. func cacheDir(version string) (string, error) { dir, err := os.UserCacheDir() if err != nil { @@ -105,11 +127,18 @@ type build struct { options []string } -// builds lists the default build first, then one per extension. +// builds lists the dialect builds: the default build first, then one per +// extension. func builds() []build { return append([]build{{"default", nil}}, extensions...) } +// shells lists every build Install makes: the dialect builds and the +// analysis shell. +func shells() []build { + return append(builds(), analysisShell) +} + // flags are every option a build is compiled with. func (b build) flags() []string { return append(append([]string{}, defaultOptions...), b.options...) @@ -125,7 +154,7 @@ func Locate() (string, error) { if err != nil { return "", err } - for _, b := range builds() { + for _, b := range shells() { if _, err := os.Stat(b.binary(dir)); err != nil { return "", fmt.Errorf("sqlite %s is not built with %s: run `go run ./cmd/goldeneye install sqlite` in internal/goldeneye", DefaultVersion, strings.Join(b.flags(), " ")) } @@ -153,7 +182,7 @@ func Install(ctx context.Context, version, goos, goarch string, progress io.Writ return "", err } var missing []build - for _, b := range builds() { + for _, b := range shells() { if _, err := os.Stat(b.binary(dir)); err != nil { missing = append(missing, b) } diff --git a/internal/goldeneye/sqlite/shell.go b/internal/goldeneye/sqlite/shell.go new file mode 100644 index 0000000000..f17b9333ae --- /dev/null +++ b/internal/goldeneye/sqlite/shell.go @@ -0,0 +1,254 @@ +package sqlite + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" +) + +// The analysis shell is driven with a script on its standard input, made +// of SQL and of the shell's own dot-commands: `.print` labels the sections +// of the output so that each statement's results can be found again, +// `.mode` chooses how rows print — json for the catalog and the bytecode, +// quote for a query's own rows, since quote mode is the one that tells a +// blob from a string and a real from an integer — and `.stats stmt` makes +// the shell print what it knows about each result column of the statement +// it has just run, which with column metadata compiled in is the column's +// name, declared type, database, table and origin column. + +// script is a shell script under construction, counting its lines so that +// an error the shell reports "near line N" can be placed. +type script struct { + buf strings.Builder + line int // the number of the next line written +} + +func newScript() *script { + return &script{line: 1} +} + +// add writes text on lines of its own, and returns the line it starts on. +func (s *script) add(text string) int { + start := s.line + text = strings.TrimSpace(text) + if text == "" { + return start + } + s.buf.WriteString(text) + s.buf.WriteByte('\n') + s.line += strings.Count(text, "\n") + 1 + return start +} + +// sql writes statements, ending them with a semicolon when they lack one. +func (s *script) sql(text string) int { + text = strings.TrimRight(strings.TrimSpace(text), "; \t\r\n") + if text == "" { + return s.line + } + return s.add(text + ";") +} + +// section labels what follows, up to the next label. +func (s *script) section(name string) { + s.add(".print @@" + name) +} + +// columnMeta is what `.stats stmt` prints about one result column. Table +// and Origin name the table column a result column is read from, and are +// empty for an expression. +type columnMeta struct { + Name string + DeclType string + Database string + Table string + Origin string +} + +// section is the output between two labels. +type section struct { + // blocks holds every JSON result set printed in the section. + blocks []json.RawMessage + // rows holds every other line printed before the statement's + // statistics: in quote mode, one row each. + rows []string + // prepared says a statement's statistics were printed, which the shell + // does only for a statement it could prepare. + prepared bool + columns []columnMeta +} + +// decode reads the section's nth JSON result set. +func (s *section) decode(n int, v any) error { + if n >= len(s.blocks) { + return fmt.Errorf("expected at least %d result set(s), got %d", n+1, len(s.blocks)) + } + return json.Unmarshal(s.blocks[n], v) +} + +// output is what one run of the shell printed, by section. +type output struct { + sections map[string]*section + stderr string +} + +var errLineRe = regexp.MustCompile(`near line (\d+):`) + +// errorsBefore returns the errors the shell reported for statements before +// the given line: the ones that are not the statement under analysis +// failing to run, which it may, with nothing bound to its parameters. +func (o *output) errorsBefore(line int) []string { + var errs []string + for _, l := range strings.Split(o.stderr, "\n") { + if m := errLineRe.FindStringSubmatch(l); m != nil { + if n, _ := strconv.Atoi(m[1]); n < line { + errs = append(errs, strings.TrimSpace(l)) + } + } + } + return errs +} + +// run feeds a script to the shell over a fresh in-memory database and +// parses what it printed. A statement that fails does not stop the shell, +// so a failure is read from the output: a section whose statement has no +// statistics was not prepared, and stderr says why. +func run(ctx context.Context, binary string, s *script) (*output, error) { + cmd := exec.CommandContext(ctx, binary, ":memory:") + cmd.Stdin = strings.NewReader(s.buf.String()) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + var exit *exec.ExitError + if !errors.As(err, &exit) { + return nil, fmt.Errorf("sqlite3: %w", err) + } + } + out := &output{sections: map[string]*section{}, stderr: stderr.String()} + var ( + cur *section + block []string // lines of the JSON result set being read + row []string // lines of the quote-mode row being read + ) + for _, line := range strings.Split(stdout.String(), "\n") { + switch { + case block != nil: + block = append(block, line) + if strings.HasSuffix(line, "]") { + cur.blocks = append(cur.blocks, json.RawMessage(strings.Join(block, "\n"))) + block = nil + } + case row != nil: + row = append(row, line) + if joined := strings.Join(row, "\n"); strings.Count(joined, "'")%2 == 0 { + cur.rows = append(cur.rows, joined) + row = nil + } + case strings.HasPrefix(line, "@@"): + cur = §ion{} + out.sections[line[2:]] = cur + case cur == nil || line == "": + case strings.HasPrefix(line, "["): + if strings.HasSuffix(line, "]") { + cur.blocks = append(cur.blocks, json.RawMessage(line)) + } else { + block = []string{line} + } + case strings.HasPrefix(line, "Number of output columns:"): + cur.prepared = true + case cur.prepared: + if i, field, value, ok := statLine(line); ok { + for len(cur.columns) <= i { + cur.columns = append(cur.columns, columnMeta{}) + } + c := &cur.columns[i] + switch field { + case "name": + c.Name = value + case "declared type": + c.DeclType = value + case "database name": + c.Database = value + case "table name": + c.Table = value + case "origin name": + c.Origin = value + } + } + default: + // A quote-mode row, complete unless a string in it spans lines. + if strings.Count(line, "'")%2 == 0 { + cur.rows = append(cur.rows, line) + } else { + row = []string{line} + } + } + } + return out, nil +} + +var statLineRe = regexp.MustCompile(`^Column (\d+) ([a-z ]+):`) + +// statLine reads one line of `.stats stmt` column output, which the shell +// prints as the label padded to 36 columns, a space and the value; a value +// the library has no answer for prints as the C library prints a null +// string. +func statLine(line string) (int, string, string, bool) { + m := statLineRe.FindStringSubmatch(line) + if m == nil { + return 0, "", "", false + } + i, _ := strconv.Atoi(m[1]) + label := m[0] + value := "" + if start := max(len(label), 36) + 1; start < len(line) { + value = line[start:] + } + if value == "(null)" { + value = "" + } + return i, m[2], value, true +} + +// cells splits a quote-mode row into its values as printed. +func cells(row string) []string { + var out []string + start := 0 + quoted := false + for i := 0; i < len(row); i++ { + switch { + case row[i] == '\'': + quoted = !quoted + case row[i] == ',' && !quoted: + out = append(out, row[start:i]) + start = i + 1 + } + } + return append(out, row[start:]) +} + +// storageClass reads the storage class of a value as quote mode prints +// it: a string is quoted, a blob is quoted with an X, NULL is spelled +// out, and a real has a point, an exponent or is infinite or not a number +// where an integer is digits. +func storageClass(cell string) string { + switch { + case cell == "NULL": + return "null" + case strings.HasPrefix(cell, "'"): + return "text" + case strings.HasPrefix(cell, "x'") || strings.HasPrefix(cell, "X'"): + return "blob" + case strings.ContainsAny(cell, ".eEIN"): + return "real" + default: + return "integer" + } +} diff --git a/internal/goldeneye/sqlite/signatures.go b/internal/goldeneye/sqlite/signatures.go index 121bf36507..5551061bd5 100644 --- a/internal/goldeneye/sqlite/signatures.go +++ b/internal/goldeneye/sqlite/signatures.go @@ -10,10 +10,14 @@ import "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" // many arguments an overload takes, and how many of them it requires, // comes from the shell. Nullable is decided afterwards: for an aggregate by // running it over no rows, for a scalar by the nullable list below. +// Numeric says the function returns an integer for an integer argument +// and a real for a real one, and Returns what it returns for anything +// else. type signature struct { Args []string Variadic string Returns string + Numeric bool Nullable bool } @@ -57,6 +61,14 @@ var inlineReturns = map[string]string{ "INLINEFUNC_sqlite_offset": "integer", } +// realResults are the functions that return an integer or a real on +// something other than which their argument is, and so are typed real +// alone: unixepoch takes a time string and returns a real only with the +// 'subsec' modifier. +var realResults = map[string]bool{ + "unixepoch": true, +} + // omitted are functions the dialect leaves out: ones that exist for their // side effect and return nothing a query can use, and ones an extension // uses to pass pointers to itself. diff --git a/internal/goldeneye/sqlite/source.go b/internal/goldeneye/sqlite/source.go index 1599e839f9..d7c551891e 100644 --- a/internal/goldeneye/sqlite/source.go +++ b/internal/goldeneye/sqlite/source.go @@ -441,10 +441,13 @@ func single(kinds map[string]bool) string { // signature derives what the source says a SQL function returns and takes. // A result of one kind is that type. A function that returns one of its // arguments, or a mixture of kinds, takes the type of its first argument, -// which the seed spells "any" — except that integer and real together widen -// to real, as SQLite's own arithmetic does, and text and blob together to -// text, since a function that returns either is handing back the bytes it -// was given, and the legacy compiler cannot follow "any" to an argument. +// which the seed spells "any" — except that integer and real together are +// real, and Numeric, since abs, ceil and sum hand back an integer for an +// integer and a real for a real, and a real for the text or blob they +// convert, unless the function is one of the few whose choice turns on +// something else; and text and blob together are text, since a function +// that returns either is handing back the bytes it was given, and the +// legacy compiler cannot follow "any" to an argument. func (s *source) signature(name string) (signature, error) { r, ok := s.regs[strings.ToLower(name)] if !ok { @@ -482,6 +485,7 @@ func (s *source) signature(name string) (signature, error) { sig.Returns = r.json case len(kinds) == 2 && kinds["integer"] && kinds["real"]: sig.Returns = "real" + sig.Numeric = !realResults[strings.ToLower(name)] case len(kinds) == 2 && kinds["text"] && kinds["blob"]: sig.Returns = "text" default: diff --git a/internal/goldeneye/sqlite/sqlite.go b/internal/goldeneye/sqlite/sqlite.go index d8aa1b62ba..c176a55b36 100644 --- a/internal/goldeneye/sqlite/sqlite.go +++ b/internal/goldeneye/sqlite/sqlite.go @@ -181,12 +181,25 @@ type shell struct { // option lists changed would otherwise describe the wrong dialect. func readShell(ctx context.Context, dir string, b build) (*shell, error) { s := &shell{build: b, binary: b.binary(dir)} + if err := checkOptions(ctx, dir, b); err != nil { + return nil, err + } + if err := query(ctx, s.binary, functionList, &s.rows); err != nil { + return nil, err + } + return s, nil +} + +// checkOptions asks a build's shell whether it was compiled with every +// option any build turns on, and fails when the answers do not match the +// build's own list. +func checkOptions(ctx context.Context, dir string, b build) error { var used []struct { Option string `json:"option"` Used int `json:"used"` } known := map[string]bool{} - for _, b := range builds() { + for _, b := range shells() { for _, opt := range b.flags() { known[opt] = true } @@ -195,8 +208,8 @@ func readShell(ctx context.Context, dir string, b build) (*shell, error) { for _, opt := range slices.Sorted(maps.Keys(known)) { clauses = append(clauses, fmt.Sprintf("SELECT '%s' AS option, sqlite_compileoption_used('%s') AS used", opt, opt)) } - if err := query(ctx, s.binary, strings.Join(clauses, " UNION ALL "), &used); err != nil { - return nil, err + if err := query(ctx, b.binary(dir), strings.Join(clauses, " UNION ALL "), &used); err != nil { + return err } for _, u := range used { want := 0 @@ -204,13 +217,10 @@ func readShell(ctx context.Context, dir string, b build) (*shell, error) { want = 1 } if u.Used != want { - return nil, fmt.Errorf("sqlite: the %s shell was not built with the options it should have been (%s is %d): remove %s and run `go run ./cmd/goldeneye install sqlite` again", b.name, u.Option, u.Used, dir) + return fmt.Errorf("sqlite: the %s shell was not built with the options it should have been (%s is %d): remove %s and run `go run ./cmd/goldeneye install sqlite` again", b.name, u.Option, u.Used, filepath.Join(dir, b.name)) } } - if err := query(ctx, s.binary, functionList, &s.rows); err != nil { - return nil, err - } - return s, nil + return nil } // generator accumulates the functions of every build, reading their @@ -221,6 +231,9 @@ type generator struct { ctx context.Context src *source reported map[string]bool + // numeric holds every spelling types.jsonl gives integer and real, in + // its order, for the functions written once per spelling. + numeric map[string][]string } // overload is one row of a shell's list with what was found out about it. @@ -273,13 +286,30 @@ func (g *generator) functions(s *shell, base bool) ([]dialect.Function, error) { if o.kind == "a" { isNullable = empty[o.row.key()] == "null" } - funcs = append(funcs, dialect.Function{ + fn := dialect.Function{ Name: o.row.Name, Kind: o.kind, Args: o.sig.args(o.row.NArg), Returns: o.sig.Returns, Nullable: isNullable, - }) + } + funcs = append(funcs, fn) + // A function that returns an integer for an integer and a real for + // a real is written once more per spelling of each, the way + // PostgreSQL's catalog has a sum per numeric type. The overload over + // any comes first: the analysis core picks the overload whose + // parameter is the argument's type, but the legacy compiler takes + // the first of the right arity, and keeps what it had. + if o.sig.Numeric && len(fn.Args) > 0 { + for _, typ := range []string{"integer", "real"} { + for _, spelling := range g.numeric[typ] { + typed := fn + typed.Args = append([]dialect.Arg{{Type: spelling}}, fn.Args[1:]...) + typed.Returns = typ + funcs = append(funcs, typed) + } + } + } } return funcs, nil } @@ -355,7 +385,20 @@ func Generate(ctx context.Context, dir string) (dialect.Files, error) { if err != nil { return nil, err } - g := &generator{ctx: ctx, src: src, reported: map[string]bool{}} + g := &generator{ctx: ctx, src: src, reported: map[string]bool{}, numeric: map[string][]string{}} + dialectDir, err := dialect.Dir(Engine) + if err != nil { + return nil, err + } + types, err := dialect.ReadTypes(dialectDir) + if err != nil { + return nil, err + } + for _, t := range types { + if t.Name == "integer" || t.Name == "real" { + g.numeric[t.Name] = append([]string{t.Name}, t.Aliases...) + } + } all := builds() base, err := readShell(ctx, dir, all[0]) if err != nil { @@ -386,7 +429,7 @@ func Generate(ctx context.Context, dir string) (dialect.Files, error) { } } var stale []string - for _, list := range []map[string]bool{omitted, nullable} { + for _, list := range []map[string]bool{omitted, nullable, realResults} { for name := range list { if !g.reported[name] { stale = append(stale, name) diff --git a/internal/goldeneye/sqlite/sqlite_test.go b/internal/goldeneye/sqlite/sqlite_test.go index 028f015fbf..8327090d7e 100644 --- a/internal/goldeneye/sqlite/sqlite_test.go +++ b/internal/goldeneye/sqlite/sqlite_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" ) // TestDialect verifies the committed SQLite dialect against what the pinned @@ -35,3 +36,31 @@ func TestDialect(t *testing.T) { t.Errorf("%s does not match what %s reports:\n%s", dir, version, report) } } + +// TestAnalyzeCases verifies every SQLite analyze case under +// internal/endtoend/testdata against what SQLite reports. It skips unless +// the shells are installed. +func TestAnalyzeCases(t *testing.T) { + dir, err := Locate() + if err != nil { + t.Skip(err) + } + cases, err := endtoend.Cases(Engine) + if err != nil { + t.Fatal(err) + } + if len(cases) == 0 { + t.Fatal("no sqlite analyze cases found") + } + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + diff, err := Check(context.Background(), dir, c) + if err != nil { + t.Fatal(err) + } + if diff != "" { + t.Errorf("%s does not match what SQLite reports (-committed +sqlite):\n%s", c.Output, diff) + } + }) + } +}