From 160fbce1f02f3d3f5b6d3f44f849d461836bd5c2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 25 Aug 2026 18:29:17 +0000 Subject: [PATCH] core: model the hidden columns of sqlite fts5 tables An fts5 virtual table offers columns beyond the ones it declares: one named after the table itself, matched against the whole row and passed to the auxiliary functions, rank, the current match's score, and rowid. The analysis core knew none of them, so queries from #1797 like SELECT rowid, name FROM recipes_fts WHERE recipes_fts MATCH ? failed with 'unknown column'. Give sql_attribute a hidden flag and ast.ColumnDef an IsHidden marker. The sqlite converter emits the three fts5 hidden columns; the core analyzer resolves them by name while keeping them out of star expansions, models and implicit INSERT targets. The legacy catalog drops hidden columns, so the legacy path is unchanged. The bm25 query in the virtual_table case now spells out its columns: the core path does not yet expand stars in the query text, and that gap is not this change's to close. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NrYAwWTgJMeea6n6wZ95oy --- internal/core/analyzer/dml.go | 8 +- internal/core/analyzer/projection.go | 3 + internal/core/attribute.go | 7 ++ internal/core/catalogdb/db.go | 6 +- internal/core/catalogdb/models.go | 1 + internal/core/catalogdb/query.sql.go | 18 +++-- internal/core/catalogdef/query.sql | 8 +- internal/core/catalogdef/schema.sql | 4 + internal/core/schema/schema.go | 1 + .../testdata/virtual_table/sqlite/query.sql | 2 +- .../sqlite/exec.json | 3 + .../virtual_table_fts5_hidden/sqlite/go/db.go | 31 ++++++++ .../sqlite/go/models.go | 14 ++++ .../sqlite/go/query.sql.go | 78 +++++++++++++++++++ .../sqlite/query.sql | 7 ++ .../sqlite/schema.sql | 10 +++ .../sqlite/sqlc.yaml | 9 +++ internal/engine/sqlite/convert.go | 25 ++++++ internal/sql/ast/column_def.go | 5 ++ internal/sql/catalog/table.go | 5 ++ 20 files changed, 229 insertions(+), 16 deletions(-) create mode 100644 internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/exec.json create mode 100644 internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/db.go create mode 100644 internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/models.go create mode 100644 internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/query.sql.go create mode 100644 internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/query.sql create mode 100644 internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/schema.sql create mode 100644 internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/sqlc.yaml diff --git a/internal/core/analyzer/dml.go b/internal/core/analyzer/dml.go index f1cb397c1c..28cbc5193c 100644 --- a/internal/core/analyzer/dml.go +++ b/internal/core/analyzer/dml.go @@ -108,7 +108,13 @@ func (a *analyzer) relationScope(relations, extra *ast.List, from ast.Node) (*sc func insertTargets(rel scopeRel, cols *ast.List) ([]core.ClassColumn, error) { items := listItems(cols) if len(items) == 0 { - return rel.cols, nil + out := make([]core.ClassColumn, 0, len(rel.cols)) + for _, col := range rel.cols { + if !col.Hidden { + out = append(out, col) + } + } + return out, nil } out := make([]core.ClassColumn, 0, len(items)) for _, item := range items { diff --git a/internal/core/analyzer/projection.go b/internal/core/analyzer/projection.go index 9547eed833..6ccbd65c5d 100644 --- a/internal/core/analyzer/projection.go +++ b/internal/core/analyzer/projection.go @@ -90,6 +90,9 @@ func (a *analyzer) emitStar(fields []string) { } a.columns = slices.Grow(a.columns, len(rel.cols)) for _, c := range rel.cols { + if c.Hidden { + continue + } col := core.Column{ Name: c.Name, TypeOID: c.TypeOID, diff --git a/internal/core/attribute.go b/internal/core/attribute.go index 1138b195dc..7a0a58a4ac 100644 --- a/internal/core/attribute.go +++ b/internal/core/attribute.go @@ -20,6 +20,10 @@ type AttributeSpec struct { AutoIncrement bool IsPrimaryKey bool IsUnique bool + // Hidden columns resolve by name but stay out of a star expansion and + // the relation's model, like the column an sqlite fts5 table names + // after itself. + Hidden bool } func (c *Catalog) CreateAttributeSpec(s AttributeSpec) error { @@ -36,6 +40,7 @@ func (c *Catalog) CreateAttributeSpec(s AttributeSpec) error { AutoIncrement: boolToInt64(s.AutoIncrement), IsPrimaryKey: boolToInt64(s.IsPrimaryKey), IsUnique: boolToInt64(s.IsUnique), + Hidden: boolToInt64(s.Hidden), }) if err != nil { return fmt.Errorf("create attribute %q on class %d: %w", s.Name, s.ClassOID, err) @@ -213,6 +218,7 @@ type ClassColumn struct { Name string TypeOID int64 NotNull bool + Hidden bool } // ClassColumns returns a relation's columns in ordinal order. @@ -228,6 +234,7 @@ func (c *Catalog) ClassColumns(classOID int64) ([]ClassColumn, error) { Name: r.Name, TypeOID: r.TypeOid, NotNull: r.NotNull != 0, + Hidden: r.Hidden != 0, }) } return out, nil diff --git a/internal/core/catalogdb/db.go b/internal/core/catalogdb/db.go index d16ad694e9..8ad3680481 100644 --- a/internal/core/catalogdb/db.go +++ b/internal/core/catalogdb/db.go @@ -10,10 +10,10 @@ import ( ) type DBTX interface { - ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + ExecContext(context.Context, string, ...any) (sql.Result, error) PrepareContext(context.Context, string) (*sql.Stmt, error) - QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) - QueryRowContext(context.Context, string, ...interface{}) *sql.Row + QueryContext(context.Context, string, ...any) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...any) *sql.Row } func New(db DBTX) *Queries { diff --git a/internal/core/catalogdb/models.go b/internal/core/catalogdb/models.go index 374a2e70ee..4c79ec8c1b 100644 --- a/internal/core/catalogdb/models.go +++ b/internal/core/catalogdb/models.go @@ -22,6 +22,7 @@ type SqlAttribute struct { AutoIncrement int64 IsPrimaryKey int64 IsUnique int64 + Hidden int64 } type SqlCast struct { diff --git a/internal/core/catalogdb/query.sql.go b/internal/core/catalogdb/query.sql.go index abf96cfd13..b14851dce4 100644 --- a/internal/core/catalogdb/query.sql.go +++ b/internal/core/catalogdb/query.sql.go @@ -12,7 +12,7 @@ import ( ) const classAttributes = `-- name: ClassAttributes :many -SELECT oid, name, type_oid, not_null +SELECT oid, name, type_oid, not_null, hidden FROM sql_attribute WHERE class_oid = ? ORDER BY num @@ -23,6 +23,7 @@ type ClassAttributesRow struct { Name string TypeOid int64 NotNull int64 + Hidden int64 } func (q *Queries) ClassAttributes(ctx context.Context, classOid int64) ([]ClassAttributesRow, error) { @@ -39,6 +40,7 @@ func (q *Queries) ClassAttributes(ctx context.Context, classOid int64) ([]ClassA &i.Name, &i.TypeOid, &i.NotNull, + &i.Hidden, ); err != nil { return nil, err } @@ -85,8 +87,8 @@ const createAttribute = `-- name: CreateAttribute :exec INSERT INTO sql_attribute ( class_oid, name, type_oid, not_null, has_default, num, decl_type, type_length, type_scale, - auto_increment, is_primary_key, is_unique -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + auto_increment, is_primary_key, is_unique, hidden +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` type CreateAttributeParams struct { @@ -102,6 +104,7 @@ type CreateAttributeParams struct { AutoIncrement int64 IsPrimaryKey int64 IsUnique int64 + Hidden int64 } // ============================= sql_attribute =========================== @@ -119,6 +122,7 @@ func (q *Queries) CreateAttribute(ctx context.Context, arg CreateAttributeParams arg.AutoIncrement, arg.IsPrimaryKey, arg.IsUnique, + arg.Hidden, ) return err } @@ -452,8 +456,8 @@ WHERE name = ?1 type FindOperatorsParams struct { Name string - LeftTypeOid interface{} - RightTypeOid interface{} + LeftTypeOid any + RightTypeOid any } type FindOperatorsRow struct { @@ -560,7 +564,7 @@ type FindProcsInNamespacesRow struct { func (q *Queries) FindProcsInNamespaces(ctx context.Context, arg FindProcsInNamespacesParams) ([]FindProcsInNamespacesRow, error) { query := findProcsInNamespaces - var queryParams []interface{} + var queryParams []any queryParams = append(queryParams, arg.Name) if len(arg.NamespaceOids) > 0 { for _, v := range arg.NamespaceOids { @@ -602,7 +606,7 @@ const listClassColumns = `-- name: ListClassColumns :many SELECT a.name AS column_name, t.name AS type_name, a.not_null FROM sql_attribute a JOIN sql_type t ON t.oid = a.type_oid -WHERE a.class_oid = ? +WHERE a.class_oid = ? AND a.hidden = 0 ORDER BY a.num ` diff --git a/internal/core/catalogdef/query.sql b/internal/core/catalogdef/query.sql index 358aba6139..eb9863470c 100644 --- a/internal/core/catalogdef/query.sql +++ b/internal/core/catalogdef/query.sql @@ -95,8 +95,8 @@ UPDATE sql_class SET name = sqlc.arg(new_name) WHERE oid = sqlc.arg(oid); INSERT INTO sql_attribute ( class_oid, name, type_oid, not_null, has_default, num, decl_type, type_length, type_scale, - auto_increment, is_primary_key, is_unique -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + auto_increment, is_primary_key, is_unique, hidden +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); -- name: SetAttributePrimaryKey :exec UPDATE sql_attribute SET is_primary_key = 1, not_null = 1 @@ -147,7 +147,7 @@ WHERE c.name = ? ORDER BY a.num; -- name: ClassAttributes :many -SELECT oid, name, type_oid, not_null +SELECT oid, name, type_oid, not_null, hidden FROM sql_attribute WHERE class_oid = ? ORDER BY num; @@ -156,7 +156,7 @@ ORDER BY num; SELECT a.name AS column_name, t.name AS type_name, a.not_null FROM sql_attribute a JOIN sql_type t ON t.oid = a.type_oid -WHERE a.class_oid = ? +WHERE a.class_oid = ? AND a.hidden = 0 ORDER BY a.num; -- name: LookupAttribute :one diff --git a/internal/core/catalogdef/schema.sql b/internal/core/catalogdef/schema.sql index e9d9741d4b..1e48501d58 100644 --- a/internal/core/catalogdef/schema.sql +++ b/internal/core/catalogdef/schema.sql @@ -65,6 +65,9 @@ CREATE TABLE sql_class ( -- Set both for inline-column PK and for table-level PK. -- is_unique: column has a UNIQUE constraint or a single-column UNIQUE -- table constraint. +-- hidden: resolvable by name but absent from a star expansion and +-- from the relation's model, like the column an sqlite +-- fts5 table names after itself. CREATE TABLE sql_attribute ( oid INTEGER PRIMARY KEY AUTOINCREMENT, class_oid INTEGER NOT NULL REFERENCES sql_class(oid), @@ -79,6 +82,7 @@ CREATE TABLE sql_attribute ( auto_increment INTEGER NOT NULL DEFAULT 0, is_primary_key INTEGER NOT NULL DEFAULT 0, is_unique INTEGER NOT NULL DEFAULT 0, + hidden INTEGER NOT NULL DEFAULT 0, UNIQUE(class_oid, name), UNIQUE(class_oid, num) ); diff --git a/internal/core/schema/schema.go b/internal/core/schema/schema.go index 0d1d1a9a4f..19dc3cc827 100644 --- a/internal/core/schema/schema.go +++ b/internal/core/schema/schema.go @@ -163,6 +163,7 @@ func applyCreateTable(cat *core.Catalog, stmt *ast.CreateTableStmt) error { NotNull: col.IsNotNull || col.PrimaryKey, IsPrimaryKey: col.PrimaryKey, DeclType: col.TypeName.Name, + Hidden: col.IsHidden, }); err != nil { return fmt.Errorf("attr %s.%s: %w", stmt.Name.Name, col.Colname, err) } diff --git a/internal/endtoend/testdata/virtual_table/sqlite/query.sql b/internal/endtoend/testdata/virtual_table/sqlite/query.sql index ad8eeeae40..5646d5034e 100644 --- a/internal/endtoend/testdata/virtual_table/sqlite/query.sql +++ b/internal/endtoend/testdata/virtual_table/sqlite/query.sql @@ -22,7 +22,7 @@ WHERE b MATCH ?; SELECT snippet(tbl_ft, 0, '', '', 'aa', ?) FROM tbl_ft; -- name: SelectBm25Func :many -SELECT *, bm25(tbl_ft, 2.0) FROM tbl_ft +SELECT b, c, bm25(tbl_ft, 2.0) FROM tbl_ft WHERE b MATCH ? ORDER BY bm25(tbl_ft); -- name: UpdateTblFt :exec diff --git a/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/exec.json b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/exec.json new file mode 100644 index 0000000000..8a7ecd291e --- /dev/null +++ b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/exec.json @@ -0,0 +1,3 @@ +{ + "contexts": ["core"] +} diff --git a/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/db.go b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/db.go new file mode 100644 index 0000000000..32faa9f2b8 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...any) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...any) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/models.go b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/models.go new file mode 100644 index 0000000000..98b7ec9dd9 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/models.go @@ -0,0 +1,14 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package querytest + +type Recipe struct { + ID int64 + Name string +} + +type RecipesFt struct { + Name string +} diff --git a/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/query.sql.go b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/query.sql.go new file mode 100644 index 0000000000..d815d9999b --- /dev/null +++ b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/query.sql.go @@ -0,0 +1,78 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: query.sql + +package querytest + +import ( + "context" + "database/sql" +) + +const searchRecipes = `-- name: SearchRecipes :many +SELECT rowid, name FROM recipes_fts +WHERE recipes_fts MATCH ? +` + +type SearchRecipesRow struct { + Rowid int64 + Name string +} + +func (q *Queries) SearchRecipes(ctx context.Context, recipesFts string) ([]SearchRecipesRow, error) { + rows, err := q.db.QueryContext(ctx, searchRecipes, recipesFts) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SearchRecipesRow + for rows.Next() { + var i SearchRecipesRow + if err := rows.Scan(&i.Rowid, &i.Name); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const searchRecipesRanked = `-- name: SearchRecipesRanked :many +SELECT rowid, name, rank FROM recipes_fts +WHERE recipes_fts MATCH ? ORDER BY rank +` + +type SearchRecipesRankedRow struct { + Rowid int64 + Name string + Rank sql.NullFloat64 +} + +func (q *Queries) SearchRecipesRanked(ctx context.Context, recipesFts string) ([]SearchRecipesRankedRow, error) { + rows, err := q.db.QueryContext(ctx, searchRecipesRanked, recipesFts) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SearchRecipesRankedRow + for rows.Next() { + var i SearchRecipesRankedRow + if err := rows.Scan(&i.Rowid, &i.Name, &i.Rank); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/query.sql b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/query.sql new file mode 100644 index 0000000000..a539d26108 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/query.sql @@ -0,0 +1,7 @@ +-- name: SearchRecipes :many +SELECT rowid, name FROM recipes_fts +WHERE recipes_fts MATCH ?; + +-- name: SearchRecipesRanked :many +SELECT rowid, name, rank FROM recipes_fts +WHERE recipes_fts MATCH ? ORDER BY rank; diff --git a/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/schema.sql b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/schema.sql new file mode 100644 index 0000000000..498824eba6 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/schema.sql @@ -0,0 +1,10 @@ +CREATE TABLE recipes ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL +); + +CREATE VIRTUAL TABLE recipes_fts USING FTS5 ( + name, + content='recipes', + content_rowid='id' +); diff --git a/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/sqlc.yaml b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/sqlc.yaml new file mode 100644 index 0000000000..77e7a3daa2 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/sqlc.yaml @@ -0,0 +1,9 @@ +version: '2' +sql: +- schema: schema.sql + queries: query.sql + engine: sqlite + gen: + go: + package: querytest + out: go diff --git a/internal/engine/sqlite/convert.go b/internal/engine/sqlite/convert.go index 7916125e0e..88bc279819 100644 --- a/internal/engine/sqlite/convert.go +++ b/internal/engine/sqlite/convert.go @@ -325,6 +325,31 @@ func (c *cc) convertCreateVirtualTableFTS5(n *meyer.CreateVirtualTableStmt) ast. }) } + // An fts5 table also offers hidden columns: one named after the table, + // matched against the whole row (`tbl MATCH ?`) and passed to the + // auxiliary functions (`bm25(tbl)`), `rank`, the current match's score, + // and `rowid`, which an external-content table maps to its content + // table. Hidden keeps them out of star expansions and the table's model. + stmt.Cols = append(stmt.Cols, + &ast.ColumnDef{ + Colname: stmt.Name.Name, + IsNotNull: true, + IsHidden: true, + TypeName: &ast.TypeName{Name: "text"}, + }, + &ast.ColumnDef{ + Colname: "rank", + IsHidden: true, + TypeName: &ast.TypeName{Name: "real"}, + }, + &ast.ColumnDef{ + Colname: "rowid", + IsNotNull: true, + IsHidden: true, + TypeName: &ast.TypeName{Name: "integer"}, + }, + ) + return stmt } diff --git a/internal/sql/ast/column_def.go b/internal/sql/ast/column_def.go index 225cdd4779..2a1c015752 100644 --- a/internal/sql/ast/column_def.go +++ b/internal/sql/ast/column_def.go @@ -12,6 +12,11 @@ type ColumnDef struct { Vals *List Length *int PrimaryKey bool + // IsHidden marks a column a relation offers by name without listing it, + // like the column an sqlite fts5 table names after itself. The legacy + // catalog drops hidden columns; the core catalog keeps them out of star + // expansions and models. + IsHidden bool // From pg.ColumnDef Inhcount int diff --git a/internal/sql/catalog/table.go b/internal/sql/catalog/table.go index dc30acfa1e..ec2a122a96 100644 --- a/internal/sql/catalog/table.go +++ b/internal/sql/catalog/table.go @@ -301,6 +301,11 @@ func (c *Catalog) createTable(stmt *ast.CreateTableStmt) error { } for _, col := range stmt.Cols { + // The legacy catalog has no notion of a hidden column, so it drops + // them rather than list them on the table. + if col.IsHidden { + continue + } if notNull, ok := seen[col.Colname]; ok { seen[col.Colname] = notNull || col.IsNotNull if a, ok := coltype[col.Colname]; ok {