diff --git a/internal/core/analyzer/dml.go b/internal/core/analyzer/dml.go index f1cb397c1c..28cbc5193c 100644 --- a/internal/core/analyzer/dml.go +++ b/internal/core/analyzer/dml.go @@ -108,7 +108,13 @@ func (a *analyzer) relationScope(relations, extra *ast.List, from ast.Node) (*sc func insertTargets(rel scopeRel, cols *ast.List) ([]core.ClassColumn, error) { items := listItems(cols) if len(items) == 0 { - return rel.cols, nil + out := make([]core.ClassColumn, 0, len(rel.cols)) + for _, col := range rel.cols { + if !col.Hidden { + out = append(out, col) + } + } + return out, nil } out := make([]core.ClassColumn, 0, len(items)) for _, item := range items { diff --git a/internal/core/analyzer/projection.go b/internal/core/analyzer/projection.go index 9547eed833..6ccbd65c5d 100644 --- a/internal/core/analyzer/projection.go +++ b/internal/core/analyzer/projection.go @@ -90,6 +90,9 @@ func (a *analyzer) emitStar(fields []string) { } a.columns = slices.Grow(a.columns, len(rel.cols)) for _, c := range rel.cols { + if c.Hidden { + continue + } col := core.Column{ Name: c.Name, TypeOID: c.TypeOID, diff --git a/internal/core/attribute.go b/internal/core/attribute.go index 1138b195dc..7a0a58a4ac 100644 --- a/internal/core/attribute.go +++ b/internal/core/attribute.go @@ -20,6 +20,10 @@ type AttributeSpec struct { AutoIncrement bool IsPrimaryKey bool IsUnique bool + // Hidden columns resolve by name but stay out of a star expansion and + // the relation's model, like the column an sqlite fts5 table names + // after itself. + Hidden bool } func (c *Catalog) CreateAttributeSpec(s AttributeSpec) error { @@ -36,6 +40,7 @@ func (c *Catalog) CreateAttributeSpec(s AttributeSpec) error { AutoIncrement: boolToInt64(s.AutoIncrement), IsPrimaryKey: boolToInt64(s.IsPrimaryKey), IsUnique: boolToInt64(s.IsUnique), + Hidden: boolToInt64(s.Hidden), }) if err != nil { return fmt.Errorf("create attribute %q on class %d: %w", s.Name, s.ClassOID, err) @@ -213,6 +218,7 @@ type ClassColumn struct { Name string TypeOID int64 NotNull bool + Hidden bool } // ClassColumns returns a relation's columns in ordinal order. @@ -228,6 +234,7 @@ func (c *Catalog) ClassColumns(classOID int64) ([]ClassColumn, error) { Name: r.Name, TypeOID: r.TypeOid, NotNull: r.NotNull != 0, + Hidden: r.Hidden != 0, }) } return out, nil diff --git a/internal/core/catalogdb/db.go b/internal/core/catalogdb/db.go index d16ad694e9..8ad3680481 100644 --- a/internal/core/catalogdb/db.go +++ b/internal/core/catalogdb/db.go @@ -10,10 +10,10 @@ import ( ) type DBTX interface { - ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + ExecContext(context.Context, string, ...any) (sql.Result, error) PrepareContext(context.Context, string) (*sql.Stmt, error) - QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) - QueryRowContext(context.Context, string, ...interface{}) *sql.Row + QueryContext(context.Context, string, ...any) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...any) *sql.Row } func New(db DBTX) *Queries { diff --git a/internal/core/catalogdb/models.go b/internal/core/catalogdb/models.go index 374a2e70ee..4c79ec8c1b 100644 --- a/internal/core/catalogdb/models.go +++ b/internal/core/catalogdb/models.go @@ -22,6 +22,7 @@ type SqlAttribute struct { AutoIncrement int64 IsPrimaryKey int64 IsUnique int64 + Hidden int64 } type SqlCast struct { diff --git a/internal/core/catalogdb/query.sql.go b/internal/core/catalogdb/query.sql.go index abf96cfd13..b14851dce4 100644 --- a/internal/core/catalogdb/query.sql.go +++ b/internal/core/catalogdb/query.sql.go @@ -12,7 +12,7 @@ import ( ) const classAttributes = `-- name: ClassAttributes :many -SELECT oid, name, type_oid, not_null +SELECT oid, name, type_oid, not_null, hidden FROM sql_attribute WHERE class_oid = ? ORDER BY num @@ -23,6 +23,7 @@ type ClassAttributesRow struct { Name string TypeOid int64 NotNull int64 + Hidden int64 } func (q *Queries) ClassAttributes(ctx context.Context, classOid int64) ([]ClassAttributesRow, error) { @@ -39,6 +40,7 @@ func (q *Queries) ClassAttributes(ctx context.Context, classOid int64) ([]ClassA &i.Name, &i.TypeOid, &i.NotNull, + &i.Hidden, ); err != nil { return nil, err } @@ -85,8 +87,8 @@ const createAttribute = `-- name: CreateAttribute :exec INSERT INTO sql_attribute ( class_oid, name, type_oid, not_null, has_default, num, decl_type, type_length, type_scale, - auto_increment, is_primary_key, is_unique -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + auto_increment, is_primary_key, is_unique, hidden +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ` type CreateAttributeParams struct { @@ -102,6 +104,7 @@ type CreateAttributeParams struct { AutoIncrement int64 IsPrimaryKey int64 IsUnique int64 + Hidden int64 } // ============================= sql_attribute =========================== @@ -119,6 +122,7 @@ func (q *Queries) CreateAttribute(ctx context.Context, arg CreateAttributeParams arg.AutoIncrement, arg.IsPrimaryKey, arg.IsUnique, + arg.Hidden, ) return err } @@ -452,8 +456,8 @@ WHERE name = ?1 type FindOperatorsParams struct { Name string - LeftTypeOid interface{} - RightTypeOid interface{} + LeftTypeOid any + RightTypeOid any } type FindOperatorsRow struct { @@ -560,7 +564,7 @@ type FindProcsInNamespacesRow struct { func (q *Queries) FindProcsInNamespaces(ctx context.Context, arg FindProcsInNamespacesParams) ([]FindProcsInNamespacesRow, error) { query := findProcsInNamespaces - var queryParams []interface{} + var queryParams []any queryParams = append(queryParams, arg.Name) if len(arg.NamespaceOids) > 0 { for _, v := range arg.NamespaceOids { @@ -602,7 +606,7 @@ const listClassColumns = `-- name: ListClassColumns :many SELECT a.name AS column_name, t.name AS type_name, a.not_null FROM sql_attribute a JOIN sql_type t ON t.oid = a.type_oid -WHERE a.class_oid = ? +WHERE a.class_oid = ? AND a.hidden = 0 ORDER BY a.num ` diff --git a/internal/core/catalogdef/query.sql b/internal/core/catalogdef/query.sql index 358aba6139..eb9863470c 100644 --- a/internal/core/catalogdef/query.sql +++ b/internal/core/catalogdef/query.sql @@ -95,8 +95,8 @@ UPDATE sql_class SET name = sqlc.arg(new_name) WHERE oid = sqlc.arg(oid); INSERT INTO sql_attribute ( class_oid, name, type_oid, not_null, has_default, num, decl_type, type_length, type_scale, - auto_increment, is_primary_key, is_unique -) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + auto_increment, is_primary_key, is_unique, hidden +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); -- name: SetAttributePrimaryKey :exec UPDATE sql_attribute SET is_primary_key = 1, not_null = 1 @@ -147,7 +147,7 @@ WHERE c.name = ? ORDER BY a.num; -- name: ClassAttributes :many -SELECT oid, name, type_oid, not_null +SELECT oid, name, type_oid, not_null, hidden FROM sql_attribute WHERE class_oid = ? ORDER BY num; @@ -156,7 +156,7 @@ ORDER BY num; SELECT a.name AS column_name, t.name AS type_name, a.not_null FROM sql_attribute a JOIN sql_type t ON t.oid = a.type_oid -WHERE a.class_oid = ? +WHERE a.class_oid = ? AND a.hidden = 0 ORDER BY a.num; -- name: LookupAttribute :one diff --git a/internal/core/catalogdef/schema.sql b/internal/core/catalogdef/schema.sql index e9d9741d4b..1e48501d58 100644 --- a/internal/core/catalogdef/schema.sql +++ b/internal/core/catalogdef/schema.sql @@ -65,6 +65,9 @@ CREATE TABLE sql_class ( -- Set both for inline-column PK and for table-level PK. -- is_unique: column has a UNIQUE constraint or a single-column UNIQUE -- table constraint. +-- hidden: resolvable by name but absent from a star expansion and +-- from the relation's model, like the column an sqlite +-- fts5 table names after itself. CREATE TABLE sql_attribute ( oid INTEGER PRIMARY KEY AUTOINCREMENT, class_oid INTEGER NOT NULL REFERENCES sql_class(oid), @@ -79,6 +82,7 @@ CREATE TABLE sql_attribute ( auto_increment INTEGER NOT NULL DEFAULT 0, is_primary_key INTEGER NOT NULL DEFAULT 0, is_unique INTEGER NOT NULL DEFAULT 0, + hidden INTEGER NOT NULL DEFAULT 0, UNIQUE(class_oid, name), UNIQUE(class_oid, num) ); diff --git a/internal/core/schema/schema.go b/internal/core/schema/schema.go index 0d1d1a9a4f..19dc3cc827 100644 --- a/internal/core/schema/schema.go +++ b/internal/core/schema/schema.go @@ -163,6 +163,7 @@ func applyCreateTable(cat *core.Catalog, stmt *ast.CreateTableStmt) error { NotNull: col.IsNotNull || col.PrimaryKey, IsPrimaryKey: col.PrimaryKey, DeclType: col.TypeName.Name, + Hidden: col.IsHidden, }); err != nil { return fmt.Errorf("attr %s.%s: %w", stmt.Name.Name, col.Colname, err) } diff --git a/internal/endtoend/testdata/virtual_table/sqlite/query.sql b/internal/endtoend/testdata/virtual_table/sqlite/query.sql index ad8eeeae40..5646d5034e 100644 --- a/internal/endtoend/testdata/virtual_table/sqlite/query.sql +++ b/internal/endtoend/testdata/virtual_table/sqlite/query.sql @@ -22,7 +22,7 @@ WHERE b MATCH ?; SELECT snippet(tbl_ft, 0, '', '', 'aa', ?) FROM tbl_ft; -- name: SelectBm25Func :many -SELECT *, bm25(tbl_ft, 2.0) FROM tbl_ft +SELECT b, c, bm25(tbl_ft, 2.0) FROM tbl_ft WHERE b MATCH ? ORDER BY bm25(tbl_ft); -- name: UpdateTblFt :exec diff --git a/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/exec.json b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/exec.json new file mode 100644 index 0000000000..8a7ecd291e --- /dev/null +++ b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/exec.json @@ -0,0 +1,3 @@ +{ + "contexts": ["core"] +} diff --git a/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/db.go b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/db.go new file mode 100644 index 0000000000..32faa9f2b8 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...any) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...any) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/models.go b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/models.go new file mode 100644 index 0000000000..98b7ec9dd9 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/models.go @@ -0,0 +1,14 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package querytest + +type Recipe struct { + ID int64 + Name string +} + +type RecipesFt struct { + Name string +} diff --git a/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/query.sql.go b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/query.sql.go new file mode 100644 index 0000000000..d815d9999b --- /dev/null +++ b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/go/query.sql.go @@ -0,0 +1,78 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: query.sql + +package querytest + +import ( + "context" + "database/sql" +) + +const searchRecipes = `-- name: SearchRecipes :many +SELECT rowid, name FROM recipes_fts +WHERE recipes_fts MATCH ? +` + +type SearchRecipesRow struct { + Rowid int64 + Name string +} + +func (q *Queries) SearchRecipes(ctx context.Context, recipesFts string) ([]SearchRecipesRow, error) { + rows, err := q.db.QueryContext(ctx, searchRecipes, recipesFts) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SearchRecipesRow + for rows.Next() { + var i SearchRecipesRow + if err := rows.Scan(&i.Rowid, &i.Name); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const searchRecipesRanked = `-- name: SearchRecipesRanked :many +SELECT rowid, name, rank FROM recipes_fts +WHERE recipes_fts MATCH ? ORDER BY rank +` + +type SearchRecipesRankedRow struct { + Rowid int64 + Name string + Rank sql.NullFloat64 +} + +func (q *Queries) SearchRecipesRanked(ctx context.Context, recipesFts string) ([]SearchRecipesRankedRow, error) { + rows, err := q.db.QueryContext(ctx, searchRecipesRanked, recipesFts) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SearchRecipesRankedRow + for rows.Next() { + var i SearchRecipesRankedRow + if err := rows.Scan(&i.Rowid, &i.Name, &i.Rank); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/query.sql b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/query.sql new file mode 100644 index 0000000000..a539d26108 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/query.sql @@ -0,0 +1,7 @@ +-- name: SearchRecipes :many +SELECT rowid, name FROM recipes_fts +WHERE recipes_fts MATCH ?; + +-- name: SearchRecipesRanked :many +SELECT rowid, name, rank FROM recipes_fts +WHERE recipes_fts MATCH ? ORDER BY rank; diff --git a/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/schema.sql b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/schema.sql new file mode 100644 index 0000000000..498824eba6 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/schema.sql @@ -0,0 +1,10 @@ +CREATE TABLE recipes ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL +); + +CREATE VIRTUAL TABLE recipes_fts USING FTS5 ( + name, + content='recipes', + content_rowid='id' +); diff --git a/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/sqlc.yaml b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/sqlc.yaml new file mode 100644 index 0000000000..77e7a3daa2 --- /dev/null +++ b/internal/endtoend/testdata/virtual_table_fts5_hidden/sqlite/sqlc.yaml @@ -0,0 +1,9 @@ +version: '2' +sql: +- schema: schema.sql + queries: query.sql + engine: sqlite + gen: + go: + package: querytest + out: go diff --git a/internal/engine/sqlite/convert.go b/internal/engine/sqlite/convert.go index 7916125e0e..88bc279819 100644 --- a/internal/engine/sqlite/convert.go +++ b/internal/engine/sqlite/convert.go @@ -325,6 +325,31 @@ func (c *cc) convertCreateVirtualTableFTS5(n *meyer.CreateVirtualTableStmt) ast. }) } + // An fts5 table also offers hidden columns: one named after the table, + // matched against the whole row (`tbl MATCH ?`) and passed to the + // auxiliary functions (`bm25(tbl)`), `rank`, the current match's score, + // and `rowid`, which an external-content table maps to its content + // table. Hidden keeps them out of star expansions and the table's model. + stmt.Cols = append(stmt.Cols, + &ast.ColumnDef{ + Colname: stmt.Name.Name, + IsNotNull: true, + IsHidden: true, + TypeName: &ast.TypeName{Name: "text"}, + }, + &ast.ColumnDef{ + Colname: "rank", + IsHidden: true, + TypeName: &ast.TypeName{Name: "real"}, + }, + &ast.ColumnDef{ + Colname: "rowid", + IsNotNull: true, + IsHidden: true, + TypeName: &ast.TypeName{Name: "integer"}, + }, + ) + return stmt } diff --git a/internal/sql/ast/column_def.go b/internal/sql/ast/column_def.go index 225cdd4779..2a1c015752 100644 --- a/internal/sql/ast/column_def.go +++ b/internal/sql/ast/column_def.go @@ -12,6 +12,11 @@ type ColumnDef struct { Vals *List Length *int PrimaryKey bool + // IsHidden marks a column a relation offers by name without listing it, + // like the column an sqlite fts5 table names after itself. The legacy + // catalog drops hidden columns; the core catalog keeps them out of star + // expansions and models. + IsHidden bool // From pg.ColumnDef Inhcount int diff --git a/internal/sql/catalog/table.go b/internal/sql/catalog/table.go index dc30acfa1e..ec2a122a96 100644 --- a/internal/sql/catalog/table.go +++ b/internal/sql/catalog/table.go @@ -301,6 +301,11 @@ func (c *Catalog) createTable(stmt *ast.CreateTableStmt) error { } for _, col := range stmt.Cols { + // The legacy catalog has no notion of a hidden column, so it drops + // them rather than list them on the table. + if col.IsHidden { + continue + } if notNull, ok := seen[col.Colname]; ok { seen[col.Colname] = notNull || col.IsNotNull if a, ok := coltype[col.Colname]; ok {