diff --git a/README.md b/README.md index 9b7db35..9be6450 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ This is a monorepo containing multiple standalone projects. Each project lives i code-samples/ ├── typesense-angular-search-bar/ # Angular + Typesense search implementation ├── typesense-astro-search/ # Astro + Typesense search implementation +├── typesense-deno-hono-full-text-search/ # Deno (Hono) + Typesense backend implementation ├── typesense-django-full-text-search/ # Python (Django) + Typesense backend implementation ├── typesense-gin-full-text-search/ # Go (Gin) + Typesense backend implementation ├── typesense-kotlin/ # Kotlin (Android) + Typesense search implementation @@ -35,6 +36,7 @@ code-samples/ | ---------------------------------------------------------------------------- | ------------- | --------------------------------------------------------------- | | [typesense-angular-search-bar](./typesense-angular-search-bar) | Angular | A modern search bar with instant search capabilities | | [typesense-astro-search](./typesense-astro-search) | Astro | A modern search bar with instant search capabilities | +| [typesense-deno-hono-full-text-search](./typesense-deno-hono-full-text-search) | Deno (Hono) | Backend API with full-text search using Typesense | | [typesense-django-full-text-search](./typesense-django-full-text-search) | Python (Django) | Backend API with full-text search using Typesense | | [typesense-gin-full-text-search](./typesense-gin-full-text-search) | Go (Gin) | Backend API with full-text search using Typesense | | [typesense-kotlin](./typesense-kotlin) | Kotlin (Android) | A native Android search bar with instant search capabilities | diff --git a/typesense-deno-hono-full-text-search/.env.example b/typesense-deno-hono-full-text-search/.env.example new file mode 100644 index 0000000..f188551 --- /dev/null +++ b/typesense-deno-hono-full-text-search/.env.example @@ -0,0 +1,15 @@ +PORT=3000 + +# PostgreSQL +DATABASE_URL=postgres://:@:/ + +# Typesense +TYPESENSE_HOST=localhost +TYPESENSE_PORT=8108 +TYPESENSE_PROTOCOL=http +TYPESENSE_API_KEY=xyz +TYPESENSE_COLLECTION=books + +# Sync tuning +SYNC_INTERVAL_SECONDS=60 +SYNC_BATCH_SIZE=1000 diff --git a/typesense-deno-hono-full-text-search/.gitignore b/typesense-deno-hono-full-text-search/.gitignore new file mode 100644 index 0000000..42c072b --- /dev/null +++ b/typesense-deno-hono-full-text-search/.gitignore @@ -0,0 +1,4 @@ +.env +deno.lock +node_modules/ +typesense-data/ diff --git a/typesense-deno-hono-full-text-search/README.md b/typesense-deno-hono-full-text-search/README.md new file mode 100644 index 0000000..0284c68 --- /dev/null +++ b/typesense-deno-hono-full-text-search/README.md @@ -0,0 +1,57 @@ +# Typesense Deno + Hono Full-Text Search App + +A production-ready RESTful search API built with Deno, Hono, PostgreSQL, and Typesense. + +This application maintains PostgreSQL as the primary source of truth while keeping Typesense synchronously and asynchronously updated to handle fast, typo-tolerant full-text searches. + +## Features +- **Deno Native**: No bundler and no build step. Dependencies are declared in `deno.json`. +- **Plain SQL**: Queries are hand-written with [postgres.js](https://github.com/porsager/postgres). No ORM and no migration tool. +- **Batched Incremental Sync**: Handles millions of rows without memory bloat using keyset pagination. +- **Soft Delete Support**: Properly handles `deleted_at` fields and purges ghosts from Typesense. +- **Background Worker**: Keeps the database and Typesense index synchronized automatically. + +## Prerequisites +- Deno v2+ +- Docker + +## Setup & Running + +1. **Start Typesense and PostgreSQL:** +```bash +docker run -d -p 8108:8108 \ + -v "$(pwd)"/typesense-data:/data \ + typesense/typesense:30.2 \ + --data-dir /data \ + --api-key=xyz \ + --enable-cors + +docker run -d \ + --name local_postgres \ + -e POSTGRES_USER=admin \ + -e POSTGRES_PASSWORD=admin123 \ + -e POSTGRES_DB=typesense_books \ + -p 5432:5432 \ + postgres:16 +``` + +If PostgreSQL is already running locally on port `5432`, map the container to a free port instead (for example `-p 5433:5432`) and update `DATABASE_URL` to match. + +2. **Set up environment variables:** +Copy the template and fill in your PostgreSQL and Typesense values. +```bash +cp .env.example .env +``` + +3. **Create the database schema:** +Apply `db/schema.sql`, which creates the `books` table, its indexes, the `updated_at` trigger, and a few seed books. +```bash +deno task db:migrate +``` + +4. **Start the application:** +```bash +deno task dev +``` + +The app will connect to PostgreSQL, initialize the Typesense collection, perform a startup sync (if needed), start the background sync worker, and bind to `http://localhost:3000`. diff --git a/typesense-deno-hono-full-text-search/db/schema.sql b/typesense-deno-hono-full-text-search/db/schema.sql new file mode 100644 index 0000000..87c70e4 --- /dev/null +++ b/typesense-deno-hono-full-text-search/db/schema.sql @@ -0,0 +1,40 @@ +CREATE TABLE IF NOT EXISTS books ( + id SERIAL PRIMARY KEY, + title VARCHAR(255) NOT NULL, + authors TEXT[] NOT NULL DEFAULT '{}', + publication_year INTEGER, + average_rating NUMERIC(3, 2), + image_url VARCHAR(512), + ratings_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + deleted_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS books_updated_at_idx ON books (updated_at); +CREATE INDEX IF NOT EXISTS books_deleted_at_idx ON books (deleted_at); +CREATE INDEX IF NOT EXISTS books_active_id_idx ON books (id) WHERE deleted_at IS NULL; + +CREATE OR REPLACE FUNCTION books_touch_updated_at() RETURNS trigger AS $$ +BEGIN + NEW.updated_at = now(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS books_touch_updated_at ON books; +CREATE TRIGGER books_touch_updated_at + BEFORE UPDATE ON books + FOR EACH ROW + EXECUTE FUNCTION books_touch_updated_at(); + +INSERT INTO books (title, authors, publication_year, average_rating, image_url, ratings_count) +SELECT * FROM ( + VALUES + ('Harry Potter and the Philosopher''s Stone', ARRAY['J.K. Rowling'], 1997, 4.47, 'https://covers.openlibrary.org/b/id/10521270-L.jpg', 9278000), + ('The Hobbit', ARRAY['J.R.R. Tolkien'], 1937, 4.28, 'https://covers.openlibrary.org/b/id/6979861-L.jpg', 3400000), + ('Dune', ARRAY['Frank Herbert'], 1965, 4.25, 'https://covers.openlibrary.org/b/id/11481354-L.jpg', 1100000), + ('Good Omens', ARRAY['Terry Pratchett', 'Neil Gaiman'], 1990, 4.26, 'https://covers.openlibrary.org/b/id/8231990-L.jpg', 600000), + ('The Left Hand of Darkness', ARRAY['Ursula K. Le Guin'], 1969, 4.07, 'https://covers.openlibrary.org/b/id/8231856-L.jpg', 180000) +) AS seed (title, authors, publication_year, average_rating, image_url, ratings_count) +WHERE NOT EXISTS (SELECT 1 FROM books); diff --git a/typesense-deno-hono-full-text-search/deno.json b/typesense-deno-hono-full-text-search/deno.json new file mode 100644 index 0000000..f200471 --- /dev/null +++ b/typesense-deno-hono-full-text-search/deno.json @@ -0,0 +1,28 @@ +{ + "name": "typesense-deno-hono-full-text-search", + "version": "1.0.0", + "exports": "./src/server.ts", + "tasks": { + "dev": "deno run --allow-net --allow-env --allow-read --allow-sys --env-file=.env --watch src/server.ts", + "start": "deno run --allow-net --allow-env --allow-read --allow-sys --env-file=.env src/server.ts", + "db:migrate": "deno run --allow-net --allow-env --allow-read --allow-sys --env-file=.env src/migrate.ts", + "check": "deno check src/server.ts src/migrate.ts" + }, + "imports": { + "hono": "jsr:@hono/hono@^4.13.4", + "hono/cors": "jsr:@hono/hono@^4.13.4/cors", + "postgres": "npm:postgres@^3.4.9", + "typesense": "npm:typesense@^3.0.6" + }, + "compilerOptions": { + "strict": true + }, + "fmt": { + "include": [ + "src/" + ], + "lineWidth": 100, + "singleQuote": true + }, + "nodeModulesDir": "auto" +} diff --git a/typesense-deno-hono-full-text-search/src/config/database.ts b/typesense-deno-hono-full-text-search/src/config/database.ts new file mode 100644 index 0000000..3164dc8 --- /dev/null +++ b/typesense-deno-hono-full-text-search/src/config/database.ts @@ -0,0 +1,7 @@ +import postgres from 'postgres'; +import { env } from './env.ts'; + +export const sql = postgres(env.DATABASE_URL, { + max: 10, + onnotice: (notice) => console.log(`postgres notice: ${notice.message}`), +}); diff --git a/typesense-deno-hono-full-text-search/src/config/env.ts b/typesense-deno-hono-full-text-search/src/config/env.ts new file mode 100644 index 0000000..3117d91 --- /dev/null +++ b/typesense-deno-hono-full-text-search/src/config/env.ts @@ -0,0 +1,34 @@ +const REQUIRED = [ + 'DATABASE_URL', + 'TYPESENSE_HOST', + 'TYPESENSE_PORT', + 'TYPESENSE_PROTOCOL', + 'TYPESENSE_API_KEY', + 'TYPESENSE_COLLECTION', +] as const; + +for (const key of REQUIRED) { + if (!Deno.env.get(key)) { + throw new Error(`Missing required environment variable: ${key}`); + } +} + +const number = (key: string, fallback: number): number => { + const raw = Deno.env.get(key); + if (!raw) return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed)) throw new Error(`Environment variable ${key} must be a number`); + return parsed; +}; + +export const env = { + PORT: number('PORT', 3000), + DATABASE_URL: Deno.env.get('DATABASE_URL')!, + TYPESENSE_HOST: Deno.env.get('TYPESENSE_HOST')!, + TYPESENSE_PORT: number('TYPESENSE_PORT', 8108), + TYPESENSE_PROTOCOL: Deno.env.get('TYPESENSE_PROTOCOL')!, + TYPESENSE_API_KEY: Deno.env.get('TYPESENSE_API_KEY')!, + TYPESENSE_COLLECTION: Deno.env.get('TYPESENSE_COLLECTION')!, + SYNC_INTERVAL_SECONDS: number('SYNC_INTERVAL_SECONDS', 60), + SYNC_BATCH_SIZE: number('SYNC_BATCH_SIZE', 1000), +}; diff --git a/typesense-deno-hono-full-text-search/src/db/books.ts b/typesense-deno-hono-full-text-search/src/db/books.ts new file mode 100644 index 0000000..1f247b2 --- /dev/null +++ b/typesense-deno-hono-full-text-search/src/db/books.ts @@ -0,0 +1,121 @@ +import { sql } from '../config/database.ts'; + +export interface Book { + id: number; + title: string; + authors: string[]; + publication_year: number | null; + average_rating: string | null; + image_url: string | null; + ratings_count: number; + created_at: Date; + updated_at: Date; + deleted_at: Date | null; +} + +export interface BookInput { + title?: string; + authors?: string[]; + publication_year?: number | null; + average_rating?: number | null; + image_url?: string | null; + ratings_count?: number; +} + +export async function dbNow(): Promise { + const [row] = await sql<{ now: Date }[]>`SELECT now() AS now`; + return row.now; +} + +export async function countActiveBooks(): Promise { + const [row] = await sql<{ total: string }[]>` + SELECT count(*)::text AS total FROM books WHERE deleted_at IS NULL`; + return Number(row.total); +} + +export async function listActiveBooks(limit: number, offset: number): Promise { + return await sql` + SELECT * FROM books + WHERE deleted_at IS NULL + ORDER BY id + LIMIT ${limit} OFFSET ${offset}`; +} + +export async function findActiveBookById(id: number): Promise { + const [row] = await sql` + SELECT * FROM books WHERE id = ${id} AND deleted_at IS NULL`; + return row; +} + +export async function insertBook(input: BookInput): Promise { + const [row] = await sql` + INSERT INTO books (title, authors, publication_year, average_rating, image_url, ratings_count) + VALUES ( + ${input.title ?? ''}, + ${input.authors ?? []}, + ${input.publication_year ?? null}, + ${input.average_rating ?? null}, + ${input.image_url ?? null}, + ${input.ratings_count ?? 0} + ) + RETURNING *`; + return row; +} + +export async function updateBook(id: number, input: BookInput): Promise { + const patch: Record = {}; + if (input.title !== undefined) patch.title = input.title; + if (input.authors !== undefined) patch.authors = input.authors; + if (input.publication_year !== undefined) patch.publication_year = input.publication_year; + if (input.average_rating !== undefined) patch.average_rating = input.average_rating; + if (input.image_url !== undefined) patch.image_url = input.image_url; + if (input.ratings_count !== undefined) patch.ratings_count = input.ratings_count; + + if (Object.keys(patch).length === 0) return await findActiveBookById(id); + + const [row] = await sql` + UPDATE books SET ${sql(patch)} + WHERE id = ${id} AND deleted_at IS NULL + RETURNING *`; + return row; +} + +export async function softDeleteBook(id: number): Promise { + const [row] = await sql` + UPDATE books SET deleted_at = now() + WHERE id = ${id} AND deleted_at IS NULL + RETURNING *`; + return row; +} + +export async function fetchActiveBooksAfterId(lastId: number, limit: number): Promise { + return await sql` + SELECT * FROM books + WHERE id > ${lastId} AND deleted_at IS NULL + ORDER BY id + LIMIT ${limit}`; +} + +export async function fetchBooksUpdatedSince( + since: Date, + lastId: number, + limit: number, +): Promise { + return await sql` + SELECT * FROM books + WHERE updated_at > ${since} AND deleted_at IS NULL AND id > ${lastId} + ORDER BY id + LIMIT ${limit}`; +} + +export async function fetchBooksDeletedSince( + since: Date, + lastId: number, + limit: number, +): Promise<{ id: number }[]> { + return await sql<{ id: number }[]>` + SELECT id FROM books + WHERE deleted_at > ${since} AND id > ${lastId} + ORDER BY id + LIMIT ${limit}`; +} diff --git a/typesense-deno-hono-full-text-search/src/migrate.ts b/typesense-deno-hono-full-text-search/src/migrate.ts new file mode 100644 index 0000000..dc6679f --- /dev/null +++ b/typesense-deno-hono-full-text-search/src/migrate.ts @@ -0,0 +1,16 @@ +import { sql } from './config/database.ts'; + +const schemaPath = new URL('../db/schema.sql', import.meta.url); +const ddl = await Deno.readTextFile(schemaPath); + +console.log('Applying db/schema.sql...'); + +try { + await sql.unsafe(ddl).simple(); + console.log('Schema applied successfully.'); +} catch (error) { + console.error('Migration failed:', error); + Deno.exit(1); +} finally { + await sql.end(); +} diff --git a/typesense-deno-hono-full-text-search/src/routes/books.ts b/typesense-deno-hono-full-text-search/src/routes/books.ts new file mode 100644 index 0000000..3474d93 --- /dev/null +++ b/typesense-deno-hono-full-text-search/src/routes/books.ts @@ -0,0 +1,105 @@ +import { Hono } from 'hono'; +import { + type BookInput, + countActiveBooks, + findActiveBookById, + insertBook, + listActiveBooks, + softDeleteBook, + updateBook, +} from '../db/books.ts'; +import { deleteBookDocument, upsertBookDocument } from '../search/sync.ts'; + +const router = new Hono(); + +const BOOK_FIELDS = [ + 'title', + 'authors', + 'publication_year', + 'average_rating', + 'image_url', + 'ratings_count', +] as const; + +function pickBookInput(body: Record): BookInput { + const input: Record = {}; + for (const field of BOOK_FIELDS) { + if (body[field] !== undefined) input[field] = body[field]; + } + return input as BookInput; +} + +router.get('/', async (c) => { + const page = parseInt(c.req.query('page') ?? '1', 10); + const limit = parseInt(c.req.query('limit') ?? '10', 10); + const offset = (page - 1) * limit; + + try { + const [total, data] = await Promise.all([ + countActiveBooks(), + listActiveBooks(limit, offset), + ]); + + return c.json({ total, page, limit, data }); + } catch (error) { + console.error(error); + return c.json({ error: 'Failed to fetch books' }, 500); + } +}); + +router.get('/:id', async (c) => { + try { + const book = await findActiveBookById(Number(c.req.param('id'))); + if (!book) return c.json({ error: 'Book not found' }, 404); + return c.json(book); + } catch (_error) { + return c.json({ error: 'Failed to fetch book' }, 500); + } +}); + +router.post('/', async (c) => { + try { + const book = await insertBook(pickBookInput(await c.req.json())); + await upsertBookDocument(book).catch((error) => + console.error(`Failed to sync book ${book.id} to Typesense:`, error) + ); + + return c.json(book, 201); + } catch (error) { + return c.json({ error: (error as Error).message }, 400); + } +}); + +router.put('/:id', async (c) => { + try { + const book = await updateBook(Number(c.req.param('id')), pickBookInput(await c.req.json())); + if (!book) return c.json({ error: 'Book not found' }, 404); + + await upsertBookDocument(book).catch((error) => + console.error(`Failed to sync book ${book.id} to Typesense:`, error) + ); + + return c.json(book); + } catch (error) { + return c.json({ error: (error as Error).message }, 400); + } +}); + +router.delete('/:id', async (c) => { + const id = Number(c.req.param('id')); + + try { + const book = await softDeleteBook(id); + if (!book) return c.json({ error: 'Book not found' }, 404); + + await deleteBookDocument(id).catch((error) => + console.error(`Failed to delete book ${id} from Typesense:`, error) + ); + + return c.body(null, 204); + } catch (error) { + return c.json({ error: (error as Error).message }, 500); + } +}); + +export default router; diff --git a/typesense-deno-hono-full-text-search/src/routes/search.ts b/typesense-deno-hono-full-text-search/src/routes/search.ts new file mode 100644 index 0000000..de5bc18 --- /dev/null +++ b/typesense-deno-hono-full-text-search/src/routes/search.ts @@ -0,0 +1,51 @@ +import { Hono } from 'hono'; +import { typesenseClient } from '../search/client.ts'; +import { BOOKS_COLLECTION_NAME } from '../search/collections.ts'; +import { getLastSyncTime, runFullSync } from '../search/sync.ts'; +import { isWorkerRunning } from '../search/worker.ts'; + +const router = new Hono(); + +router.get('/search', async (c) => { + const query = c.req.query('q') ?? ''; + + try { + const searchResults = await typesenseClient + .collections(BOOKS_COLLECTION_NAME) + .documents() + .search({ q: query, query_by: 'title,authors' }); + + return c.json({ + query, + found: searchResults.found, + results: searchResults.hits, + facet_counts: searchResults.facet_counts ?? [], + }); + } catch (error) { + console.error('Search failed:', error); + return c.json({ error: 'Failed to fetch books' }, 500); + } +}); + +router.post('/sync', async (c) => { + try { + const result = await runFullSync(); + if (result.failed) return c.json({ error: 'Failed to sync books' }, 500); + + return c.json({ + message: 'Sync completed', + syncedAt: getLastSyncTime().toISOString(), + }); + } catch (error) { + console.error('Manual sync failed:', error); + return c.json({ error: 'Failed to sync books' }, 500); + } +}); + +router.get('/sync/status', (c) => + c.json({ + lastSyncTime: getLastSyncTime().toISOString(), + syncWorkerRunning: isWorkerRunning(), + })); + +export default router; diff --git a/typesense-deno-hono-full-text-search/src/search/client.ts b/typesense-deno-hono-full-text-search/src/search/client.ts new file mode 100644 index 0000000..a449482 --- /dev/null +++ b/typesense-deno-hono-full-text-search/src/search/client.ts @@ -0,0 +1,14 @@ +import { Client } from 'typesense'; +import { env } from '../config/env.ts'; + +export const typesenseClient = new Client({ + nodes: [{ + host: env.TYPESENSE_HOST, + port: env.TYPESENSE_PORT, + protocol: env.TYPESENSE_PROTOCOL, + }], + apiKey: env.TYPESENSE_API_KEY, + connectionTimeoutSeconds: 5, + retryIntervalSeconds: 1, + numRetries: 3, +}); diff --git a/typesense-deno-hono-full-text-search/src/search/collections.ts b/typesense-deno-hono-full-text-search/src/search/collections.ts new file mode 100644 index 0000000..4858922 --- /dev/null +++ b/typesense-deno-hono-full-text-search/src/search/collections.ts @@ -0,0 +1,35 @@ +import type { CollectionCreateSchema } from 'typesense/lib/Typesense/Collections'; +import { typesenseClient } from './client.ts'; +import { env } from '../config/env.ts'; + +export const BOOKS_COLLECTION_NAME = env.TYPESENSE_COLLECTION; + +const booksSchema: CollectionCreateSchema = { + name: BOOKS_COLLECTION_NAME, + fields: [ + { name: 'title', type: 'string', facet: false }, + { name: 'authors', type: 'string[]', facet: true }, + { name: 'publication_year', type: 'int32', facet: true }, + { name: 'average_rating', type: 'float', facet: true }, + { name: 'image_url', type: 'string', facet: false, index: false, optional: true }, + { name: 'ratings_count', type: 'int32', facet: true }, + ], + default_sorting_field: 'ratings_count', +}; + +export async function initializeTypesense(): Promise { + try { + await typesenseClient.collections(BOOKS_COLLECTION_NAME).retrieve(); + console.log(`Collection '${BOOKS_COLLECTION_NAME}' already exists.`); + } catch (error) { + if ((error as { httpStatus?: number }).httpStatus !== 404) throw error; + console.log(`Collection '${BOOKS_COLLECTION_NAME}' not found. Creating...`); + await typesenseClient.collections().create(booksSchema); + console.log(`Collection '${BOOKS_COLLECTION_NAME}' created.`); + } +} + +export async function getCollectionDocumentCount(): Promise { + const collection = await typesenseClient.collections(BOOKS_COLLECTION_NAME).retrieve(); + return collection.num_documents ?? 0; +} diff --git a/typesense-deno-hono-full-text-search/src/search/sync.ts b/typesense-deno-hono-full-text-search/src/search/sync.ts new file mode 100644 index 0000000..a3a8ff4 --- /dev/null +++ b/typesense-deno-hono-full-text-search/src/search/sync.ts @@ -0,0 +1,211 @@ +import { env } from '../config/env.ts'; +import { + type Book, + dbNow, + fetchActiveBooksAfterId, + fetchBooksDeletedSince, + fetchBooksUpdatedSince, +} from '../db/books.ts'; +import { typesenseClient } from './client.ts'; +import { BOOKS_COLLECTION_NAME, getCollectionDocumentCount } from './collections.ts'; + +export interface BookDocument { + id: string; + title: string; + authors: string[]; + publication_year: number; + average_rating: number; + image_url: string; + ratings_count: number; +} + +export interface SyncResult { + startedAt: Date; + upserted: number; + deleted: number; + failed: boolean; +} + +const BATCH_SIZE = env.SYNC_BATCH_SIZE; +const EPOCH = new Date(0); + +let lastSyncTime: Date = EPOCH; + +export function getLastSyncTime(): Date { + return lastSyncTime; +} + +export function mapBookToDocument(book: Book): BookDocument { + return { + id: String(book.id), + title: book.title, + authors: book.authors ?? [], + publication_year: book.publication_year ?? 0, + average_rating: book.average_rating === null ? 0 : Number(book.average_rating), + image_url: book.image_url ?? '', + ratings_count: book.ratings_count ?? 0, + }; +} + +export async function upsertBookDocument(book: Book): Promise { + await typesenseClient + .collections(BOOKS_COLLECTION_NAME) + .documents() + .upsert(mapBookToDocument(book)); +} + +export async function deleteBookDocument(id: number): Promise { + try { + await typesenseClient.collections(BOOKS_COLLECTION_NAME).documents(String(id)).delete(); + } catch (error) { + if ((error as { httpStatus?: number }).httpStatus === 404) return; + throw error; + } +} + +async function importBatch(documents: BookDocument[]): Promise { + const results = await typesenseClient + .collections(BOOKS_COLLECTION_NAME) + .documents() + .import(documents, { action: 'upsert' }); + + const failures = (Array.isArray(results) ? results : []).filter((result) => !result.success); + if (failures.length > 0) { + for (const failure of failures.slice(0, 5)) { + console.error(`Import rejected a document: ${failure.error}`); + } + throw new Error(`${failures.length} of ${documents.length} documents failed to import`); + } + return documents.length; +} + +export async function runFullSync(): Promise { + const startedAt = await dbNow(); + console.log(`Full sync started, stamped ${startedAt.toISOString()}`); + + let lastId = 0; + let upserted = 0; + let failed = false; + + while (true) { + let batch: Book[]; + try { + batch = await fetchActiveBooksAfterId(lastId, BATCH_SIZE); + } catch (error) { + console.error('Full sync: database read failed:', error); + failed = true; + break; + } + + if (batch.length === 0) break; + lastId = batch[batch.length - 1].id; + + try { + upserted += await importBatch(batch.map(mapBookToDocument)); + console.log(`Full sync: ${upserted} books indexed so far.`); + } catch (error) { + console.error('Full sync: Typesense import failed:', error); + failed = true; + break; + } + } + + if (failed) { + console.warn( + `Full sync incomplete after ${upserted} books; last sync time stays at ${lastSyncTime.toISOString()}`, + ); + } else { + lastSyncTime = startedAt; + console.log(`Full sync completed: ${upserted} books indexed.`); + } + + return { startedAt, upserted, deleted: 0, failed }; +} + +export async function runIncrementalSync(): Promise { + const startedAt = await dbNow(); + const since = lastSyncTime; + console.log(`Incremental sync started for changes after ${since.toISOString()}`); + + let upserted = 0; + let deleted = 0; + let failed = false; + + let lastUpsertId = 0; + while (!failed) { + let batch: Book[]; + try { + batch = await fetchBooksUpdatedSince(since, lastUpsertId, BATCH_SIZE); + } catch (error) { + console.error('Incremental sync: database read failed during upsert phase:', error); + failed = true; + break; + } + + if (batch.length === 0) break; + lastUpsertId = batch[batch.length - 1].id; + + try { + upserted += await importBatch(batch.map(mapBookToDocument)); + } catch (error) { + console.error('Incremental sync: Typesense import failed:', error); + failed = true; + break; + } + } + + let lastDeleteId = 0; + while (!failed) { + let batch: { id: number }[]; + try { + batch = await fetchBooksDeletedSince(since, lastDeleteId, BATCH_SIZE); + } catch (error) { + console.error('Incremental sync: database read failed during delete phase:', error); + failed = true; + break; + } + + if (batch.length === 0) break; + lastDeleteId = batch[batch.length - 1].id; + + for (const row of batch) { + try { + await deleteBookDocument(row.id); + deleted++; + } catch (error) { + console.error(`Incremental sync: failed to delete document ${row.id}:`, error); + failed = true; + break; + } + } + } + + if (failed) { + console.warn( + `Incremental sync incomplete; last sync time stays at ${lastSyncTime.toISOString()} so the next run retries it`, + ); + } else { + lastSyncTime = startedAt; + if (upserted || deleted) { + console.log(`Incremental sync completed: ${upserted} upserted, ${deleted} deleted.`); + } else { + console.log('Incremental sync completed: no changes.'); + } + } + + return { startedAt, upserted, deleted, failed }; +} + +export async function determineAndRunStartupSync(): Promise { + const documentCount = await getCollectionDocumentCount(); + + if (documentCount === 0) { + console.log('Typesense collection is empty — running a full sync.'); + await runFullSync(); + return; + } + + console.log(`Typesense collection holds ${documentCount} documents — catching up from epoch.`); + lastSyncTime = EPOCH; + await runIncrementalSync(); +} diff --git a/typesense-deno-hono-full-text-search/src/search/worker.ts b/typesense-deno-hono-full-text-search/src/search/worker.ts new file mode 100644 index 0000000..4bed525 --- /dev/null +++ b/typesense-deno-hono-full-text-search/src/search/worker.ts @@ -0,0 +1,41 @@ +import { env } from '../config/env.ts'; +import { runIncrementalSync } from './sync.ts'; + +let timer: ReturnType | undefined; +let workerRunning = false; +let syncInProgress = false; + +export function isWorkerRunning(): boolean { + return workerRunning; +} + +export function startBackgroundSyncWorker(): void { + if (workerRunning) return; + + const intervalMs = env.SYNC_INTERVAL_SECONDS * 1000; + console.log(`Starting background sync worker (every ${env.SYNC_INTERVAL_SECONDS}s)...`); + + timer = setInterval(async () => { + if (syncInProgress) { + console.log('Sync already in progress — skipping this tick.'); + return; + } + + syncInProgress = true; + try { + await runIncrementalSync(); + } catch (error) { + console.error('Background sync threw:', error); + } finally { + syncInProgress = false; + } + }, intervalMs); + + workerRunning = true; +} + +export function stopBackgroundSyncWorker(): void { + if (timer !== undefined) clearInterval(timer); + timer = undefined; + workerRunning = false; +} diff --git a/typesense-deno-hono-full-text-search/src/server.ts b/typesense-deno-hono-full-text-search/src/server.ts new file mode 100644 index 0000000..4287f05 --- /dev/null +++ b/typesense-deno-hono-full-text-search/src/server.ts @@ -0,0 +1,29 @@ +import { Hono } from 'hono'; +import { cors } from 'hono/cors'; +import { env } from './config/env.ts'; +import { initializeTypesense } from './search/collections.ts'; +import { determineAndRunStartupSync } from './search/sync.ts'; +import { startBackgroundSyncWorker } from './search/worker.ts'; +import booksRouter from './routes/books.ts'; +import searchRouter from './routes/search.ts'; + +const app = new Hono(); + +app.use('*', cors()); +app.route('/books', booksRouter); +app.route('/', searchRouter); + +console.log('Initializing Typesense collection...'); +await initializeTypesense(); + +console.log('Running startup sync...'); +try { + await determineAndRunStartupSync(); +} catch (error) { + console.error('Startup sync failed, continuing anyway:', error); +} + +startBackgroundSyncWorker(); + +Deno.serve({ port: env.PORT }, app.fetch); +console.log(`Server is running on http://localhost:${env.PORT}`);