From 1fbecf3d81e5131c7a8397e5d24c02d408798b31 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 2 Sep 2026 12:44:09 -0400 Subject: [PATCH 01/18] fix(search): match dataset keyword against dbinfo when subject filters are set --- backend/src/controllers/couchdb.controller.js | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index ab12d0f..264b2fa 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -259,12 +259,34 @@ const searchAllDatabases = async (req, res) => { // "ABIDE - CMU_a" matches stored names regardless of separator style. // The whole group is parenthesised so it ANDs cleanly with other filters. if (isFilter(f.keyword)) { - where.push(`( - search_vector @@ plainto_tsquery('english', :keyword) - OR dbname ILIKE :keywordLike - OR dsname ILIKE :keywordLike - OR (json->>'name') ILIKE :keywordLike - )`); + // The keyword matches dataset-level text (name / README / AI summary), + // which lives only in dbinfo rows. On a subjects search the current row + // has no such text, so match the keyword against the dataset's dbinfo + // row via EXISTS — mirrors the modalities cross-view pattern above. + // Without this, any dataset-level keyword wrongly drops all subjects (it + // only appeared to work when the keyword happened to be a subject-level + // token such as a task name, e.g. "memory"). + if (isSubjectSearch) { + where.push(`EXISTS ( + SELECT 1 FROM ioviews dsi + WHERE dsi.dbname = ioviews.dbname + AND dsi.dsname = ioviews.dsname + AND dsi.view = 'dbinfo' + AND ( + dsi.search_vector @@ plainto_tsquery('english', :keyword) + OR dsi.dbname ILIKE :keywordLike + OR dsi.dsname ILIKE :keywordLike + OR (dsi.json->>'name') ILIKE :keywordLike + ) + )`); + } else { + where.push(`( + search_vector @@ plainto_tsquery('english', :keyword) + OR dbname ILIKE :keywordLike + OR dsname ILIKE :keywordLike + OR (json->>'name') ILIKE :keywordLike + )`); + } repl.keyword = String(f.keyword); repl.keywordLike = `%${String(f.keyword).replace(/[\s-]+/g, "%")}%`; } From cebafee973013a1fe1180b57d5e368a0eeba04c1 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Wed, 2 Sep 2026 14:29:30 -0400 Subject: [PATCH 02/18] fix(search): stop "no matching subjects" warning flashing on slow searches --- src/pages/SearchPage.tsx | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index df49761..afafa7e 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -648,17 +648,27 @@ const SearchPage: React.FC = () => { // check if has database/dataset matches // const hasDbMatches = !!keywordInput && registryMatches.length > 0; const hasDbMatches = registryMatches.length > 0; + // Display/pagination use the local `results` state (kept — also gives TS + // array-narrowing inside {hasDatasetMatches && ...} render blocks). const hasDatasetMatches = Array.isArray(results) && results.length > 0; - // when backend find nothing - const backendEmpty = - !Array.isArray(results) && (results as any)?.msg === "empty output"; + + // The empty-state WARNING is based on the Redux `searchResults` instead, + // which updates in the SAME render as `loading` (both set in the fulfilled + // reducer). The local `results` is set a render later (in the dispatch + // .then()), so using it for the warning made it FLASH on slow searches: + // loading turns false while `results` is still stale/empty. + const storeHasMatches = + Array.isArray(searchResults) && searchResults.length > 0; + const storeEmpty = + !Array.isArray(searchResults) && + (searchResults as any)?.msg === "empty output"; // show red message only if nothing matched at all const showNoResults = hasSearched && !loading && // !hasDbMatches && - (!hasDatasetMatches || backendEmpty); + (!storeHasMatches || storeEmpty); // Tailored empty-state message: when the user combined a file_type filter // with any subject-level filter and got nothing back, it's almost certainly From c4e2b9cb57af185731a75b238ea4f720da59e95a Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 3 Sep 2026 10:56:10 -0400 Subject: [PATCH 03/18] perf(search): index iolinks(dbname, dsname, view) for file-type filter --- ...21-add-iolinks-dbname-dsname-view-index.js | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 backend/migrations/20260903145021-add-iolinks-dbname-dsname-view-index.js diff --git a/backend/migrations/20260903145021-add-iolinks-dbname-dsname-view-index.js b/backend/migrations/20260903145021-add-iolinks-dbname-dsname-view-index.js new file mode 100644 index 0000000..a985659 --- /dev/null +++ b/backend/migrations/20260903145021-add-iolinks-dbname-dsname-view-index.js @@ -0,0 +1,26 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // Composite index for the file-type filter's EXISTS subquery in + // searchAllDatabases: + // EXISTS (SELECT 1 FROM iolinks l + // WHERE l.dbname = ioviews.dbname AND l.dsname = ioviews.dsname + // AND l.view IN (:fileTypes)) + // Without it the planner uses idx_iolinks_dbname alone and scans every row + // of a database (openneuro ~1.4M), filtering dsname/view in memory — the + // dominant cost of a subject search combined with a file_type filter. This + // turns that into a direct lookup on (dbname, dsname, view). + async up (queryInterface, Sequelize) { + await queryInterface.addIndex("iolinks", ["dbname", "dsname", "view"], { + name: "idx_iolinks_dbname_dsname_view", + }); + }, + + async down (queryInterface, Sequelize) { + await queryInterface.removeIndex( + "iolinks", + "idx_iolinks_dbname_dsname_view" + ); + } +}; From 6dd4ac31d6e3dc170de29680b3db8808db47b89c Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 3 Sep 2026 12:08:25 -0400 Subject: [PATCH 04/18] feat(stats): compute landing-page db stats from Postgres instead of CGI --- backend/src/controllers/couchdb.controller.js | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index 264b2fa..a21b7d8 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -23,13 +23,43 @@ const getDbList = async (req, res) => { } }; -// get db stats +// get db stats — Postgres-backed. Previously proxied to the legacy CGI, kept +// here for reference; that source is decoupled from the synced Postgres data: +// const response = await axios.get( +// "https://neurojson.org/io/search.cgi?dbstats=1" +// ); +// res.status(200).json(response.data); +// +// Returns the same [{view, num, size}] shape the landing-page StatisticsBanner +// expects: +// - one row per file extension: num = file count, size = total bytes +// (iolinks.subj holds the byte size as text) +// - a 'dbinfo' row → num = dataset count +// - a 'subjects' row → num = subject count const getDbStats = async (req, res) => { try { - const response = await axios.get( - "https://neurojson.org/io/search.cgi?dbstats=1" + const rows = await sequelize.query( + `SELECT view, + count(*) AS num, + sum(CASE WHEN subj ~ '^[0-9]+$' THEN subj::bigint ELSE 0 END) AS size + FROM iolinks + GROUP BY view + UNION ALL + SELECT 'dbinfo' AS view, count(*) AS num, 0 AS size + FROM ioviews WHERE view = 'dbinfo' + UNION ALL + SELECT 'subjects' AS view, count(*) AS num, 0 AS size + FROM ioviews WHERE view = 'subjects'`, + { type: sequelize.QueryTypes.SELECT } ); - res.status(200).json(response.data); + // Sequelize returns bigint as a string; the frontend sums `size` and `num` + // numerically, so coerce them to Number. + const stats = rows.map((r) => ({ + view: r.view, + num: Number(r.num), + size: Number(r.size), + })); + res.status(200).json(stats); } catch (error) { console.error("Error fetching db stats:", error.message); res.status(error.response?.status || 500).json({ From 0e42d23d8b97656369ecd1332a80806734e40630 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 3 Sep 2026 14:30:18 -0400 Subject: [PATCH 05/18] feat(stats): add stats_history table for persistent sync stats --- .../20260903180911-create-stats-history.js | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 backend/migrations/20260903180911-create-stats-history.js diff --git a/backend/migrations/20260903180911-create-stats-history.js b/backend/migrations/20260903180911-create-stats-history.js new file mode 100644 index 0000000..93be320 --- /dev/null +++ b/backend/migrations/20260903180911-create-stats-history.js @@ -0,0 +1,61 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // One row per scheduled sync run. Doubles as (1) persistent landing-page + // stats and (2) sync execution history. Lifecycle: inserted 'running' at + // sync start; finalized 'success' (with totals) or 'failed' (with error) + // after the run completes. The latest 'success' row = current NeuroJSON state. + async up (queryInterface, Sequelize) { + await queryInterface.createTable("stats_history", { + id: { + type: Sequelize.INTEGER, + autoIncrement: true, + primaryKey: true, + allowNull: false, + }, + started_at: { + type: Sequelize.DATE, // TIMESTAMPTZ + allowNull: false, + }, + completed_at: { + type: Sequelize.DATE, + allowNull: true, + }, + status: { + type: Sequelize.TEXT, // 'running' | 'success' | 'failed' + allowNull: false, + defaultValue: "running", + }, + total_datasets: { + type: Sequelize.BIGINT, + allowNull: true, + }, + total_subjects: { + type: Sequelize.BIGINT, + allowNull: true, + }, + total_files: { + type: Sequelize.BIGINT, + allowNull: true, + }, + total_size_bytes: { + type: Sequelize.BIGINT, + allowNull: true, + }, + error: { + type: Sequelize.TEXT, // failure reason when status = 'failed' + allowNull: true, + }, + created_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + }, + + async down (queryInterface, Sequelize) { + await queryInterface.dropTable("stats_history"); + } +}; From 0e683264779043167d180e97b959a4e0f30ab356 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 3 Sep 2026 14:42:14 -0400 Subject: [PATCH 06/18] feat(stats): write a stats_history snapshot at the end of each sync --- backend/sync/incrementalSync.js | 67 +++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js index f563905..267207f 100644 --- a/backend/sync/incrementalSync.js +++ b/backend/sync/incrementalSync.js @@ -550,6 +550,46 @@ async function syncDatabase(dbname) { } } +// === stats_history: one row per sync run === + +// Insert a 'running' row at sync start; return its id. The id will also back +// per-entity change logging (db_change_log) in a later phase. +async function createStatsHistory() { + const [rows] = await sequelize.query( + `INSERT INTO stats_history (started_at, status) + VALUES (NOW(), 'running') RETURNING id` + ); + return rows[0].id; +} + +// Finalize the run's row. On success, compute totals from the fully-synced +// tables (done ONCE here, never per landing-page visit); on failure, record +// the error. Totals stay NULL for a failed/partial run. +async function finalizeStatsHistory(id, status, errorMessage) { + if (status !== "success") { + await sequelize.query( + `UPDATE stats_history + SET completed_at = NOW(), status = :status, error = :error + WHERE id = :id`, + { replacements: { id, status, error: errorMessage || null } } + ); + return; + } + await sequelize.query( + `UPDATE stats_history SET + completed_at = NOW(), + status = 'success', + total_datasets = (SELECT count(*) FROM ioviews WHERE view = 'dbinfo'), + total_subjects = (SELECT count(*) FROM ioviews WHERE view = 'subjects'), + total_files = (SELECT count(*) FROM iolinks), + total_size_bytes = (SELECT COALESCE( + sum(CASE WHEN subj ~ '^[0-9]+$' THEN subj::bigint ELSE 0 END), 0) + FROM iolinks) + WHERE id = :id`, + { replacements: { id } } + ); +} + // === Main === async function runSync() { @@ -557,16 +597,27 @@ async function runSync() { console.log(new Date().toISOString()); console.log(`CouchDB: ${COUCHDB_URL}`); - const databases = await getDatabases(); - console.log(`Databases: ${databases.length}`); + // Create the run's history row up front so it exists throughout the sync. + const historyId = await createStatsHistory(); - for (const db of databases) { - await syncDatabase(db); - } + try { + const databases = await getDatabases(); + console.log(`Databases: ${databases.length}`); - await sequelize.close(); - console.log("\n=== Sync complete ==="); - console.log(new Date().toISOString()); + for (const db of databases) { + await syncDatabase(db); + } + + // Compute + publish totals only after the whole run completed. + await finalizeStatsHistory(historyId, "success"); + console.log(`\n=== Sync complete (stats_history #${historyId}) ===`); + console.log(new Date().toISOString()); + } catch (err) { + await finalizeStatsHistory(historyId, "failed", err.message); + throw err; + } finally { + await sequelize.close(); + } } runSync().catch((err) => { From 4a9bb83ab49e1769d13a1860176656e013d4b914 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 3 Sep 2026 14:48:38 -0400 Subject: [PATCH 07/18] perf(stats): serve /dbs/stats from the stats_history snapshot --- backend/src/controllers/couchdb.controller.js | 68 ++++++++++--------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index a21b7d8..8eb65e2 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -23,43 +23,49 @@ const getDbList = async (req, res) => { } }; -// get db stats — Postgres-backed. Previously proxied to the legacy CGI, kept -// here for reference; that source is decoupled from the synced Postgres data: -// const response = await axios.get( -// "https://neurojson.org/io/search.cgi?dbstats=1" -// ); -// res.status(200).json(response.data); +// get db stats — reads the latest finalized snapshot from stats_history +// (written once at the end of each sync), so the landing page never triggers a +// live aggregate over ioviews/iolinks. // -// Returns the same [{view, num, size}] shape the landing-page StatisticsBanner -// expects: -// - one row per file extension: num = file count, size = total bytes -// (iolinks.subj holds the byte size as text) -// - a 'dbinfo' row → num = dataset count -// - a 'subjects' row → num = subject count +// History of this endpoint: +// 1. proxied the legacy CGI: https://neurojson.org/io/search.cgi?dbstats=1 +// (decoupled from the synced Postgres — stale/junk) +// 2. live Postgres aggregate over iolinks/ioviews (correct but scanned ~1.5M +// rows on every landing-page visit) +// 3. this: read the precomputed stats_history snapshot (tiny query) +// +// Returns: +// { datasets, subjects, files, sizeBytes, lastSynced } const getDbStats = async (req, res) => { try { const rows = await sequelize.query( - `SELECT view, - count(*) AS num, - sum(CASE WHEN subj ~ '^[0-9]+$' THEN subj::bigint ELSE 0 END) AS size - FROM iolinks - GROUP BY view - UNION ALL - SELECT 'dbinfo' AS view, count(*) AS num, 0 AS size - FROM ioviews WHERE view = 'dbinfo' - UNION ALL - SELECT 'subjects' AS view, count(*) AS num, 0 AS size - FROM ioviews WHERE view = 'subjects'`, + `SELECT total_datasets, total_subjects, total_files, + total_size_bytes, completed_at + FROM stats_history + WHERE status = 'success' + ORDER BY completed_at DESC + LIMIT 1`, { type: sequelize.QueryTypes.SELECT } ); - // Sequelize returns bigint as a string; the frontend sums `size` and `num` - // numerically, so coerce them to Number. - const stats = rows.map((r) => ({ - view: r.view, - num: Number(r.num), - size: Number(r.size), - })); - res.status(200).json(stats); + const row = rows[0]; + if (!row) { + // No successful sync snapshot yet. + return res.status(200).json({ + datasets: 0, + subjects: 0, + files: 0, + sizeBytes: 0, + lastSynced: null, + }); + } + // Sequelize returns BIGINT as a string; coerce to Number for the frontend. + res.status(200).json({ + datasets: Number(row.total_datasets), + subjects: Number(row.total_subjects), + files: Number(row.total_files), + sizeBytes: Number(row.total_size_bytes), + lastSynced: row.completed_at, + }); } catch (error) { console.error("Error fetching db stats:", error.message); res.status(error.response?.status || 500).json({ From b6df62a2ad6fd3b70caa4aad4166bcfffd1ffa47 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 3 Sep 2026 14:51:39 -0400 Subject: [PATCH 08/18] feat(stats): render StatisticsBanner from the stats_history snapshot --- src/components/StatisticsBanner.tsx | 29 +++++-------------- .../neurojson/types/neurojson.interface.ts | 12 +++++++- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/components/StatisticsBanner.tsx b/src/components/StatisticsBanner.tsx index 99e148a..6d96324 100644 --- a/src/components/StatisticsBanner.tsx +++ b/src/components/StatisticsBanner.tsx @@ -9,7 +9,6 @@ import { useAppDispatch } from "hooks/useAppDispatch"; import { useAppSelector } from "hooks/useAppSelector"; import React, { useEffect } from "react"; import { fetchDbStats } from "redux/neurojson/neurojson.action"; -import { DbStatsItem } from "redux/neurojson/types/neurojson.interface"; import { RootState } from "redux/store"; const iconStyle = { @@ -42,31 +41,19 @@ const labelTextStyle = { }, }; -// function for calculate links and size -const calculateLinksAndSize = (dbStats: DbStatsItem[] | null) => { - if (!dbStats) return { totalLinks: 0, totalSizeTB: "0.00" }; - - const filtered = dbStats.filter( - (item) => item.view !== "dbinfo" && item.view !== "subjects" - ); - - const totalLinks = filtered.reduce((acc, item) => acc + item.num, 0); - const totalSizeBytes = filtered.reduce((acc, item) => acc + item.size, 0); - const totalSizeTB = Math.floor(totalSizeBytes / 1024 ** 4); - return { totalLinks, totalSizeTB }; -}; - const StatisticsBanner: React.FC = () => { const dispatch = useAppDispatch(); + // dbStats is now the flat snapshot { datasets, subjects, files, sizeBytes, + // lastSynced } from the latest successful stats_history row. const dbstats = useAppSelector((state: RootState) => state.neurojson.dbStats); const registry = useAppSelector( (state: RootState) => state.neurojson.registry ); const databaseCount = registry?.length ?? "-"; - const datasetStat = dbstats?.find((item) => item.view === "dbinfo"); - const subjectStat = dbstats?.find((item) => item.view === "subjects"); - const { totalLinks, totalSizeTB } = calculateLinksAndSize(dbstats); + const totalSizeTB = dbstats + ? Math.floor(dbstats.sizeBytes / 1024 ** 4) + : "-"; // format numbers with commas const formatNumber = (num: number | undefined) => @@ -127,19 +114,19 @@ const StatisticsBanner: React.FC = () => { {/* Datasets */} } - number={formatNumber(datasetStat?.num)} + number={formatNumber(dbstats?.datasets)} label="Datasets" /> {/* Subjects */} } - number={formatNumber(subjectStat?.num)} + number={formatNumber(dbstats?.subjects)} label="Subjects" /> {/* Links */} } - number={formatNumber(totalLinks)} + number={formatNumber(dbstats?.files)} label="Links" /> {/* Size */} diff --git a/src/redux/neurojson/types/neurojson.interface.ts b/src/redux/neurojson/types/neurojson.interface.ts index 01c8273..fc3a7a0 100644 --- a/src/redux/neurojson/types/neurojson.interface.ts +++ b/src/redux/neurojson/types/neurojson.interface.ts @@ -10,7 +10,7 @@ export interface INeuroJsonState { limit: number; hasMore: boolean; dbInfo: DBParticulars | null; // add dbInfo type - dbStats: DbStatsItem[] | null; // for dbStats on landing page + dbStats: DbStats | null; // landing-page stats snapshot (from stats_history) searchResults: any[] | { status: string; msg: string } | null; datasetViewInfo: any | null; fileTypes: string[] | null; @@ -93,3 +93,13 @@ export interface DbStatsItem { num: number; size: number; } + +// Landing-page stats snapshot returned by GET /dbs/stats (latest successful +// stats_history row). +export interface DbStats { + datasets: number; + subjects: number; + files: number; + sizeBytes: number; + lastSynced: string | null; +} From edb11419c0d414ba22f9e81ed33ae8c8337585b1 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 10 Sep 2026 12:30:53 -0400 Subject: [PATCH 09/18] feat(stats): add dataset_changes table for dataset-level sync change log; refs #145 --- .../20260910160052-create-dataset-changes.js | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 backend/migrations/20260910160052-create-dataset-changes.js diff --git a/backend/migrations/20260910160052-create-dataset-changes.js b/backend/migrations/20260910160052-create-dataset-changes.js new file mode 100644 index 0000000..0ee46b1 --- /dev/null +++ b/backend/migrations/20260910160052-create-dataset-changes.js @@ -0,0 +1,59 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // One row per dataset-level change detected during an incremental sync. + // Only the incremental path (processDatasetUpdate / delete branch) writes + // here — firstSync never logs, so a full/first sync does not mass-record + // every dataset as "added". Powers the "latest update" board on the landing + // page. history_id → stats_history.id (the sync run that produced the change). + async up (queryInterface, Sequelize) { + await queryInterface.createTable("dataset_changes", { + id: { + type: Sequelize.INTEGER, + autoIncrement: true, + primaryKey: true, + allowNull: false, + }, + history_id: { + type: Sequelize.INTEGER, // FK → stats_history.id (same INTEGER type) + allowNull: false, + references: { model: "stats_history", key: "id" }, + onDelete: "CASCADE", + }, + dbname: { + type: Sequelize.TEXT, + allowNull: false, + }, + dsname: { + type: Sequelize.TEXT, + allowNull: false, + }, + change_type: { + type: Sequelize.TEXT, // 'added' | 'updated' | 'deleted' + allowNull: false, + }, + created_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + + // Dedupe: one logical change per dataset per sync run (a dataset touched + // multiple times in one sync → ON CONFLICT DO NOTHING on this key). + await queryInterface.addIndex( + "dataset_changes", + ["history_id", "dbname", "dsname"], + { name: "uq_dataset_changes_run_dataset", unique: true } + ); + // "latest update" lookup filters/groups by run. + await queryInterface.addIndex("dataset_changes", ["history_id"], { + name: "idx_dataset_changes_history_id", + }); + }, + + async down (queryInterface, Sequelize) { + await queryInterface.dropTable("dataset_changes"); + } +}; From 8f69f2c38fd0a881d476dc597e7b6273566e846f Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 10 Sep 2026 12:35:57 -0400 Subject: [PATCH 10/18] feat(stats): log dataset-level changes during incremental sync; refs #145 --- backend/sync/incrementalSync.js | 64 ++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js index 267207f..d3599d2 100644 --- a/backend/sync/incrementalSync.js +++ b/backend/sync/incrementalSync.js @@ -317,6 +317,29 @@ async function insertIolink(dbname, dsname, subj, view, json, transaction) { ); } +// Record one dataset-level change for a sync run. Deduped by the unique +// (history_id, dbname, dsname) index, so a dataset touched twice in one run +// keeps its first logged change_type. Must share the same transaction as the +// data write so the log rolls back if the write fails. +async function logDatasetChange( + historyId, + dbname, + dsname, + changeType, + transaction +) { + if (!historyId) return; // only the incremental path logs; firstSync passes none + await sequelize.query( + `INSERT INTO dataset_changes (history_id, dbname, dsname, change_type) + VALUES (:historyId, :dbname, :dsname, :changeType) + ON CONFLICT (history_id, dbname, dsname) DO NOTHING`, + { + replacements: { historyId, dbname, dsname, changeType }, + transaction, + } + ); +} + async function deleteDataset(dbname, dsname, transaction) { await sequelize.query( "DELETE FROM ioviews WHERE dbname = :dbname AND dsname = :dsname", @@ -385,7 +408,7 @@ async function firstSync(dbname) { // === Process one changed dataset (Option A: 2 HTTP requests + local transforms) === -async function processDatasetUpdate(dbname, dsname) { +async function processDatasetUpdate(dbname, dsname, historyId) { // dbinfo view supports key filtering; raw doc carries subjects; links view // is now filterable by dataset id (key = [doc._id, ext, size]) via a range // query, so links come straight from the view — same source as firstSync. @@ -421,6 +444,16 @@ async function processDatasetUpdate(dbname, dsname) { // Rule 1: wrap all writes for this dataset in one transaction. await sequelize.transaction(async (t) => { + // Determine added vs updated BEFORE the dbinfo upsert (which would create + // the row and make every dataset look pre-existing). Same transaction as + // the write so the change log is consistent with the data. + const existing = await sequelize.query( + `SELECT 1 FROM ioviews + WHERE dbname = :dbname AND dsname = :dsname AND view = 'dbinfo' LIMIT 1`, + { replacements: { dbname, dsname }, transaction: t, type: sequelize.QueryTypes.SELECT } + ); + const changeType = existing.length > 0 ? "updated" : "added"; + const subjCount = String(dbinfoValue?.subj?.length || 0); await upsertIoview(dbname, dsname, subjCount, "dbinfo", dbinfoValue, t); @@ -474,12 +507,15 @@ async function processDatasetUpdate(dbname, dsname) { t ); } + + // Log the dataset-level change in the SAME transaction as the data write. + await logDatasetChange(historyId, dbname, dsname, changeType, t); }); } // === Incremental sync === -async function incrementalSync(dbname, lastSeq) { +async function incrementalSync(dbname, lastSeq, historyId) { // No include_docs=true: we fetch the raw doc per dataset so the _changes // payload stays small and per-dataset work runs in parallel. const { data } = await axios.get( @@ -505,12 +541,20 @@ async function incrementalSync(dbname, lastSeq) { chunk.map(async (change) => { try { if (change.deleted) { - await sequelize.transaction((t) => - deleteDataset(dbname, change.id, t) - ); + // deleteDataset + change log share one transaction. + await sequelize.transaction(async (t) => { + await deleteDataset(dbname, change.id, t); + await logDatasetChange( + historyId, + dbname, + change.id, + "deleted", + t + ); + }); console.log(` ${dbname}/${change.id}: deleted`); } else { - await processDatasetUpdate(dbname, change.id); + await processDatasetUpdate(dbname, change.id, historyId); } } catch (err) { console.error(` ${dbname}/${change.id}: failed - ${err.message}`); @@ -526,7 +570,7 @@ async function incrementalSync(dbname, lastSeq) { // === Sync a single database === -async function syncDatabase(dbname) { +async function syncDatabase(dbname, historyId) { console.log(`\nSyncing ${dbname}...`); const lastSeq = await getLastSeq(dbname); @@ -537,10 +581,12 @@ async function syncDatabase(dbname) { // get picked up by the next incremental run. const { data: info } = await axios.get(`${COUCHDB_URL}/${dbname}`); const seqAtStart = String(info.update_seq); + // firstSync intentionally receives no historyId → it never logs to + // dataset_changes (a full/first sync must not mass-record every dataset). await firstSync(dbname); nextSeq = seqAtStart; } else { - nextSeq = await incrementalSync(dbname, lastSeq); + nextSeq = await incrementalSync(dbname, lastSeq, historyId); } await saveLastSeq(dbname, String(nextSeq)); @@ -605,7 +651,7 @@ async function runSync() { console.log(`Databases: ${databases.length}`); for (const db of databases) { - await syncDatabase(db); + await syncDatabase(db, historyId); } // Compute + publish totals only after the whole run completed. From d1011e9de9d74b2b38a6d397d05865c76f4ef35e Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 10 Sep 2026 13:01:02 -0400 Subject: [PATCH 11/18] feat(stats): add SYNC_DBS test override; verified add/update/delete on sandbox1d; refs #145 --- backend/sync/incrementalSync.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js index d3599d2..e68dcb7 100644 --- a/backend/sync/incrementalSync.js +++ b/backend/sync/incrementalSync.js @@ -10,6 +10,16 @@ const CONCURRENCY = 5; // fetch database list dynamically from registry // registry doc shape: { database: [{ id, name, ... }, ...] } async function getDatabases() { + // Optional override for testing/ops: sync only the named databases instead + // of the full registry, without touching the shared sys/registry doc. + // SYNC_DBS=sandbox1d,bfnirs node sync/incrementalSync.js + if (process.env.SYNC_DBS) { + const databases = process.env.SYNC_DBS.split(",") + .map((s) => s.trim()) + .filter(Boolean); + console.log(`SYNC_DBS override: ${databases.join(", ")}`); + return databases; + } const response = await axios.get(`${COUCHDB_URL}/sys/registry`); const entries = response.data?.database || []; const databases = entries.map((db) => db.id).filter(Boolean); From 3f0455f6a36ce13e3ce37e9b7e603aa1724f65fc Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 10 Sep 2026 13:14:38 -0400 Subject: [PATCH 12/18] feat(stats): add GET /dbs/updates/latest endpoint; #145 --- backend/src/controllers/couchdb.controller.js | 68 +++++++++++++++++++ backend/src/routes/dbs.routes.js | 5 ++ 2 files changed, 73 insertions(+) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index 8eb65e2..a0b6095 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -75,6 +75,73 @@ const getDbStats = async (req, res) => { } }; +// get the latest actual DATA update — distinct from "last synced". A cron sync +// runs every ~12h and always writes a stats_history snapshot, but most runs +// change nothing. This returns the most recent sync run that actually changed +// datasets (i.e. has dataset_changes rows), with those changes. +// +// Response: +// { historyId, updatedAt, changes: {added, updated, deleted}, +// datasets: [{dbname, dsname, changeType}] } +// or { historyId: null, ... } when nothing has ever changed. +const getLatestUpdate = async (req, res) => { + try { + // The newest run that produced any dataset changes. + const runRows = await sequelize.query( + `SELECT dc.history_id, sh.completed_at + FROM dataset_changes dc + JOIN stats_history sh ON sh.id = dc.history_id + WHERE sh.status = 'success' + ORDER BY dc.history_id DESC + LIMIT 1`, + { type: sequelize.QueryTypes.SELECT } + ); + const run = runRows[0]; + if (!run) { + return res.status(200).json({ + historyId: null, + updatedAt: null, + changes: { added: 0, updated: 0, deleted: 0 }, + datasets: [], + }); + } + + const rows = await sequelize.query( + `SELECT dbname, dsname, change_type + FROM dataset_changes + WHERE history_id = :historyId + ORDER BY change_type, dbname, dsname`, + { + replacements: { historyId: run.history_id }, + type: sequelize.QueryTypes.SELECT, + } + ); + + const changes = { added: 0, updated: 0, deleted: 0 }; + const datasets = rows.map((r) => { + if (changes[r.change_type] !== undefined) changes[r.change_type] += 1; + return { + dbname: r.dbname, + dsname: r.dsname, + changeType: r.change_type, + }; + }); + + res.status(200).json({ + historyId: run.history_id, + updatedAt: run.completed_at, + changes, + datasets, + }); + } catch (error) { + console.error("Error fetching latest update:", error.message); + res.status(error.response?.status || 500).json({ + message: "Error fetching latest update", + error: error.message, + }); + } +}; + // cross-database search — old version proxied to https://neurojson.org/io/search.cgi // kept for reference; replaced by the Postgres-backed version below. // const searchAllDatabases = async (req, res) => { @@ -677,6 +744,7 @@ const getFileTypes = async (req, res) => { module.exports = { getDbList, getDbStats, + getLatestUpdate, getDbInfo, getDbDatasets, searchAllDatabases, diff --git a/backend/src/routes/dbs.routes.js b/backend/src/routes/dbs.routes.js index 45979ac..43ae5aa 100644 --- a/backend/src/routes/dbs.routes.js +++ b/backend/src/routes/dbs.routes.js @@ -3,6 +3,7 @@ const express = require("express"); const { getDbList, getDbStats, + getLatestUpdate, getDbInfo, getDbDatasets, searchAllDatabases, @@ -17,6 +18,10 @@ const router = express.Router(); router.get("/", getDbList); router.get("/stats", getDbStats); +// latest actual data update (most recent sync run with dataset changes). +// Must come BEFORE the /:dbName route so "updates" isn't read as a dbName. +router.get("/updates/latest", getLatestUpdate); + // distinct file extensions across all iolinks rows (drives the file-type // filter on the search page). Must come BEFORE the /:dbName route, otherwise // Express treats "file-types" as a dbName. From afa608369efabd8d444f2e4fc888a645bb91e676 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 10 Sep 2026 13:21:50 -0400 Subject: [PATCH 13/18] feat(stats): add fetchLatestUpdate service/action/slice + types; refs #145 --- src/redux/neurojson/neurojson.action.ts | 12 ++++++++++++ src/redux/neurojson/neurojson.slice.ts | 8 ++++++++ src/redux/neurojson/types/neurojson.interface.ts | 15 +++++++++++++++ src/services/neurojson.service.ts | 6 ++++++ 4 files changed, 41 insertions(+) diff --git a/src/redux/neurojson/neurojson.action.ts b/src/redux/neurojson/neurojson.action.ts index f8f08c2..77e7ab3 100644 --- a/src/redux/neurojson/neurojson.action.ts +++ b/src/redux/neurojson/neurojson.action.ts @@ -97,6 +97,18 @@ export const fetchDbStats = createAsyncThunk( } ); +export const fetchLatestUpdate = createAsyncThunk( + "neurojson/fetchLatestUpdate", + async (_, { rejectWithValue }) => { + try { + const response = await NeurojsonService.getLatestUpdate(); + return response; + } catch (error: any) { + return rejectWithValue("Failed to fetch latest update"); + } + } +); + export const fetchMetadataSearchResults = createAsyncThunk( "neurojson/fetchMetadataSearchResults", async (formData: any, { rejectWithValue }) => { diff --git a/src/redux/neurojson/neurojson.slice.ts b/src/redux/neurojson/neurojson.slice.ts index cfafc6a..3c948c6 100644 --- a/src/redux/neurojson/neurojson.slice.ts +++ b/src/redux/neurojson/neurojson.slice.ts @@ -5,6 +5,7 @@ import { fetchDbInfo, fetchDocumentDetails, fetchDbStats, + fetchLatestUpdate, fetchMetadataSearchResults, fetchDbInfoByDatasetId, fetchFileTypes, @@ -25,6 +26,7 @@ const initialState: INeuroJsonState = { registry: null, dbInfo: null, // add dbInfo in neurojson.interface.ts dbStats: null, + latestUpdate: null, searchResults: null, datasetViewInfo: null, fileTypes: null, @@ -142,6 +144,12 @@ const neurojsonSlice = createSlice({ state.loading = false; state.error = action.payload as string; }) + .addCase( + fetchLatestUpdate.fulfilled, + (state, action: PayloadAction) => { + state.latestUpdate = action.payload; + } + ) .addCase(fetchMetadataSearchResults.pending, (state) => { state.loading = true; state.error = null; diff --git a/src/redux/neurojson/types/neurojson.interface.ts b/src/redux/neurojson/types/neurojson.interface.ts index fc3a7a0..9d18ecf 100644 --- a/src/redux/neurojson/types/neurojson.interface.ts +++ b/src/redux/neurojson/types/neurojson.interface.ts @@ -11,6 +11,7 @@ export interface INeuroJsonState { hasMore: boolean; dbInfo: DBParticulars | null; // add dbInfo type dbStats: DbStats | null; // landing-page stats snapshot (from stats_history) + latestUpdate: LatestUpdate | null; // most recent sync run with data changes searchResults: any[] | { status: string; msg: string } | null; datasetViewInfo: any | null; fileTypes: string[] | null; @@ -103,3 +104,17 @@ export interface DbStats { sizeBytes: number; lastSynced: string | null; } + +// Response of GET /dbs/updates/latest — the most recent sync run that actually +// changed datasets (distinct from "last synced"). +export interface DatasetChange { + dbname: string; + dsname: string; + changeType: "added" | "updated" | "deleted"; +} +export interface LatestUpdate { + historyId: number | null; + updatedAt: string | null; + changes: { added: number; updated: number; deleted: number }; + datasets: DatasetChange[]; +} diff --git a/src/services/neurojson.service.ts b/src/services/neurojson.service.ts index ed1f70d..bad872e 100644 --- a/src/services/neurojson.service.ts +++ b/src/services/neurojson.service.ts @@ -90,6 +90,12 @@ export const NeurojsonService = { return response.data; }, + // GET /api/v1/dbs/updates/latest → getLatestUpdate + getLatestUpdate: async () => { + const response = await api.get(`/dbs/updates/latest`); + return response.data; + }, + // getMetadataSearchResults: async (formData: any): Promise => { // const map: Record = { // keyword: "keyword", From f6732d6c859bfd01bd99e55c798911293a89c274 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 10 Sep 2026 13:26:39 -0400 Subject: [PATCH 14/18] feat(stats): add LatestUpdateBoard to the landing page; refs #145 --- .../HomePageComponents/Section1.tsx | 2 + src/components/LatestUpdateBoard.tsx | 132 ++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 src/components/LatestUpdateBoard.tsx diff --git a/src/components/HomePageComponents/Section1.tsx b/src/components/HomePageComponents/Section1.tsx index 57a4a01..d4a1502 100644 --- a/src/components/HomePageComponents/Section1.tsx +++ b/src/components/HomePageComponents/Section1.tsx @@ -8,6 +8,7 @@ import { Grid, IconButton, } from "@mui/material"; +import LatestUpdateBoard from "components/LatestUpdateBoard"; import StatisticsBanner from "components/StatisticsBanner"; import { Colors } from "design/theme"; import pako from "pako"; @@ -168,6 +169,7 @@ const Section1: React.FC = ({ scrollToNext }) => { }} > + diff --git a/src/components/LatestUpdateBoard.tsx b/src/components/LatestUpdateBoard.tsx new file mode 100644 index 0000000..e4494f6 --- /dev/null +++ b/src/components/LatestUpdateBoard.tsx @@ -0,0 +1,132 @@ +import { Box, Chip, Typography, Link as MuiLink } from "@mui/material"; +import { Colors } from "design/theme"; +import { useAppDispatch } from "hooks/useAppDispatch"; +import { useAppSelector } from "hooks/useAppSelector"; +import React, { useEffect } from "react"; +import { Link } from "react-router-dom"; +import { fetchLatestUpdate } from "redux/neurojson/neurojson.action"; +import { DatasetChange } from "redux/neurojson/types/neurojson.interface"; +import { RootState } from "redux/store"; +import RoutesEnum from "types/routes.enum"; + +const MAX_DATASETS = 5; + +// Colors per change type (light enough for the dark hero background). +const changeColor: Record = { + added: Colors.lightGreen, + updated: Colors.accent, + deleted: "#ff8a80", +}; +const changeSign: Record = { + added: "+", + updated: "~", + deleted: "−", +}; + +const formatDate = (iso: string | null): string => { + if (!iso) return ""; + const d = new Date(iso); + if (isNaN(d.getTime())) return ""; + return d.toLocaleDateString(undefined, { + year: "numeric", + month: "short", + day: "numeric", + }); +}; + +const LatestUpdateBoard: React.FC = () => { + const dispatch = useAppDispatch(); + const latest = useAppSelector( + (state: RootState) => state.neurojson.latestUpdate + ); + + useEffect(() => { + dispatch(fetchLatestUpdate()); + }, [dispatch]); + + // Nothing has ever changed → render nothing (keeps the hero uncluttered). + if (!latest || latest.historyId === null || latest.datasets.length === 0) { + return null; + } + + const { changes, datasets } = latest; + const shown = datasets.slice(0, MAX_DATASETS); + const remaining = datasets.length - shown.length; + + const countChip = ( + label: string, + n: number, + type: DatasetChange["changeType"] + ) => + n > 0 ? ( + + ) : null; + + return ( + + + + Latest update · {formatDate(latest.updatedAt)} + + {countChip("added", changes.added, "added")} + {countChip("updated", changes.updated, "updated")} + {countChip("deleted", changes.deleted, "deleted")} + + + + {shown.map((d, i) => ( + + {d.dbname}/{d.dsname} ({d.changeType}) + + ))} + {remaining > 0 && ( + + +{remaining} more + + )} + + + ); +}; + +export default LatestUpdateBoard; From 80c257c126270d90374e66c310029740311d4e02 Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 10 Sep 2026 13:29:04 -0400 Subject: [PATCH 15/18] fix(stats): label LatestUpdateBoard chips as dataset(s) for clarity; refs #145 --- src/components/LatestUpdateBoard.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/LatestUpdateBoard.tsx b/src/components/LatestUpdateBoard.tsx index e4494f6..711d4a6 100644 --- a/src/components/LatestUpdateBoard.tsx +++ b/src/components/LatestUpdateBoard.tsx @@ -61,7 +61,7 @@ const LatestUpdateBoard: React.FC = () => { n > 0 ? ( Date: Thu, 10 Sep 2026 14:46:02 -0400 Subject: [PATCH 16/18] feat(stats): add subject/file/size deltas to /dbs/updates/latest; refs #145 --- backend/src/controllers/couchdb.controller.js | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index a0b6095..7aa565b 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -81,9 +81,13 @@ const getDbStats = async (req, res) => { // datasets (i.e. has dataset_changes rows), with those changes. // // Response: -// { historyId, updatedAt, changes: {added, updated, deleted}, +// { historyId, updatedAt, +// changes: {added, updated, deleted}, // datasets (dataset_changes) +// deltas: {subjects, files, sizeBytes} | null, // NET, from snapshot diff // datasets: [{dbname, dsname, changeType}] } // or { historyId: null, ... } when nothing has ever changed. +// deltas is null when there is no previous successful snapshot to compare +// against (so the first-ever snapshot doesn't look like a huge update). const getLatestUpdate = async (req, res) => { try { // The newest run that produced any dataset changes. @@ -102,6 +106,7 @@ const getLatestUpdate = async (req, res) => { historyId: null, updatedAt: null, changes: { added: 0, updated: 0, deleted: 0 }, + deltas: null, datasets: [], }); } @@ -127,10 +132,40 @@ const getLatestUpdate = async (req, res) => { }; }); + // Subject/file/size deltas come from stats_history snapshot differences + // (dataset_changes only tracks datasets). Anchor on THIS run's snapshot and + // the previous SUCCESSFUL snapshot before it (skip failed/running) — NOT + // the global last-two rows, since later no-change syncs would zero it out. + // No previous snapshot → deltas: null (avoid a misleading "huge" first run). + const snapRows = await sequelize.query( + `SELECT id, total_subjects, total_files, total_size_bytes + FROM stats_history + WHERE status = 'success' + AND id <= :historyId + ORDER BY id DESC + LIMIT 2`, + { + replacements: { historyId: run.history_id }, + type: sequelize.QueryTypes.SELECT, + } + ); + let deltas = null; + if (snapRows.length === 2) { + const cur = snapRows[0]; + const prev = snapRows[1]; + deltas = { + subjects: Number(cur.total_subjects) - Number(prev.total_subjects), + files: Number(cur.total_files) - Number(prev.total_files), + sizeBytes: + Number(cur.total_size_bytes) - Number(prev.total_size_bytes), + }; + } + res.status(200).json({ historyId: run.history_id, updatedAt: run.completed_at, changes, + deltas, datasets, }); } catch (error) { From 9bb6ba32e911fcc47f8f125f96e974cd197f660a Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 10 Sep 2026 15:18:57 -0400 Subject: [PATCH 17/18] style(stats): refine LatestUpdateBoard typography, colors, and alignment; refs #145 --- src/components/LatestUpdateBoard.tsx | 255 ++++++++++++++---- .../neurojson/types/neurojson.interface.ts | 3 + 2 files changed, 204 insertions(+), 54 deletions(-) diff --git a/src/components/LatestUpdateBoard.tsx b/src/components/LatestUpdateBoard.tsx index 711d4a6..0bb866e 100644 --- a/src/components/LatestUpdateBoard.tsx +++ b/src/components/LatestUpdateBoard.tsx @@ -1,5 +1,4 @@ -import { Box, Chip, Typography, Link as MuiLink } from "@mui/material"; -import { Colors } from "design/theme"; +import { Box, Typography, Link as MuiLink } from "@mui/material"; import { useAppDispatch } from "hooks/useAppDispatch"; import { useAppSelector } from "hooks/useAppSelector"; import React, { useEffect } from "react"; @@ -11,17 +10,39 @@ import RoutesEnum from "types/routes.enum"; const MAX_DATASETS = 5; -// Colors per change type (light enough for the dark hero background). -const changeColor: Record = { - added: Colors.lightGreen, - updated: Colors.accent, - deleted: "#ff8a80", +// Palette for this component (integrated with the dark navy/purple hero). +// Green-family accents only — no red styling. +const C = { + primary: "#F4F4FF", // near-white + muted: "#a0a5c2", // primary.light + link: "#a0a5c2", // primary.light (lightGreen on hover) + lightGreen: "#16FDE2", + darkGreen: "#49c6ae", }; -const changeSign: Record = { + +type ChangeType = DatasetChange["changeType"]; + +// Status-badge colors (green-family only, no red). +const badgeText: Record = { + added: C.lightGreen, + updated: C.muted, + deleted: C.darkGreen, +}; +const changeBadgeBg: Record = { + added: "rgba(22, 253, 226, 0.10)", + updated: "rgba(123, 129, 165, 0.16)", + deleted: "rgba(73, 198, 174, 0.12)", +}; +const changeSign: Record = { added: "+", updated: "~", deleted: "−", }; +const badgeLabel: Record = { + added: "Added", + updated: "Updated", + deleted: "Deleted", +}; const formatDate = (iso: string | null): string => { if (!iso) return ""; @@ -34,6 +55,54 @@ const formatDate = (iso: string | null): string => { }); }; +const signed = (n: number): string => + `${n > 0 ? "+" : n < 0 ? "−" : ""}${Math.abs(n)}`; + +const signedBytes = (n: number): string => { + const sign = n > 0 ? "+" : n < 0 ? "−" : ""; + let v = Math.abs(n); + const units = ["B", "KB", "MB", "GB", "TB", "PB"]; + let u = 0; + while (v >= 1024 && u < units.length - 1) { + v /= 1024; + u += 1; + } + return `${sign}${v.toFixed(v < 10 && u > 0 ? 1 : 0)} ${units[u]}`; +}; + +// A single metric: medium-weight number + slightly smaller label. +const Metric: React.FC<{ + value: string; + label: string; + numberColor: string; + labelColor: string; +}> = ({ value, label, numberColor, labelColor }) => ( + + + {value} + + + {label} + + +); + const LatestUpdateBoard: React.FC = () => { const dispatch = useAppDispatch(); const latest = useAppSelector( @@ -49,78 +118,156 @@ const LatestUpdateBoard: React.FC = () => { return null; } - const { changes, datasets } = latest; + const { changes, deltas, datasets } = latest; const shown = datasets.slice(0, MAX_DATASETS); const remaining = datasets.length - shown.length; - const countChip = ( - label: string, - n: number, - type: DatasetChange["changeType"] - ) => - n > 0 ? ( - - ) : null; + // Dataset-change metrics (semantic color); subject/file/size deltas (neutral). + const datasetMetrics = (["added", "updated", "deleted"] as ChangeType[]) + .filter((t) => changes[t] > 0) + .map((t) => { + const n = changes[t]; + return { + key: t, + value: `${changeSign[t]}${n}`, + label: n === 1 ? "Dataset" : "Datasets", + }; + }); return ( - + + {/* Line 1 — "Latest update · " */} + + + Latest update + + + {" · "} + {formatDate(latest.updatedAt)} + + + + {/* Line 2 — metrics, evenly spaced, lightweight (no bordered chips) */} - - Latest update · {formatDate(latest.updatedAt)} - - {countChip("added", changes.added, "added")} - {countChip("updated", changes.updated, "updated")} - {countChip("deleted", changes.deleted, "deleted")} + {datasetMetrics.map((m) => ( + + ))} + {deltas && deltas.subjects !== 0 && ( + + )} + {deltas && deltas.files !== 0 && ( + + )} + {deltas && deltas.sizeBytes !== 0 && ( + + )} + {/* Line 3+ — affected datasets, left-aligned; badge next to the name */} {shown.map((d, i) => ( - - {d.dbname}/{d.dsname} ({d.changeType}) - + + {d.dbname}/{d.dsname} + + + {badgeLabel[d.changeType]} + + ))} {remaining > 0 && ( - + +{remaining} more )} diff --git a/src/redux/neurojson/types/neurojson.interface.ts b/src/redux/neurojson/types/neurojson.interface.ts index 9d18ecf..d461526 100644 --- a/src/redux/neurojson/types/neurojson.interface.ts +++ b/src/redux/neurojson/types/neurojson.interface.ts @@ -116,5 +116,8 @@ export interface LatestUpdate { historyId: number | null; updatedAt: string | null; changes: { added: number; updated: number; deleted: number }; + // Net subject/file/size deltas vs the previous snapshot; null when there is + // no previous successful snapshot to compare against. + deltas: { subjects: number; files: number; sizeBytes: number } | null; datasets: DatasetChange[]; } From eb733289c0dfe94a26f025625e2e86feafcf96dc Mon Sep 17 00:00:00 2001 From: elainefan331 Date: Thu, 10 Sep 2026 15:28:22 -0400 Subject: [PATCH 18/18] style(stats): finalize LatestUpdateBoard styling; refs #145 --- src/components/LatestUpdateBoard.tsx | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/components/LatestUpdateBoard.tsx b/src/components/LatestUpdateBoard.tsx index 0bb866e..b8af5b2 100644 --- a/src/components/LatestUpdateBoard.tsx +++ b/src/components/LatestUpdateBoard.tsx @@ -17,22 +17,14 @@ const C = { muted: "#a0a5c2", // primary.light link: "#a0a5c2", // primary.light (lightGreen on hover) lightGreen: "#16FDE2", - darkGreen: "#49c6ae", }; type ChangeType = DatasetChange["changeType"]; -// Status-badge colors (green-family only, no red). -const badgeText: Record = { - added: C.lightGreen, - updated: C.muted, - deleted: C.darkGreen, -}; -const changeBadgeBg: Record = { - added: "rgba(22, 253, 226, 0.10)", - updated: "rgba(123, 129, 165, 0.16)", - deleted: "rgba(73, 198, 174, 0.12)", -}; +// Status badges share one consistent green; the change type is conveyed by the +// badge text (Added / Updated / Deleted), not by color. +const BADGE_TEXT = C.lightGreen; +const BADGE_BG = "rgba(22, 253, 226, 0.10)"; const changeSign: Record = { added: "+", updated: "~", @@ -250,8 +242,8 @@ const LatestUpdateBoard: React.FC = () => { fontSize: "0.75rem", fontWeight: 500, lineHeight: 1.6, - color: badgeText[d.changeType], - backgroundColor: changeBadgeBg[d.changeType], + color: BADGE_TEXT, + backgroundColor: BADGE_BG, }} > {badgeLabel[d.changeType]}