diff --git a/backend/migrations/20260903145021-add-iolinks-dbname-dsname-view-index.js b/backend/migrations/20260903145021-add-iolinks-dbname-dsname-view-index.js new file mode 100644 index 0000000..a985659 --- /dev/null +++ b/backend/migrations/20260903145021-add-iolinks-dbname-dsname-view-index.js @@ -0,0 +1,26 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // Composite index for the file-type filter's EXISTS subquery in + // searchAllDatabases: + // EXISTS (SELECT 1 FROM iolinks l + // WHERE l.dbname = ioviews.dbname AND l.dsname = ioviews.dsname + // AND l.view IN (:fileTypes)) + // Without it the planner uses idx_iolinks_dbname alone and scans every row + // of a database (openneuro ~1.4M), filtering dsname/view in memory — the + // dominant cost of a subject search combined with a file_type filter. This + // turns that into a direct lookup on (dbname, dsname, view). + async up (queryInterface, Sequelize) { + await queryInterface.addIndex("iolinks", ["dbname", "dsname", "view"], { + name: "idx_iolinks_dbname_dsname_view", + }); + }, + + async down (queryInterface, Sequelize) { + await queryInterface.removeIndex( + "iolinks", + "idx_iolinks_dbname_dsname_view" + ); + } +}; diff --git a/backend/migrations/20260903180911-create-stats-history.js b/backend/migrations/20260903180911-create-stats-history.js new file mode 100644 index 0000000..93be320 --- /dev/null +++ b/backend/migrations/20260903180911-create-stats-history.js @@ -0,0 +1,61 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // One row per scheduled sync run. Doubles as (1) persistent landing-page + // stats and (2) sync execution history. Lifecycle: inserted 'running' at + // sync start; finalized 'success' (with totals) or 'failed' (with error) + // after the run completes. The latest 'success' row = current NeuroJSON state. + async up (queryInterface, Sequelize) { + await queryInterface.createTable("stats_history", { + id: { + type: Sequelize.INTEGER, + autoIncrement: true, + primaryKey: true, + allowNull: false, + }, + started_at: { + type: Sequelize.DATE, // TIMESTAMPTZ + allowNull: false, + }, + completed_at: { + type: Sequelize.DATE, + allowNull: true, + }, + status: { + type: Sequelize.TEXT, // 'running' | 'success' | 'failed' + allowNull: false, + defaultValue: "running", + }, + total_datasets: { + type: Sequelize.BIGINT, + allowNull: true, + }, + total_subjects: { + type: Sequelize.BIGINT, + allowNull: true, + }, + total_files: { + type: Sequelize.BIGINT, + allowNull: true, + }, + total_size_bytes: { + type: Sequelize.BIGINT, + allowNull: true, + }, + error: { + type: Sequelize.TEXT, // failure reason when status = 'failed' + allowNull: true, + }, + created_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + }, + + async down (queryInterface, Sequelize) { + await queryInterface.dropTable("stats_history"); + } +}; diff --git a/backend/migrations/20260910160052-create-dataset-changes.js b/backend/migrations/20260910160052-create-dataset-changes.js new file mode 100644 index 0000000..0ee46b1 --- /dev/null +++ b/backend/migrations/20260910160052-create-dataset-changes.js @@ -0,0 +1,59 @@ +'use strict'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + // One row per dataset-level change detected during an incremental sync. + // Only the incremental path (processDatasetUpdate / delete branch) writes + // here — firstSync never logs, so a full/first sync does not mass-record + // every dataset as "added". Powers the "latest update" board on the landing + // page. history_id → stats_history.id (the sync run that produced the change). + async up (queryInterface, Sequelize) { + await queryInterface.createTable("dataset_changes", { + id: { + type: Sequelize.INTEGER, + autoIncrement: true, + primaryKey: true, + allowNull: false, + }, + history_id: { + type: Sequelize.INTEGER, // FK → stats_history.id (same INTEGER type) + allowNull: false, + references: { model: "stats_history", key: "id" }, + onDelete: "CASCADE", + }, + dbname: { + type: Sequelize.TEXT, + allowNull: false, + }, + dsname: { + type: Sequelize.TEXT, + allowNull: false, + }, + change_type: { + type: Sequelize.TEXT, // 'added' | 'updated' | 'deleted' + allowNull: false, + }, + created_at: { + type: Sequelize.DATE, + allowNull: false, + defaultValue: Sequelize.literal("CURRENT_TIMESTAMP"), + }, + }); + + // Dedupe: one logical change per dataset per sync run (a dataset touched + // multiple times in one sync → ON CONFLICT DO NOTHING on this key). + await queryInterface.addIndex( + "dataset_changes", + ["history_id", "dbname", "dsname"], + { name: "uq_dataset_changes_run_dataset", unique: true } + ); + // "latest update" lookup filters/groups by run. + await queryInterface.addIndex("dataset_changes", ["history_id"], { + name: "idx_dataset_changes_history_id", + }); + }, + + async down (queryInterface, Sequelize) { + await queryInterface.dropTable("dataset_changes"); + } +}; diff --git a/backend/src/controllers/couchdb.controller.js b/backend/src/controllers/couchdb.controller.js index ab12d0f..7aa565b 100644 --- a/backend/src/controllers/couchdb.controller.js +++ b/backend/src/controllers/couchdb.controller.js @@ -23,13 +23,49 @@ const getDbList = async (req, res) => { } }; -// get db stats +// get db stats — reads the latest finalized snapshot from stats_history +// (written once at the end of each sync), so the landing page never triggers a +// live aggregate over ioviews/iolinks. +// +// History of this endpoint: +// 1. proxied the legacy CGI: https://neurojson.org/io/search.cgi?dbstats=1 +// (decoupled from the synced Postgres — stale/junk) +// 2. live Postgres aggregate over iolinks/ioviews (correct but scanned ~1.5M +// rows on every landing-page visit) +// 3. this: read the precomputed stats_history snapshot (tiny query) +// +// Returns: +// { datasets, subjects, files, sizeBytes, lastSynced } const getDbStats = async (req, res) => { try { - const response = await axios.get( - "https://neurojson.org/io/search.cgi?dbstats=1" + const rows = await sequelize.query( + `SELECT total_datasets, total_subjects, total_files, + total_size_bytes, completed_at + FROM stats_history + WHERE status = 'success' + ORDER BY completed_at DESC + LIMIT 1`, + { type: sequelize.QueryTypes.SELECT } ); - res.status(200).json(response.data); + const row = rows[0]; + if (!row) { + // No successful sync snapshot yet. + return res.status(200).json({ + datasets: 0, + subjects: 0, + files: 0, + sizeBytes: 0, + lastSynced: null, + }); + } + // Sequelize returns BIGINT as a string; coerce to Number for the frontend. + res.status(200).json({ + datasets: Number(row.total_datasets), + subjects: Number(row.total_subjects), + files: Number(row.total_files), + sizeBytes: Number(row.total_size_bytes), + lastSynced: row.completed_at, + }); } catch (error) { console.error("Error fetching db stats:", error.message); res.status(error.response?.status || 500).json({ @@ -39,6 +75,108 @@ const getDbStats = async (req, res) => { } }; +// get the latest actual DATA update — distinct from "last synced". A cron sync +// runs every ~12h and always writes a stats_history snapshot, but most runs +// change nothing. This returns the most recent sync run that actually changed +// datasets (i.e. has dataset_changes rows), with those changes. +// +// Response: +// { historyId, updatedAt, +// changes: {added, updated, deleted}, // datasets (dataset_changes) +// deltas: {subjects, files, sizeBytes} | null, // NET, from snapshot diff +// datasets: [{dbname, dsname, changeType}] } +// or { historyId: null, ... } when nothing has ever changed. +// deltas is null when there is no previous successful snapshot to compare +// against (so the first-ever snapshot doesn't look like a huge update). +const getLatestUpdate = async (req, res) => { + try { + // The newest run that produced any dataset changes. + const runRows = await sequelize.query( + `SELECT dc.history_id, sh.completed_at + FROM dataset_changes dc + JOIN stats_history sh ON sh.id = dc.history_id + WHERE sh.status = 'success' + ORDER BY dc.history_id DESC + LIMIT 1`, + { type: sequelize.QueryTypes.SELECT } + ); + const run = runRows[0]; + if (!run) { + return res.status(200).json({ + historyId: null, + updatedAt: null, + changes: { added: 0, updated: 0, deleted: 0 }, + deltas: null, + datasets: [], + }); + } + + const rows = await sequelize.query( + `SELECT dbname, dsname, change_type + FROM dataset_changes + WHERE history_id = :historyId + ORDER BY change_type, dbname, dsname`, + { + replacements: { historyId: run.history_id }, + type: sequelize.QueryTypes.SELECT, + } + ); + + const changes = { added: 0, updated: 0, deleted: 0 }; + const datasets = rows.map((r) => { + if (changes[r.change_type] !== undefined) changes[r.change_type] += 1; + return { + dbname: r.dbname, + dsname: r.dsname, + changeType: r.change_type, + }; + }); + + // Subject/file/size deltas come from stats_history snapshot differences + // (dataset_changes only tracks datasets). Anchor on THIS run's snapshot and + // the previous SUCCESSFUL snapshot before it (skip failed/running) — NOT + // the global last-two rows, since later no-change syncs would zero it out. + // No previous snapshot → deltas: null (avoid a misleading "huge" first run). + const snapRows = await sequelize.query( + `SELECT id, total_subjects, total_files, total_size_bytes + FROM stats_history + WHERE status = 'success' + AND id <= :historyId + ORDER BY id DESC + LIMIT 2`, + { + replacements: { historyId: run.history_id }, + type: sequelize.QueryTypes.SELECT, + } + ); + let deltas = null; + if (snapRows.length === 2) { + const cur = snapRows[0]; + const prev = snapRows[1]; + deltas = { + subjects: Number(cur.total_subjects) - Number(prev.total_subjects), + files: Number(cur.total_files) - Number(prev.total_files), + sizeBytes: + Number(cur.total_size_bytes) - Number(prev.total_size_bytes), + }; + } + + res.status(200).json({ + historyId: run.history_id, + updatedAt: run.completed_at, + changes, + deltas, + datasets, + }); + } catch (error) { + console.error("Error fetching latest update:", error.message); + res.status(error.response?.status || 500).json({ + message: "Error fetching latest update", + error: error.message, + }); + } +}; + // cross-database search — old version proxied to https://neurojson.org/io/search.cgi // kept for reference; replaced by the Postgres-backed version below. // const searchAllDatabases = async (req, res) => { @@ -259,12 +397,34 @@ const searchAllDatabases = async (req, res) => { // "ABIDE - CMU_a" matches stored names regardless of separator style. // The whole group is parenthesised so it ANDs cleanly with other filters. if (isFilter(f.keyword)) { - where.push(`( - search_vector @@ plainto_tsquery('english', :keyword) - OR dbname ILIKE :keywordLike - OR dsname ILIKE :keywordLike - OR (json->>'name') ILIKE :keywordLike - )`); + // The keyword matches dataset-level text (name / README / AI summary), + // which lives only in dbinfo rows. On a subjects search the current row + // has no such text, so match the keyword against the dataset's dbinfo + // row via EXISTS — mirrors the modalities cross-view pattern above. + // Without this, any dataset-level keyword wrongly drops all subjects (it + // only appeared to work when the keyword happened to be a subject-level + // token such as a task name, e.g. "memory"). + if (isSubjectSearch) { + where.push(`EXISTS ( + SELECT 1 FROM ioviews dsi + WHERE dsi.dbname = ioviews.dbname + AND dsi.dsname = ioviews.dsname + AND dsi.view = 'dbinfo' + AND ( + dsi.search_vector @@ plainto_tsquery('english', :keyword) + OR dsi.dbname ILIKE :keywordLike + OR dsi.dsname ILIKE :keywordLike + OR (dsi.json->>'name') ILIKE :keywordLike + ) + )`); + } else { + where.push(`( + search_vector @@ plainto_tsquery('english', :keyword) + OR dbname ILIKE :keywordLike + OR dsname ILIKE :keywordLike + OR (json->>'name') ILIKE :keywordLike + )`); + } repl.keyword = String(f.keyword); repl.keywordLike = `%${String(f.keyword).replace(/[\s-]+/g, "%")}%`; } @@ -619,6 +779,7 @@ const getFileTypes = async (req, res) => { module.exports = { getDbList, getDbStats, + getLatestUpdate, getDbInfo, getDbDatasets, searchAllDatabases, diff --git a/backend/src/routes/dbs.routes.js b/backend/src/routes/dbs.routes.js index 45979ac..43ae5aa 100644 --- a/backend/src/routes/dbs.routes.js +++ b/backend/src/routes/dbs.routes.js @@ -3,6 +3,7 @@ const express = require("express"); const { getDbList, getDbStats, + getLatestUpdate, getDbInfo, getDbDatasets, searchAllDatabases, @@ -17,6 +18,10 @@ const router = express.Router(); router.get("/", getDbList); router.get("/stats", getDbStats); +// latest actual data update (most recent sync run with dataset changes). +// Must come BEFORE the /:dbName route so "updates" isn't read as a dbName. +router.get("/updates/latest", getLatestUpdate); + // distinct file extensions across all iolinks rows (drives the file-type // filter on the search page). Must come BEFORE the /:dbName route, otherwise // Express treats "file-types" as a dbName. diff --git a/backend/sync/incrementalSync.js b/backend/sync/incrementalSync.js index f563905..e68dcb7 100644 --- a/backend/sync/incrementalSync.js +++ b/backend/sync/incrementalSync.js @@ -10,6 +10,16 @@ const CONCURRENCY = 5; // fetch database list dynamically from registry // registry doc shape: { database: [{ id, name, ... }, ...] } async function getDatabases() { + // Optional override for testing/ops: sync only the named databases instead + // of the full registry, without touching the shared sys/registry doc. + // SYNC_DBS=sandbox1d,bfnirs node sync/incrementalSync.js + if (process.env.SYNC_DBS) { + const databases = process.env.SYNC_DBS.split(",") + .map((s) => s.trim()) + .filter(Boolean); + console.log(`SYNC_DBS override: ${databases.join(", ")}`); + return databases; + } const response = await axios.get(`${COUCHDB_URL}/sys/registry`); const entries = response.data?.database || []; const databases = entries.map((db) => db.id).filter(Boolean); @@ -317,6 +327,29 @@ async function insertIolink(dbname, dsname, subj, view, json, transaction) { ); } +// Record one dataset-level change for a sync run. Deduped by the unique +// (history_id, dbname, dsname) index, so a dataset touched twice in one run +// keeps its first logged change_type. Must share the same transaction as the +// data write so the log rolls back if the write fails. +async function logDatasetChange( + historyId, + dbname, + dsname, + changeType, + transaction +) { + if (!historyId) return; // only the incremental path logs; firstSync passes none + await sequelize.query( + `INSERT INTO dataset_changes (history_id, dbname, dsname, change_type) + VALUES (:historyId, :dbname, :dsname, :changeType) + ON CONFLICT (history_id, dbname, dsname) DO NOTHING`, + { + replacements: { historyId, dbname, dsname, changeType }, + transaction, + } + ); +} + async function deleteDataset(dbname, dsname, transaction) { await sequelize.query( "DELETE FROM ioviews WHERE dbname = :dbname AND dsname = :dsname", @@ -385,7 +418,7 @@ async function firstSync(dbname) { // === Process one changed dataset (Option A: 2 HTTP requests + local transforms) === -async function processDatasetUpdate(dbname, dsname) { +async function processDatasetUpdate(dbname, dsname, historyId) { // dbinfo view supports key filtering; raw doc carries subjects; links view // is now filterable by dataset id (key = [doc._id, ext, size]) via a range // query, so links come straight from the view — same source as firstSync. @@ -421,6 +454,16 @@ async function processDatasetUpdate(dbname, dsname) { // Rule 1: wrap all writes for this dataset in one transaction. await sequelize.transaction(async (t) => { + // Determine added vs updated BEFORE the dbinfo upsert (which would create + // the row and make every dataset look pre-existing). Same transaction as + // the write so the change log is consistent with the data. + const existing = await sequelize.query( + `SELECT 1 FROM ioviews + WHERE dbname = :dbname AND dsname = :dsname AND view = 'dbinfo' LIMIT 1`, + { replacements: { dbname, dsname }, transaction: t, type: sequelize.QueryTypes.SELECT } + ); + const changeType = existing.length > 0 ? "updated" : "added"; + const subjCount = String(dbinfoValue?.subj?.length || 0); await upsertIoview(dbname, dsname, subjCount, "dbinfo", dbinfoValue, t); @@ -474,12 +517,15 @@ async function processDatasetUpdate(dbname, dsname) { t ); } + + // Log the dataset-level change in the SAME transaction as the data write. + await logDatasetChange(historyId, dbname, dsname, changeType, t); }); } // === Incremental sync === -async function incrementalSync(dbname, lastSeq) { +async function incrementalSync(dbname, lastSeq, historyId) { // No include_docs=true: we fetch the raw doc per dataset so the _changes // payload stays small and per-dataset work runs in parallel. const { data } = await axios.get( @@ -505,12 +551,20 @@ async function incrementalSync(dbname, lastSeq) { chunk.map(async (change) => { try { if (change.deleted) { - await sequelize.transaction((t) => - deleteDataset(dbname, change.id, t) - ); + // deleteDataset + change log share one transaction. + await sequelize.transaction(async (t) => { + await deleteDataset(dbname, change.id, t); + await logDatasetChange( + historyId, + dbname, + change.id, + "deleted", + t + ); + }); console.log(` ${dbname}/${change.id}: deleted`); } else { - await processDatasetUpdate(dbname, change.id); + await processDatasetUpdate(dbname, change.id, historyId); } } catch (err) { console.error(` ${dbname}/${change.id}: failed - ${err.message}`); @@ -526,7 +580,7 @@ async function incrementalSync(dbname, lastSeq) { // === Sync a single database === -async function syncDatabase(dbname) { +async function syncDatabase(dbname, historyId) { console.log(`\nSyncing ${dbname}...`); const lastSeq = await getLastSeq(dbname); @@ -537,10 +591,12 @@ async function syncDatabase(dbname) { // get picked up by the next incremental run. const { data: info } = await axios.get(`${COUCHDB_URL}/${dbname}`); const seqAtStart = String(info.update_seq); + // firstSync intentionally receives no historyId → it never logs to + // dataset_changes (a full/first sync must not mass-record every dataset). await firstSync(dbname); nextSeq = seqAtStart; } else { - nextSeq = await incrementalSync(dbname, lastSeq); + nextSeq = await incrementalSync(dbname, lastSeq, historyId); } await saveLastSeq(dbname, String(nextSeq)); @@ -550,6 +606,46 @@ async function syncDatabase(dbname) { } } +// === stats_history: one row per sync run === + +// Insert a 'running' row at sync start; return its id. The id will also back +// per-entity change logging (db_change_log) in a later phase. +async function createStatsHistory() { + const [rows] = await sequelize.query( + `INSERT INTO stats_history (started_at, status) + VALUES (NOW(), 'running') RETURNING id` + ); + return rows[0].id; +} + +// Finalize the run's row. On success, compute totals from the fully-synced +// tables (done ONCE here, never per landing-page visit); on failure, record +// the error. Totals stay NULL for a failed/partial run. +async function finalizeStatsHistory(id, status, errorMessage) { + if (status !== "success") { + await sequelize.query( + `UPDATE stats_history + SET completed_at = NOW(), status = :status, error = :error + WHERE id = :id`, + { replacements: { id, status, error: errorMessage || null } } + ); + return; + } + await sequelize.query( + `UPDATE stats_history SET + completed_at = NOW(), + status = 'success', + total_datasets = (SELECT count(*) FROM ioviews WHERE view = 'dbinfo'), + total_subjects = (SELECT count(*) FROM ioviews WHERE view = 'subjects'), + total_files = (SELECT count(*) FROM iolinks), + total_size_bytes = (SELECT COALESCE( + sum(CASE WHEN subj ~ '^[0-9]+$' THEN subj::bigint ELSE 0 END), 0) + FROM iolinks) + WHERE id = :id`, + { replacements: { id } } + ); +} + // === Main === async function runSync() { @@ -557,16 +653,27 @@ async function runSync() { console.log(new Date().toISOString()); console.log(`CouchDB: ${COUCHDB_URL}`); - const databases = await getDatabases(); - console.log(`Databases: ${databases.length}`); + // Create the run's history row up front so it exists throughout the sync. + const historyId = await createStatsHistory(); - for (const db of databases) { - await syncDatabase(db); - } + try { + const databases = await getDatabases(); + console.log(`Databases: ${databases.length}`); - await sequelize.close(); - console.log("\n=== Sync complete ==="); - console.log(new Date().toISOString()); + for (const db of databases) { + await syncDatabase(db, historyId); + } + + // Compute + publish totals only after the whole run completed. + await finalizeStatsHistory(historyId, "success"); + console.log(`\n=== Sync complete (stats_history #${historyId}) ===`); + console.log(new Date().toISOString()); + } catch (err) { + await finalizeStatsHistory(historyId, "failed", err.message); + throw err; + } finally { + await sequelize.close(); + } } runSync().catch((err) => { diff --git a/src/components/HomePageComponents/Section1.tsx b/src/components/HomePageComponents/Section1.tsx index 57a4a01..d4a1502 100644 --- a/src/components/HomePageComponents/Section1.tsx +++ b/src/components/HomePageComponents/Section1.tsx @@ -8,6 +8,7 @@ import { Grid, IconButton, } from "@mui/material"; +import LatestUpdateBoard from "components/LatestUpdateBoard"; import StatisticsBanner from "components/StatisticsBanner"; import { Colors } from "design/theme"; import pako from "pako"; @@ -168,6 +169,7 @@ const Section1: React.FC = ({ scrollToNext }) => { }} > + diff --git a/src/components/LatestUpdateBoard.tsx b/src/components/LatestUpdateBoard.tsx new file mode 100644 index 0000000..b8af5b2 --- /dev/null +++ b/src/components/LatestUpdateBoard.tsx @@ -0,0 +1,271 @@ +import { Box, Typography, Link as MuiLink } from "@mui/material"; +import { useAppDispatch } from "hooks/useAppDispatch"; +import { useAppSelector } from "hooks/useAppSelector"; +import React, { useEffect } from "react"; +import { Link } from "react-router-dom"; +import { fetchLatestUpdate } from "redux/neurojson/neurojson.action"; +import { DatasetChange } from "redux/neurojson/types/neurojson.interface"; +import { RootState } from "redux/store"; +import RoutesEnum from "types/routes.enum"; + +const MAX_DATASETS = 5; + +// Palette for this component (integrated with the dark navy/purple hero). +// Green-family accents only — no red styling. +const C = { + primary: "#F4F4FF", // near-white + muted: "#a0a5c2", // primary.light + link: "#a0a5c2", // primary.light (lightGreen on hover) + lightGreen: "#16FDE2", +}; + +type ChangeType = DatasetChange["changeType"]; + +// Status badges share one consistent green; the change type is conveyed by the +// badge text (Added / Updated / Deleted), not by color. +const BADGE_TEXT = C.lightGreen; +const BADGE_BG = "rgba(22, 253, 226, 0.10)"; +const changeSign: Record = { + added: "+", + updated: "~", + deleted: "−", +}; +const badgeLabel: Record = { + added: "Added", + updated: "Updated", + deleted: "Deleted", +}; + +const formatDate = (iso: string | null): string => { + if (!iso) return ""; + const d = new Date(iso); + if (isNaN(d.getTime())) return ""; + return d.toLocaleDateString(undefined, { + year: "numeric", + month: "short", + day: "numeric", + }); +}; + +const signed = (n: number): string => + `${n > 0 ? "+" : n < 0 ? "−" : ""}${Math.abs(n)}`; + +const signedBytes = (n: number): string => { + const sign = n > 0 ? "+" : n < 0 ? "−" : ""; + let v = Math.abs(n); + const units = ["B", "KB", "MB", "GB", "TB", "PB"]; + let u = 0; + while (v >= 1024 && u < units.length - 1) { + v /= 1024; + u += 1; + } + return `${sign}${v.toFixed(v < 10 && u > 0 ? 1 : 0)} ${units[u]}`; +}; + +// A single metric: medium-weight number + slightly smaller label. +const Metric: React.FC<{ + value: string; + label: string; + numberColor: string; + labelColor: string; +}> = ({ value, label, numberColor, labelColor }) => ( + + + {value} + + + {label} + + +); + +const LatestUpdateBoard: React.FC = () => { + const dispatch = useAppDispatch(); + const latest = useAppSelector( + (state: RootState) => state.neurojson.latestUpdate + ); + + useEffect(() => { + dispatch(fetchLatestUpdate()); + }, [dispatch]); + + // Nothing has ever changed → render nothing (keeps the hero uncluttered). + if (!latest || latest.historyId === null || latest.datasets.length === 0) { + return null; + } + + const { changes, deltas, datasets } = latest; + const shown = datasets.slice(0, MAX_DATASETS); + const remaining = datasets.length - shown.length; + + // Dataset-change metrics (semantic color); subject/file/size deltas (neutral). + const datasetMetrics = (["added", "updated", "deleted"] as ChangeType[]) + .filter((t) => changes[t] > 0) + .map((t) => { + const n = changes[t]; + return { + key: t, + value: `${changeSign[t]}${n}`, + label: n === 1 ? "Dataset" : "Datasets", + }; + }); + + return ( + + {/* Line 1 — "Latest update · " */} + + + Latest update + + + {" · "} + {formatDate(latest.updatedAt)} + + + + {/* Line 2 — metrics, evenly spaced, lightweight (no bordered chips) */} + + {datasetMetrics.map((m) => ( + + ))} + {deltas && deltas.subjects !== 0 && ( + + )} + {deltas && deltas.files !== 0 && ( + + )} + {deltas && deltas.sizeBytes !== 0 && ( + + )} + + + {/* Line 3+ — affected datasets, left-aligned; badge next to the name */} + + {shown.map((d, i) => ( + + + {d.dbname}/{d.dsname} + + + {badgeLabel[d.changeType]} + + + ))} + {remaining > 0 && ( + + +{remaining} more + + )} + + + ); +}; + +export default LatestUpdateBoard; diff --git a/src/components/StatisticsBanner.tsx b/src/components/StatisticsBanner.tsx index 99e148a..6d96324 100644 --- a/src/components/StatisticsBanner.tsx +++ b/src/components/StatisticsBanner.tsx @@ -9,7 +9,6 @@ import { useAppDispatch } from "hooks/useAppDispatch"; import { useAppSelector } from "hooks/useAppSelector"; import React, { useEffect } from "react"; import { fetchDbStats } from "redux/neurojson/neurojson.action"; -import { DbStatsItem } from "redux/neurojson/types/neurojson.interface"; import { RootState } from "redux/store"; const iconStyle = { @@ -42,31 +41,19 @@ const labelTextStyle = { }, }; -// function for calculate links and size -const calculateLinksAndSize = (dbStats: DbStatsItem[] | null) => { - if (!dbStats) return { totalLinks: 0, totalSizeTB: "0.00" }; - - const filtered = dbStats.filter( - (item) => item.view !== "dbinfo" && item.view !== "subjects" - ); - - const totalLinks = filtered.reduce((acc, item) => acc + item.num, 0); - const totalSizeBytes = filtered.reduce((acc, item) => acc + item.size, 0); - const totalSizeTB = Math.floor(totalSizeBytes / 1024 ** 4); - return { totalLinks, totalSizeTB }; -}; - const StatisticsBanner: React.FC = () => { const dispatch = useAppDispatch(); + // dbStats is now the flat snapshot { datasets, subjects, files, sizeBytes, + // lastSynced } from the latest successful stats_history row. const dbstats = useAppSelector((state: RootState) => state.neurojson.dbStats); const registry = useAppSelector( (state: RootState) => state.neurojson.registry ); const databaseCount = registry?.length ?? "-"; - const datasetStat = dbstats?.find((item) => item.view === "dbinfo"); - const subjectStat = dbstats?.find((item) => item.view === "subjects"); - const { totalLinks, totalSizeTB } = calculateLinksAndSize(dbstats); + const totalSizeTB = dbstats + ? Math.floor(dbstats.sizeBytes / 1024 ** 4) + : "-"; // format numbers with commas const formatNumber = (num: number | undefined) => @@ -127,19 +114,19 @@ const StatisticsBanner: React.FC = () => { {/* Datasets */} } - number={formatNumber(datasetStat?.num)} + number={formatNumber(dbstats?.datasets)} label="Datasets" /> {/* Subjects */} } - number={formatNumber(subjectStat?.num)} + number={formatNumber(dbstats?.subjects)} label="Subjects" /> {/* Links */} } - number={formatNumber(totalLinks)} + number={formatNumber(dbstats?.files)} label="Links" /> {/* Size */} diff --git a/src/pages/SearchPage.tsx b/src/pages/SearchPage.tsx index df49761..afafa7e 100644 --- a/src/pages/SearchPage.tsx +++ b/src/pages/SearchPage.tsx @@ -648,17 +648,27 @@ const SearchPage: React.FC = () => { // check if has database/dataset matches // const hasDbMatches = !!keywordInput && registryMatches.length > 0; const hasDbMatches = registryMatches.length > 0; + // Display/pagination use the local `results` state (kept — also gives TS + // array-narrowing inside {hasDatasetMatches && ...} render blocks). const hasDatasetMatches = Array.isArray(results) && results.length > 0; - // when backend find nothing - const backendEmpty = - !Array.isArray(results) && (results as any)?.msg === "empty output"; + + // The empty-state WARNING is based on the Redux `searchResults` instead, + // which updates in the SAME render as `loading` (both set in the fulfilled + // reducer). The local `results` is set a render later (in the dispatch + // .then()), so using it for the warning made it FLASH on slow searches: + // loading turns false while `results` is still stale/empty. + const storeHasMatches = + Array.isArray(searchResults) && searchResults.length > 0; + const storeEmpty = + !Array.isArray(searchResults) && + (searchResults as any)?.msg === "empty output"; // show red message only if nothing matched at all const showNoResults = hasSearched && !loading && // !hasDbMatches && - (!hasDatasetMatches || backendEmpty); + (!storeHasMatches || storeEmpty); // Tailored empty-state message: when the user combined a file_type filter // with any subject-level filter and got nothing back, it's almost certainly diff --git a/src/redux/neurojson/neurojson.action.ts b/src/redux/neurojson/neurojson.action.ts index f8f08c2..77e7ab3 100644 --- a/src/redux/neurojson/neurojson.action.ts +++ b/src/redux/neurojson/neurojson.action.ts @@ -97,6 +97,18 @@ export const fetchDbStats = createAsyncThunk( } ); +export const fetchLatestUpdate = createAsyncThunk( + "neurojson/fetchLatestUpdate", + async (_, { rejectWithValue }) => { + try { + const response = await NeurojsonService.getLatestUpdate(); + return response; + } catch (error: any) { + return rejectWithValue("Failed to fetch latest update"); + } + } +); + export const fetchMetadataSearchResults = createAsyncThunk( "neurojson/fetchMetadataSearchResults", async (formData: any, { rejectWithValue }) => { diff --git a/src/redux/neurojson/neurojson.slice.ts b/src/redux/neurojson/neurojson.slice.ts index cfafc6a..3c948c6 100644 --- a/src/redux/neurojson/neurojson.slice.ts +++ b/src/redux/neurojson/neurojson.slice.ts @@ -5,6 +5,7 @@ import { fetchDbInfo, fetchDocumentDetails, fetchDbStats, + fetchLatestUpdate, fetchMetadataSearchResults, fetchDbInfoByDatasetId, fetchFileTypes, @@ -25,6 +26,7 @@ const initialState: INeuroJsonState = { registry: null, dbInfo: null, // add dbInfo in neurojson.interface.ts dbStats: null, + latestUpdate: null, searchResults: null, datasetViewInfo: null, fileTypes: null, @@ -142,6 +144,12 @@ const neurojsonSlice = createSlice({ state.loading = false; state.error = action.payload as string; }) + .addCase( + fetchLatestUpdate.fulfilled, + (state, action: PayloadAction) => { + state.latestUpdate = action.payload; + } + ) .addCase(fetchMetadataSearchResults.pending, (state) => { state.loading = true; state.error = null; diff --git a/src/redux/neurojson/types/neurojson.interface.ts b/src/redux/neurojson/types/neurojson.interface.ts index 01c8273..d461526 100644 --- a/src/redux/neurojson/types/neurojson.interface.ts +++ b/src/redux/neurojson/types/neurojson.interface.ts @@ -10,7 +10,8 @@ export interface INeuroJsonState { limit: number; hasMore: boolean; dbInfo: DBParticulars | null; // add dbInfo type - dbStats: DbStatsItem[] | null; // for dbStats on landing page + dbStats: DbStats | null; // landing-page stats snapshot (from stats_history) + latestUpdate: LatestUpdate | null; // most recent sync run with data changes searchResults: any[] | { status: string; msg: string } | null; datasetViewInfo: any | null; fileTypes: string[] | null; @@ -93,3 +94,30 @@ export interface DbStatsItem { num: number; size: number; } + +// Landing-page stats snapshot returned by GET /dbs/stats (latest successful +// stats_history row). +export interface DbStats { + datasets: number; + subjects: number; + files: number; + sizeBytes: number; + lastSynced: string | null; +} + +// Response of GET /dbs/updates/latest — the most recent sync run that actually +// changed datasets (distinct from "last synced"). +export interface DatasetChange { + dbname: string; + dsname: string; + changeType: "added" | "updated" | "deleted"; +} +export interface LatestUpdate { + historyId: number | null; + updatedAt: string | null; + changes: { added: number; updated: number; deleted: number }; + // Net subject/file/size deltas vs the previous snapshot; null when there is + // no previous successful snapshot to compare against. + deltas: { subjects: number; files: number; sizeBytes: number } | null; + datasets: DatasetChange[]; +} diff --git a/src/services/neurojson.service.ts b/src/services/neurojson.service.ts index ed1f70d..bad872e 100644 --- a/src/services/neurojson.service.ts +++ b/src/services/neurojson.service.ts @@ -90,6 +90,12 @@ export const NeurojsonService = { return response.data; }, + // GET /api/v1/dbs/updates/latest → getLatestUpdate + getLatestUpdate: async () => { + const response = await api.get(`/dbs/updates/latest`); + return response.data; + }, + // getMetadataSearchResults: async (formData: any): Promise => { // const map: Record = { // keyword: "keyword",