fix: visualization and dashboard scalability for large datasets

api/src/routes/visualizations.rs

@@ -15,6 +15,8 @@ use crate::AppState;
 #[derive(Deserialize)]
 pub struct ListParams {
     pub project_id: Option<Uuid>,
+    pub page: Option<i64>,
+    pub per_page: Option<i64>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
@@ -34,6 +36,21 @@ pub struct Visualization {
     pub updated_at: Option<DateTime<Utc>>,
 }
 
+/// Lightweight struct for list endpoints — excludes heavy rendered_output and code blobs.
+#[derive(Debug, Clone, Serialize, Deserialize, FromRow)]
+pub struct VisualizationSummary {
+    pub id: Uuid,
+    pub project_id: Option<Uuid>,
+    pub name: String,
+    pub description: Option<String>,
+    pub backend: String,
+    pub output_type: String,
+    pub refresh_interval: Option<i32>,
+    pub published: bool,
+    pub created_at: Option<DateTime<Utc>>,
+    pub updated_at: Option<DateTime<Utc>>,
+}
+
 #[derive(Deserialize)]
 pub struct CreateVisualization {
     pub project_id: Option<Uuid>,
@@ -58,34 +75,74 @@ pub struct UpdateVisualization {
     pub rendered_output: Option<String>,
 }
 
+/// Paginated response wrapper.
+#[derive(Serialize)]
+pub struct PaginatedResponse<T: Serialize> {
+    pub items: Vec<T>,
+    pub total: i64,
+    pub page: i64,
+    pub per_page: i64,
+}
+
 pub async fn list_all(
     State(state): State<AppState>,
     AuthUser(_claims): AuthUser,
     Query(params): Query<ListParams>,
-) -> AppResult<Json<Vec<Visualization>>> {
-    let rows: Vec<Visualization> = if let Some(pid) = params.project_id {
-        sqlx::query_as(
-            "SELECT id, project_id, name, description, backend, output_type, code,
-                    config, rendered_output, refresh_interval, published,
-                    created_at, updated_at
+) -> AppResult<Json<serde_json::Value>> {
+    let page = params.page.unwrap_or(1).max(1);
+    let per_page = params.per_page.unwrap_or(24).clamp(1, 200);
+    let offset = (page - 1) * per_page;
+
+    let (rows, total): (Vec<VisualizationSummary>, i64) = if let Some(pid) = params.project_id {
+        let rows: Vec<VisualizationSummary> = sqlx::query_as(
+            "SELECT id, project_id, name, description, backend, output_type,
+                    refresh_interval, published, created_at, updated_at
              FROM visualizations WHERE project_id = $1
-             ORDER BY updated_at DESC"
+             ORDER BY updated_at DESC
+             LIMIT $2 OFFSET $3"
         )
         .bind(pid)
+        .bind(per_page)
+        .bind(offset)
         .fetch_all(&state.db)
-        .await?
+        .await?;
+
+        let (count,): (i64,) = sqlx::query_as(
+            "SELECT COUNT(*) FROM visualizations WHERE project_id = $1"
+        )
+        .bind(pid)
+        .fetch_one(&state.db)
+        .await?;
+
+        (rows, count)
     } else {
-        sqlx::query_as(
-            "SELECT id, project_id, name, description, backend, output_type, code,
-                    config, rendered_output, refresh_interval, published,
-                    created_at, updated_at
+        let rows: Vec<VisualizationSummary> = sqlx::query_as(
+            "SELECT id, project_id, name, description, backend, output_type,
+                    refresh_interval, published, created_at, updated_at
              FROM visualizations
-             ORDER BY updated_at DESC"
+             ORDER BY updated_at DESC
+             LIMIT $1 OFFSET $2"
         )
+        .bind(per_page)
+        .bind(offset)
         .fetch_all(&state.db)
-        .await?
+        .await?;
+
+        let (count,): (i64,) = sqlx::query_as(
+            "SELECT COUNT(*) FROM visualizations"
+        )
+        .fetch_one(&state.db)
+        .await?;
+
+        (rows, count)
     };
-    Ok(Json(rows))
+
+    Ok(Json(serde_json::json!({
+        "items": rows,
+        "total": total,
+        "page": page,
+        "per_page": per_page,
+    })))
 }
 
 pub async fn create(

web/src/app/dashboards/[id]/page.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { useState, useEffect, useCallback, useMemo, useRef } from "react";
+import { useState, useEffect, useCallback, useMemo, useRef, type ReactNode } from "react";
 import { useParams, useRouter } from "next/navigation";
 import { AppShell } from "@/components/layout/app-shell";
 import { AnimatedPage } from "@/components/shared/animated-page";
@@ -98,6 +98,42 @@ const backendColors: Record<string, string> = {
 
 const ROW_HEIGHT = 120;
 
+/**
+ * Lazy-renders children only when the panel is visible in the viewport.
+ * Uses IntersectionObserver with a generous rootMargin so panels render
+ * slightly before scrolling into view. When a panel scrolls out, the
+ * VizRenderer unmounts — freeing WebGL contexts and preventing the
+ * browser's ~8-16 concurrent WebGL context limit from being exceeded.
+ */
+function LazyVizPanel({ children }: { children: ReactNode }) {
+  const ref = useRef<HTMLDivElement>(null);
+  const [isVisible, setIsVisible] = useState(false);
+
+  useEffect(() => {
+    const el = ref.current;
+    if (!el) return;
+
+    const observer = new IntersectionObserver(
+      ([entry]) => setIsVisible(entry.isIntersecting),
+      { rootMargin: "200px" }
+    );
+    observer.observe(el);
+    return () => observer.disconnect();
+  }, []);
+
+  return (
+    <div ref={ref} className="h-full w-full">
+      {isVisible ? (
+        children
+      ) : (
+        <div className="flex items-center justify-center w-full h-full min-h-[80px]">
+          <BarChart3 className="h-6 w-6 text-muted-foreground/20 animate-pulse" />
+        </div>
+      )}
+    </div>
+  );
+}
+
 export default function DashboardDetailPage() {
   const params = useParams();
   const router = useRouter();
@@ -143,13 +179,13 @@ export default function DashboardDetailPage() {
     setError(null);
     Promise.all([
       api.get<Dashboard>(`/dashboards/${id}`),
-      api.get<VisualizationSummary[]>("/visualizations"),
+      api.get<{ items: VisualizationSummary[] }>("/visualizations?per_page=200"),
     ])
-      .then(([dash, vizs]) => {
+      .then(([dash, vizsResp]) => {
         setDashboard(dash);
         const items: DashboardLayoutItem[] = Array.isArray(dash.layout) ? dash.layout : [];
         setPanels(items);
-        setAllVisualizations(vizs);
+        setAllVisualizations(vizsResp.items);
 
         // Fetch full detail for each panel's visualization
         const uniqueIds = [...new Set(items.map((p) => p.visualization_id))];
@@ -485,18 +521,20 @@ export default function DashboardDetailPage() {
                         </div>
                       </div>
 
-                      {/* Visualization content */}
+                      {/* Visualization content — lazy-loaded to limit WebGL contexts */}
                       <div
                         className="flex-1 min-h-0 px-2 pb-2"
                         ref={(el) => {
                           if (el) panelRefs.current.set(index, el);
                         }}
                       >
                         <div className="h-full rounded-md overflow-hidden bg-black/10">
-                          <VizRenderer
-                            outputType={outputType}
-                            renderedOutput={renderedOutput}
-                          />
+                          <LazyVizPanel>
+                            <VizRenderer
+                              outputType={outputType}
+                              renderedOutput={renderedOutput}
+                            />
+                          </LazyVizPanel>
                         </div>
                       </div>
                     </GlassCard>

web/src/app/visualizations/[id]/page.tsx

@@ -319,10 +319,13 @@ export default function VisualizationDetailPage() {
   // Live preview state for interactive backends (plotly, vega-lite)
   const [previewOutput, setPreviewOutput] = useState<string | null>(null);
   const [previewType, setPreviewType] = useState<string>("svg");
+  // Track whether the user has edited code (vs initial load from DB)
+  const userEditedCode = useRef(false);
 
   const fetchViz = useCallback(() => {
     setLoading(true);
     setError(null);
+    userEditedCode.current = false;
     api
       .get<Visualization>(`/visualizations/${id}`)
       .then((v) => {
@@ -354,11 +357,11 @@ export default function VisualizationDetailPage() {
   }, [fetchViz]);
 
   // Live preview for JSON-based backends (plotly, altair/vega-lite)
+  // Only update preview when the user actively edits code, not on initial load
   useEffect(() => {
-    if (!viz) return;
+    if (!viz || !userEditedCode.current) return;
     const backend = viz.backend.toLowerCase();
     if (backend === "plotly" || backend === "altair") {
-      // For JSON-based backends, the code IS the spec
       try {
         JSON.parse(code);
         setPreviewOutput(code);
@@ -434,6 +437,7 @@ export default function VisualizationDetailPage() {
   };
 
   const handleCodeChange = (value: string | undefined) => {
+    userEditedCode.current = true;
     setCode(value || "");
     setHasChanges(true);
   };