fix(bigmind): apply 4 health-check fixes — BUG-1/2/3 + PERF-1

BUG-1: fix test_server_tools.py assert "ALWAYS" → "Always" (case mismatch) BUG-2: export_memory() now includes hypotheses, upgrade_requests, token_saves, people tables; renamed bigmind_version → bigmind_schema_version (int) BUG-3: auto_close.py replaced CURRENT_TIMESTAMP (SQLite) with Python datetime.now(timezone.utc).isoformat() for consistent UTC timestamps PERF-1: context_builder.py caps get_facts() at _MAX_CONTEXT_FACTS=50 with overflow hint to prevent unbounded context growth All 297 tests passing. Upgrade requests #6-9 resolved. Health report: plans/BIGMIND_HEALTH_REPORT_2026-04-04.md
2026-04-04 09:49:13 +02:00
parent 155d56e8e8
commit 42ffc85f0b
7 changed files with 265 additions and 11 deletions
@@ -3,6 +3,7 @@ import logging
 from datetime import datetime, timezone, timedelta
 from bigmind.db import db

+
 logger = logging.getLogger("BigMindAutoClose")

 STALE_THRESHOLD_HOURS = 24
@@ -17,6 +18,7 @@ def auto_close_stale_sessions(user_id: str) -> int:
        datetime.now(timezone.utc) - timedelta(hours=STALE_THRESHOLD_HOURS)
    ).isoformat()

+    now = datetime.now(timezone.utc).isoformat()
    with db() as conn:
        stale = conn.execute(
            """SELECT id, started_at FROM sessions
@@ -27,11 +29,11 @@ def auto_close_stale_sessions(user_id: str) -> int:
        for session in stale:
            conn.execute(
                """UPDATE sessions
-                   SET ended_at=CURRENT_TIMESTAMP,
+                   SET ended_at=?,
                       one_liner='[auto-closed — session exceeded 24h]',
                       outcome='Session automatically closed after exceeding 24h without a proper close call.'
                   WHERE id=?""",
-                (session["id"],),
+                (now, session["id"]),
            )
            logger.info(
                "Auto-closed stale session %s (started %s)",
@@ -57,15 +59,16 @@ def close_orphaned_sessions(user_id: str, keep_session_id: str) -> list[str]:
            (user_id, keep_session_id),
        ).fetchall()

+        now = datetime.now(timezone.utc).isoformat()
        closed_ids = []
        for session in orphans:
            conn.execute(
                """UPDATE sessions
-                   SET ended_at=CURRENT_TIMESTAMP,
+                   SET ended_at=?,
                       one_liner='[orphaned — closed by memory_close_stale_sessions]',
                       outcome='Session was open but never properly closed (IDE crash or forgotten). Cleaned up manually.'
                   WHERE id=?""",
-                (session["id"],),
+                (now, session["id"]),
            )
            closed_ids.append(session["id"])
            logger.info(
@@ -21,6 +21,12 @@ def _format_date(iso_str: Optional[str]) -> str:
        return iso_str[:10]


+# Maximum facts loaded into context on session start.
+# Keeps context lean as the facts table grows — older/excess facts remain
+# searchable via memory_search_facts() but won't bloat every session startup.
+_MAX_CONTEXT_FACTS = 50
+
+
 def build_context(user_id: str, n_sessions: int = 10) -> str:
    """
    Assemble the full bootstrapped context markdown for injection at session start.
@@ -55,13 +61,19 @@ def build_context(user_id: str, n_sessions: int = 10) -> str:
    lines.append("")

    # ── FACTS: Atomic personal facts ─────────────────────────────────────────
-    facts = memory_store.get_facts(user_id)
+    all_facts = memory_store.get_facts(user_id)
+    facts = all_facts[:_MAX_CONTEXT_FACTS]
+    overflow = len(all_facts) - len(facts)
    if facts:
        lines.append("### 🗂️ Stored facts")
        for f in facts:
            cat = f.get("category", "")
            fact = f.get("fact", "")
            lines.append(f"- **[{cat}]** {fact}")
+        if overflow > 0:
+            lines.append(
+                f"- *… {overflow} more facts available via `memory_search_facts()`*"
+            )
        lines.append("")

    # ── TIER 1: Recent Sessions ───────────────────────────────────────────────
@@ -721,9 +721,14 @@ def health_check(user_id: str, stale_days: int = 30) -> dict:
 # ── EXPORT ───────────────────────────────────────────────────────────────────────

 def export_memory(user_id: str, output_path: str = None) -> dict:
-    """Export all memory for a user to a portable JSON file."""
+    """Export all memory for a user to a portable JSON file.
+
+    Exports ALL tables: facts, sessions (with Tier-2), conversation chunks,
+    people/contacts, hypotheses, token saves, and upgrade requests.
+    """
    import json
    from pathlib import Path
+    from bigmind.db import SCHEMA_VERSION

    if not output_path:
        date_str = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
@@ -768,18 +773,57 @@ def export_memory(user_id: str, output_path: str = None) -> dict:
            ).fetchall()
        ]

+        # ── v3+ tables ───────────────────────────────────────────────────────
+        hypotheses = [
+            dict(r) for r in conn.execute(
+                "SELECT * FROM hypotheses WHERE user_id=? ORDER BY created_at",
+                (user_id,),
+            ).fetchall()
+        ]
+
+        upgrade_requests = [
+            dict(r) for r in conn.execute(
+                "SELECT * FROM upgrade_requests WHERE user_id=? ORDER BY created_at",
+                (user_id,),
+            ).fetchall()
+        ]
+
+        # ── v6+ tables ───────────────────────────────────────────────────────
+        token_saves = [
+            dict(r) for r in conn.execute(
+                "SELECT * FROM token_saves WHERE user_id=? ORDER BY created_at",
+                (user_id,),
+            ).fetchall()
+        ]
+
+        # ── v7+ tables ───────────────────────────────────────────────────────
+        people = [
+            dict(r) for r in conn.execute(
+                "SELECT * FROM people WHERE user_id=? ORDER BY created_at",
+                (user_id,),
+            ).fetchall()
+        ]
+
    export_data = {
        "export_date": datetime.now(timezone.utc).isoformat(),
-        "bigmind_version": "1.0",
+        "bigmind_schema_version": SCHEMA_VERSION,
        "user": user_info,
        "identity_profile": profile,
        "facts": facts,
        "sessions": sessions,
        "conversation_chunks": chunks,
+        "hypotheses": hypotheses,
+        "upgrade_requests": upgrade_requests,
+        "token_saves": token_saves,
+        "people": people,
        "stats": {
            "facts_count": len(facts),
            "sessions_count": len(sessions),
            "chunks_count": len(chunks),
+            "hypotheses_count": len(hypotheses),
+            "upgrade_requests_count": len(upgrade_requests),
+            "token_saves_count": len(token_saves),
+            "people_count": len(people),
        },
    }

@@ -791,6 +835,8 @@ def export_memory(user_id: str, output_path: str = None) -> dict:
        "facts_count": len(facts),
        "sessions_count": len(sessions),
        "chunks_count": len(chunks),
+        "hypotheses_count": len(hypotheses),
+        "people_count": len(people),
        "file_size_kb": round(output.stat().st_size / 1024, 1),
    }

@@ -771,6 +771,8 @@ def memory_export(output_path: str = None) -> str:
        f"| **Facts** | {result['facts_count']} |\n"
        f"| **Sessions** | {result['sessions_count']} |\n"
        f"| **Chunks (Tier 3)** | {result['chunks_count']} |\n"
+        f"| **Hypotheses** | {result['hypotheses_count']} |\n"
+        f"| **People** | {result['people_count']} |\n"
        f"| **File size** | {result['file_size_kb']} KB |"
    )

@@ -571,8 +571,9 @@ class TestExportMemory:
        memory_store.export_memory(user["id"], out)
        data = json.loads(Path(out).read_text())
        for key in (
-            "export_date", "bigmind_version", "user", "identity_profile",
-            "facts", "sessions", "conversation_chunks", "stats",
+            "export_date", "bigmind_schema_version", "user", "identity_profile",
+            "facts", "sessions", "conversation_chunks", "hypotheses",
+            "upgrade_requests", "token_saves", "people", "stats",
        ):
            assert key in data

@@ -541,7 +541,7 @@ class TestMemoryGetInstructions:

    def test_contains_mandatory_language(self, temp_db):
        result = memory_get_instructions()
-        assert "ALWAYS" in result
+        assert "Always" in result


 # ── memory_health_check ────────────────────────────────────────────────────────
@@ -674,7 +674,8 @@ class TestMemoryExport:
        memory_export(output_path=out)
        data = json.loads(Path(out).read_text())
        assert "export_date" in data
-        assert data["bigmind_version"] == "1.0"
+        assert "bigmind_schema_version" in data
+        assert isinstance(data["bigmind_schema_version"], int)


 # ── memory_deprecate_fact ──────────────────────────────────────────────────────
@@ -0,0 +1,189 @@
+# 🧠 BigMind Health Report
+**Date:** 2026-04-04 | **Machine:** Fedora Homelab (AMD Ryzen 5900X) | **Schema:** v7 | **Analyst:** Lumen
+
+---
+
+## 🟢 Executive Summary
+
+BigMind is **healthy and operational** on the homelab machine. The brain is alive, the DB is clean, and the test suite is 99.7% green. Three bugs and one scalability risk were found during the code scan — all logged as upgrade requests. No data corruption, no orphaned sessions, FTS index is in sync.
+
+**Verdict: 🟢 HEALTHY** — 1 real bug to fix, 3 improvements to queue.
+
+---
+
+## 📊 Memory Statistics
+
+| Metric | Value | Notes |
+|--------|-------|-------|
+| Sessions | **98** | ~5 per day average since March 30 |
+| Facts | **97** | Growing fast — see Perf-1 |
+| Tier-3 Chunks | **41** | Lean and targeted |
+| Global Knowledge | **0** | Phase 3 (Company Brain) not started |
+| Database size | **744 KB** | Very healthy |
+| DB location | `/home/pplate/.mcp/bigmind/memory.db` | WAL mode active |
+| Schema version | **v7** | People/contacts directory live |
+| Python version | **3.12.13** | ✅ correct |
+
+---
+
+## 🩺 Health Check Results
+
+| Check | Status | Detail |
+|-------|--------|--------|
+| FTS index integrity | ✅ IN SYNC | 41 chunks / 41 index rows — no drift |
+| Open sessions | 🟡 1 | Current health check session only — clean |
+| Orphaned sessions | ✅ NONE | `memory_close_stale_sessions` returned clean |
+| Sessions without Tier-2 | ⚠️ 22 | 22 closed sessions have no narrative summary |
+| Stale facts (>30d) | ✅ NONE | All 97 facts updated recently |
+| Low-confidence facts (<0.8) | ✅ NONE | All facts at full confidence |
+
+> **On the 22 sessions without Tier-2:** These are mostly the orphaned sessions that were force-closed by `memory_close_stale_sessions` (IDE crashes). They never had a chance to store a summary. Not a bug — expected behaviour. Worth vacuuming if they clutter the index.
+
+---
+
+## 🧪 Test Suite Results
+
+**297 tests collected | 296 ✅ PASSED | 1 ❌ FAILED**
+
+```
+platform linux -- Python 3.12.13, pytest-9.0.2
+rootdir: /home/pplate/pi_mcps/mcp/bigmind
+Duration: 2.36s
+```
+
+### ❌ Failing Test
+
+```
+tests/test_server_tools.py::TestMemoryGetInstructions::test_contains_mandatory_language
+AssertionError: assert 'ALWAYS' in '...Rule 1: Session Start Ritual (Always First Action)...'
+```
+
+**Root cause:** The `BIGMIND_INSTRUCTIONS` constant in [`src/server.py`](mcp/bigmind/src/server.py:38) was rewritten to use title-case language ("Always First Action") but the test in [`tests/test_server_tools.py`](mcp/bigmind/tests/test_server_tools.py:544) still asserts that the uppercase word `"ALWAYS"` is present. One of them needs to change — the test expectation is the right one to update since "Always" is clearer prose anyway. **Logged as Upgrade Request #6 (HIGH).**
+
+---
+
+## 🐛 Bugs & Issues Found
+
+### BUG-1 — Test/Code Mismatch (HIGH) `upgrade #6`
+- **File:** [`src/server.py:38`](mcp/bigmind/src/server.py:38) + [`tests/test_server_tools.py:544`](mcp/bigmind/tests/test_server_tools.py:544)
+- **Issue:** `BIGMIND_INSTRUCTIONS` string was rewritten without updating the test. Test asserts `"ALWAYS"` (uppercase), instructions say `"Always"` (title-case).
+- **Impact:** 1 test failing in CI. Easy fix: update the test to assert `"Always"` or add `"ALWAYS"` to the instructions.
+- **Fix:** `assert "Always" in result` — one-line change.
+
+---
+
+### BUG-2 — Data Loss in Export (HIGH) `upgrade #7`
+- **File:** [`bigmind/memory_store.py:723`](mcp/bigmind/bigmind/memory_store.py:723) — `export_memory()`
+- **Issue:** `memory_export` was written before the `people` (v7) and `hypotheses` (v3) tables existed. It exports **facts, sessions, and chunks only**. A full machine migration using the export would **silently lose:**
+  - The entire contacts directory (Elias, Klaus, etc.)
+  - The entire thought journal (all 3 open hypotheses + resolved ones)
+  - Token saves history
+  - Upgrade requests
+- **Impact:** 🔴 Data loss on machine migration. The export is marketed as a full backup but it's not.
+- **Fix:** Add `hypotheses`, `people`, `token_saves`, and `upgrade_requests` queries to the export dict. Also fix `bigmind_version: "1.0"` hardcode — should use `SCHEMA_VERSION`.
+
+---
+
+### BUG-3 — Timestamp Format Inconsistency (MEDIUM) `upgrade #8`
+- **File:** [`bigmind/auto_close.py:30`](mcp/bigmind/bigmind/auto_close.py:30) and [`auto_close.py:68`](mcp/bigmind/bigmind/auto_close.py:68)
+- **Issue:** `auto_close_stale_sessions()` and `close_orphaned_sessions()` write `ended_at` using SQLite's `CURRENT_TIMESTAMP` keyword:
+  ```sql
+  SET ended_at=CURRENT_TIMESTAMP
+  ```
+  This produces: `"2026-04-04 09:36:00"` (no T, no timezone)
+  
+  The **entire rest of the codebase** uses:
+  ```python
+  datetime.now(timezone.utc).isoformat()
+  ```
+  Which produces: `"2026-04-04T09:36:00+00:00"` (ISO 8601 with T and timezone)
+  
+- **Impact:** `datetime.fromisoformat()` in `get_active_sessions()` handles the mixed formats via `.replace("Z", "+00:00")` but does NOT handle the no-T format from SQLite `CURRENT_TIMESTAMP`. Sessions auto-closed by `auto_close_stale_sessions` may have `idle_minutes = None` in the profile page Live Sessions panel.
+- **Fix:** Replace `CURRENT_TIMESTAMP` in `auto_close.py` with a Python-level timestamp passed as a parameter:
+  ```python
+  now = datetime.now(timezone.utc).isoformat()
+  conn.execute("UPDATE sessions SET ended_at=? ... WHERE id=?", (now, session["id"]))
+  ```
+
+---
+
+## ⚡ Performance / Scalability Concern
+
+### PERF-1 — Unbounded Fact Loading in Context Builder (MEDIUM) `upgrade #9`
+- **File:** [`bigmind/context_builder.py:58`](mcp/bigmind/bigmind/context_builder.py:58)
+- **Issue:** `build_context()` calls `get_facts(user_id)` with **no limit**. Currently 97 facts are loaded on every single `memory_start_session()` call.
+- **Projection:**
+  | Facts count | Estimated chars | Estimated tokens |
+  |-------------|-----------------|------------------|
+  | 97 (now) | ~12,000 | ~3,000 |
+  | 200 (3 months) | ~25,000 | ~6,250 |
+  | 500 (1 year) | ~62,000 | ~15,500 |
+- **Impact:** Context window bloat, increased API cost per session, slower startup. The AI also can't meaningfully use 500 facts in a single context — search is more effective for recall.
+- **Fix:** Cap at 50 most recently updated facts with a `"... N more facts available via memory_search_facts()"` footer. Optionally sort by `updated_at DESC` to surface the freshest knowledge first.
+
+---
+
+## 📐 Code Quality Observations (Non-Bug)
+
+| Location | Issue | Severity |
+|----------|-------|----------|
+| [`bigmind/db.py:357`](mcp/bigmind/bigmind/db.py:357) | Migration functions defined out of numeric order: `v5→v6`, then `v3→v4`, then `_migrate_v6_to_v7`. Works fine in Python (runtime resolution) but confusing for maintainers. | Style |
+| [`bigmind/web.py:66`](mcp/bigmind/bigmind/web.py:66) | Bare `except Exception: pass` in `api_search()` for facts, chunks, and sessions searches. Silently swallows errors — makes it impossible to diagnose search bugs through the profile page. | Low |
+| [`src/server.py:911`](mcp/bigmind/src/server.py:911) | Double blank line before `memory_open_profile` tool — minor style inconsistency. | Cosmetic |
+| [`bigmind/memory_store.py:239`](mcp/bigmind/bigmind/memory_store.py:239) | `update_fields: list = [description, files_json, now]` in `announce_focus()` is built but never actually used (the code branches into two separate `conn.execute` calls). Dead code. | Minor |
+
+---
+
+## 💭 Open Hypotheses Review
+
+3 hypotheses remain open from previous sessions:
+
+| # | Age | Confidence | Summary | Action |
+|---|-----|------------|---------|--------|
+| #38 | ~2d | 92% | NPE at `Center.java:425` from empty `errors` variable | This is an ADP/Paisy hypothesis — still valid, likely confirmed but never closed. Should be resolved. |
+| #14 | ~4d | 65% | Elias will hit Windows setup issues with BigMind | Still open/unknown — no update since Elias received the repo. |
+| #5 | ~4d | 75% | Elias's AI will have a different name/personality | Still open — awaiting first session report from Elias. |
+
+> Hypothesis #38 is stale — it's about a Paisy bug investigated days ago. If the fix was merged, it should be resolved as `confirmed`.
+
+---
+
+## 🔧 Upgrade Requests Summary
+
+| ID | Priority | Status | Title |
+|----|----------|--------|-------|
+| #2 | MEDIUM | ⏳ OPEN | Session Health Panel on Profile Page |
+| #6 | HIGH | ⏳ OPEN | Fix test failure: "ALWAYS" vs "Always" in instructions |
+| #7 | HIGH | ⏳ OPEN | `export_memory()` missing `people` + `hypotheses` tables |
+| #8 | MEDIUM | ⏳ OPEN | `auto_close.py` timestamp format inconsistency |
+| #9 | MEDIUM | ⏳ OPEN | Unbounded fact loading in `context_builder.py` |
+
+(#1, #3, #4, #5 are already resolved ✅)
+
+---
+
+## 🗓️ Recommended Fix Order
+
+1. **[30 min] BUG-1** — Fix the failing test. One line. Do it now before it becomes noise.
+2. **[2h] BUG-2** — Fix `export_memory()` to include all tables. Critical for machine migration safety.
+3. **[1h] BUG-3** — Fix `auto_close.py` timestamp format. Simple Python replace, no schema change.
+4. **[1d] PERF-1** — Design and implement smart fact pagination in `context_builder.py`. Needs thought about ranking strategy.
+5. **[1d] Upgrade #2** — Session Health Panel on profile page. Quality of life.
+
+---
+
+## ✅ What's Working Well
+
+- **WAL mode** is active and correctly configured — concurrent IDE access is safe
+- **FTS5 index** is perfectly in sync with the chunks table (41/41)
+- **Schema migrations** run correctly through v1→v7 on fresh DB
+- **Conflict detection** in `announce_focus()` is atomic (uses `BEGIN IMMEDIATE`) — race-condition safe
+- **Port conflict handling** in `web.py` is well thought out — second IDE gracefully defers to first
+- **People/contacts directory** (v7) is fully functional with FTS search
+- **Test coverage** is excellent: 297 tests across 8 test files covering all layers
+- **Token efficiency tracker** is working and recording saves
+- **`_fts_safe_query()`** helper correctly handles FTS5 reserved words and AND-match semantics
+
+---
+
+*Report generated by Lumen | BigMind session `e04fc9c9` | 2026-04-04 09:40 UTC+2*