my-project-public

repository

loading code, commits, and activity

repositories

loading repo index

my-project-public — gitlawb

#1	#!/usr/bin/env python3
#2	"""
#3	Mnemosyne Statistics Dashboard
#4	==============================
#5	Terminal-based health check for the local Mnemosyne BEAM memory system.
#6	Reads directly from SQLite — no external dependencies.
#7
#8	Usage:
#9	python3 mnemosyne-stats.py # Full dashboard + auto-snapshot
#10	python3 mnemosyne-stats.py --compact # Summary only
#11	python3 mnemosyne-stats.py --json # JSON output
#12	python3 mnemosyne-stats.py --save-snapshot # Save snapshot + show trends
#13	python3 mnemosyne-stats.py --trends # Show trend data only
#14	"""
#15
#16	import os
#17	import sqlite3, sys, json
#18	from pathlib import Path
#19	from datetime import datetime
#20
#21	DB_PATH = Path(
#22	os.environ.get("MNEMOSYNE_DATA_DIR")
#23	or Path.home() / ".hermes" / "mnemosyne" / "data"
#24	) / "mnemosyne.db"
#25	WIKI_PATH = Path.home() / "wiki"
#26	SNAPSHOT_DIR = Path.home() / ".hermes" / "mnemosyne" / "stats"
#27	W = 60
#28
#29	def q(db, sql, params=()):
#30	try: return db.execute(sql, params).fetchall()
#31	except: return []
#32
#33	def cnt(db, table):
#34	r = q(db, f"SELECT COUNT(*) FROM {table}")
#35	return r[0][0] if r else 0
#36
#37	def pct(a, b): return (a / b * 100) if b > 0 else 0
#38
#39	def collect():
#40	if not DB_PATH.exists():
#41	return {"error": f"DB not found at {DB_PATH}"}
#42
#43	db = sqlite3.connect(str(DB_PATH))
#44	s = {"db_size_mb": round(DB_PATH.stat().st_size / 1048576, 2)}
#45
#46	# Working Memory
#47	wm_total = cnt(db, "working_memory")
#48	wm = {"total": wm_total, "by_source": {}, "by_scope": {},
#49	"importance_dist": {}, "recall_dist": {},
#50	"global_count": 0, "never_recalled": 0, "noise_pct": 0}
#51
#52	if wm_total:
#53	for r in q(db, "SELECT source, COUNT(), AVG(importance), AVG(recall_count) FROM working_memory GROUP BY source ORDER BY COUNT() DESC"):
#54	wm["by_source"][r[0] or "?"] = {"count": r[1], "imp": round(r[2] or 0, 3), "recall": round(r[3] or 0, 1)}
#55	for r in q(db, "SELECT scope, COUNT(*), AVG(importance) FROM working_memory GROUP BY scope"):
#56	wm["by_scope"][r[0] or "?"] = {"count": r[1], "imp": round(r[2] or 0, 3)}
#57	for lo, hi in [(0,.2),(.2,.4),(.4,.6),(.6,.8),(.8,1.01)]:
#58	c = q(db, "SELECT COUNT(*) FROM working_memory WHERE importance>=? AND importance<?", (lo,hi))
#59	wm["importance_dist"][f"{lo:.1f}-{hi:.1f}"] = c[0][0]
#60	for lo, hi, lbl in [(0,1,"never"),(1,5,"low"),(5,10,"med"),(10,50,"high"),(50,99999,"vhigh")]:
#61	c = q(db, "SELECT COUNT(*) FROM working_memory WHERE recall_count>=? AND recall_count<?", (lo,hi))
#62	wm["recall_dist"][lbl] = c[0][0]
#63	g = q(db, "SELECT COUNT(*) FROM working_memory WHERE scope='global'")
#64	wm["global_count"] = g[0][0]
#65	nr = q(db, "SELECT COUNT(*) FROM working_memory WHERE recall_count=0")
#66	wm["never_recalled"] = nr[0][0]
#67	noise = q(db, "SELECT COUNT(*) FROM working_memory WHERE importance<0.3 AND recall_count=0")
#68	wm["noise_pct"] = round(noise[0][0] / wm_total * 100, 1)
#69	s["working_memory"] = wm
#70
#71	# Episodic
#72	ep_total = cnt(db, "episodic_memory")
#73	ep = {"total": ep_total, "avg_imp": 0, "recent": []}
#74	if ep_total:
#75	r = q(db, "SELECT AVG(importance) FROM episodic_memory")
#76	ep["avg_imp"] = round(r[0][0] or 0, 3)
#77	for r in q(db, "SELECT id, content, importance, created_at FROM episodic_memory ORDER BY created_at DESC LIMIT 5"):
#78	ep["recent"].append({"id": r[0], "preview": (r[1] or "")[:70], "imp": r[2], "date": r[3]})
#79	s["episodic"] = ep
#80
#81	# Triples
#82	tg = {"total": cnt(db, "triples"), "predicates": {}}
#83	for r in q(db, "SELECT predicate, COUNT() FROM triples GROUP BY predicate ORDER BY COUNT() DESC LIMIT 10"):
#84	tg["predicates"][r[0]] = r[1]
#85	s["triples"] = tg
#86
#87	# Consolidation
#88	con = {"events": cnt(db, "consolidation_log"), "items": 0, "recent": []}
#89	r = q(db, "SELECT SUM(items_consolidated) FROM consolidation_log")
#90	con["items"] = r[0][0] if r and r[0][0] else 0
#91	for r in q(db, "SELECT session_id, items_consolidated, summary_preview, created_at FROM consolidation_log ORDER BY created_at DESC LIMIT 5"):
#92	con["recent"].append({"session": r[0][:30], "items": r[1], "summary": r[2], "date": r[3]})
#93	s["consolidation"] = con
#94
#95	# Dreamer
#96	dr = {"runs": cnt(db, "dreamer_runs"), "proposals": 0, "conflicts": 0, "recent": []}
#97	r = q(db, "SELECT SUM(proposals_generated), SUM(conflicts_detected) FROM dreamer_runs")
#98	if r and r[0][0]: dr["proposals"] = r[0][0]; dr["conflicts"] = r[0][1] or 0
#99	for r in q(db, "SELECT started_at, status, memories_scanned, proposals_generated FROM dreamer_runs ORDER BY started_at DESC LIMIT 5"):
#100	dr["recent"].append({"start": r[0][:19], "status": r[1], "scanned": r[2], "proposals": r[3]})
#101	s["dreamer"] = dr
#102
#103	s["embeddings"] = cnt(db, "memory_embeddings")
#104	s["scratchpad"] = cnt(db, "scratchpad")
#105
#106	# Wiki
#107	wk = {"total": 0, "memories": 0, "concepts": 0}
#108	if WIKI_PATH.exists():
#109	wk["total"] = len(list(WIKI_PATH.rglob("*.md")))
#110	mdir = WIKI_PATH / "memories"
#111	cdir = WIKI_PATH / "concepts"
#112	wk["memories"] = len(list(mdir.glob("*.md"))) if mdir.exists() else 0
#113	wk["concepts"] = len(list(cdir.glob("*.md"))) if cdir.exists() else 0
#114	wk["promotion_pct"] = round(wk["memories"] / wm_total * 100, 1) if wm_total else 0
#115	s["wiki"] = wk
#116
#117	# Top recalled
#118	tr = []
#119	for r in q(db, "SELECT content, recall_count, importance FROM working_memory WHERE recall_count>5 ORDER BY recall_count DESC LIMIT 10"):
#120	tr.append({"text": (r[0] or "")[:50].replace("\n", " "), "recalls": r[1], "imp": r[2]})
#121	s["top_recalled"] = tr
#122
#123	# Memory age
#124	age = []
#125	for r in q(db, """
#126	SELECT CASE
#127	WHEN julianday('now')-julianday(created_at)<1 THEN 'Today'
#128	WHEN julianday('now')-julianday(created_at)<7 THEN 'This week'
#129	WHEN julianday('now')-julianday(created_at)<30 THEN 'This month'
#130	ELSE 'Older'
#131	END, COUNT(*), AVG(importance), AVG(recall_count)
#132	FROM working_memory GROUP BY 1
#133	"""):
#134	age.append({"group": r[0], "count": r[1], "imp": round(r[2] or 0, 3), "recall": round(r[3] or 0, 1)})
#135	s["memory_age"] = age
#136
#137	# Session activity
#138	act = []
#139	for r in q(db, "SELECT DATE(created_at), COUNT(*) FROM working_memory GROUP BY 1 ORDER BY 1 DESC LIMIT 7"):
#140	act.append({"date": r[0], "count": r[1]})
#141	s["activity"] = act
#142
#143	# Quality score
#144	q_score = sum([
#145	1 if wm["noise_pct"] < 30 else 0,
#146	1 if pct(wm["never_recalled"], wm_total) < 70 else 0,
#147	1 if wm["global_count"] > wm_total * 0.05 else 0,
#148	1 if s["embeddings"] > 0 or wm_total == 0 else 0,
#149	1 if con["events"] > 0 else 0,
#150	1 if dr["runs"] > 0 else 0,
#151	1 if wk["memories"] > 10 else 0,
#152	])
#153	s["quality_score"] = q_score
#154
#155	db.close()
#156	return s
#157
#158	def show(s, compact=False):
#159	if "error" in s: print(f"ERROR: {s['error']}"); return
#160
#161	score = s["quality_score"]
#162	grade = "A" if score >= 6 else "B" if score >= 5 else "C" if score >= 4 else "D" if score >= 3 else "F"
#163	bar = "█" * score + "░" * (7 - score)
#164
#165	print("=" * W)
#166	print(" MNEMOSYNE HEALTH DASHBOARD")
#167	print(f" {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
#168	print("=" * W)
#169	print(f"\n Health: [{bar}] {score}/7 ({grade}) DB: {s['db_size_mb']} MB")
#170
#171	wm = s["working_memory"]
#172	print(f"\n{'─'*W}")
#173	print(f" WORKING MEMORY: {wm['total']} items")
#174	print(f"{'─'*W}")
#175
#176	if not compact:
#177	print("\n By Source:")
#178	for src, d in sorted(wm["by_source"].items(), key=lambda x: -x[1]["count"]):
#179	p = pct(d["count"], wm["total"])
#180	print(f" {src:25s} {d['count']:4d} ({p:4.1f}%) imp={d['imp']:.2f} recall={d['recall']:.0f}")
#181	print("\n Importance Distribution:")
#182	for bucket, c in wm["importance_dist"].items():
#183	p = pct(c, wm["total"])
#184	print(f" {bucket}: {c:4d} {'█' * max(0, int(p / 2.5))}")
#185	print("\n Recall Distribution:")
#186	for bucket, c in wm["recall_dist"].items():
#187	p = pct(c, wm["total"])
#188	print(f" {bucket:8s}: {c:4d} {'█' * max(0, int(p / 2.5))}")
#189
#190	print(f"\n Global: {wm['global_count']} ({pct(wm['global_count'], wm['total']):.1f}%)"
#191	f" Never recalled: {wm['never_recalled']} ({pct(wm['never_recalled'], wm['total']):.1f}%)"
#192	f" Noise: {wm['noise_pct']}%")
#193
#194	ep = s["episodic"]
#195	print(f"\n{'─'*W}")
#196	print(f" EPISODIC: {ep['total']} items (avg imp: {ep['avg_imp']:.3f})")
#197	print(f"{'─'*W}")
#198	if ep["recent"] and not compact:
#199	for item in ep["recent"]:
#200	pv = item["preview"][:50] + "..." if len(item["preview"]) > 50 else item["preview"]
#201	print(f" [{item['imp']:.2f}] {pv}")
#202
#203	tg = s["triples"]
#204	print(f"\n{'─'*W}")
#205	print(f" KNOWLEDGE GRAPH: {tg['total']} triples")
#206	print(f"{'─'*W}")
#207	if tg["predicates"] and not compact:
#208	for pred, c in list(tg["predicates"].items())[:8]:
#209	print(f" {pred:30s} {c:3d}")
#210
#211	con = s["consolidation"]
#212	print(f"\n{'─'*W}")
#213	print(f" CONSOLIDATION: {con['events']} events, {con['items']} items")
#214	print(f"{'─'*W}")
#215	if con["recent"] and not compact:
#216	for ev in con["recent"][:3]:
#217	print(f" {ev['date']} — {ev['items']} items")
#218
#219	dr = s["dreamer"]
#220	print(f"\n{'─'*W}")
#221	print(f" DREAMER: {dr['runs']} runs, {dr['proposals']} proposals, {dr['conflicts']} conflicts")
#222	print(f"{'─'*W}")
#223	if dr["recent"] and not compact:
#224	for run in dr["recent"]:
#225	icon = "✓" if run["status"] == "success" else "✗"
#226	print(f" {icon} {run['start']} scanned={run['scanned']} proposals={run['proposals']}")
#227
#228	emb_status = "OK" if s["embeddings"] > 0 else "EMPTY"
#229	print(f"\n Embeddings: {s['embeddings']} ({emb_status})")
#230
#231	wk = s["wiki"]
#232	print(f"\n{'─'*W}")
#233	print(f" WIKI: {wk['total']} pages ({wk['memories']} memories, {wk['concepts']} concepts)")
#234	print(f" Promotion: {wk['promotion_pct']}% of working memory → wiki")
#235	print(f"{'─'*W}")
#236
#237	if not compact:
#238	if s["top_recalled"]:
#239	print(f"\n TOP RECALLED:")
#240	for item in s["top_recalled"]:
#241	print(f" {item['recalls']:4d}x [{item['imp']:.2f}] {item['text']}")
#242	if s["memory_age"]:
#243	print(f"\n MEMORY AGE:")
#244	for item in s["memory_age"]:
#245	print(f" {item['group']:12s}: {item['count']:4d} items imp={item['imp']:.3f} recall={item['recall']:.1f}")
#246	if s["activity"]:
#247	print(f"\n SESSION ACTIVITY:")
#248	mx = max(item["count"] for item in s["activity"]) if s["activity"] else 1
#249	for item in s["activity"]:
#250	bl = int(item["count"] / mx * 35) if mx > 0 else 0
#251	print(f" {item['date']}: {item['count']:4d} {'█' * bl}")
#252
#253	# Quality
#254	print(f"\n{'─'*W}")
#255	print(f" QUALITY INDICATORS")
#256	print(f"{'─'*W}")
#257	indicators = [
#258	("Noise < 30%", wm["noise_pct"] < 30, f"{wm['noise_pct']}%"),
#259	("Recall > 30%", pct(wm["never_recalled"], wm["total"]) < 70, f"{100 - pct(wm['never_recalled'], wm['total']):.1f}%"),
#260	("Global > 5%", wm["global_count"] > wm["total"] * 0.05, f"{pct(wm['global_count'], wm['total']):.1f}%"),
#261	("Embeddings OK", s["embeddings"] > 0, f"{s['embeddings']} vectors"),
#262	("Consolidation", con["events"] > 0, f"{con['events']} events"),
#263	("Dreamer active", dr["runs"] > 0, f"{dr['runs']} runs"),
#264	("Wiki promoted", wk["memories"] > 10, f"{wk['memories']} pages"),
#265	]
#266	for label, ok, detail in indicators:
#267	print(f" {'✓' if ok else '✗'} {label:25s} {detail}")
#268
#269	# Recommendations
#270	print(f"\n{'─'*W}")
#271	print(f" RECOMMENDATIONS")
#272	print(f"{'─'*W}")
#273	recs = []
#274	if wm["noise_pct"] > 30: recs.append(f" ! High noise ({wm['noise_pct']}%) — run cleanup-mnemosyne.py")
#275	if wm["never_recalled"] > wm["total"] * 0.5: recs.append(f" ! {wm['never_recalled']} never recalled — prune low-value items")
#276	if ep["total"] == 0 and wm["total"] > 0: recs.append(" ! No episodic memories — consolidation not promoting")
#277	if s["embeddings"] == 0 and wm["total"] > 10: recs.append(" ! Embedding pipeline broken — vec_episodes empty")
#278	if wk["memories"] < 10 and wm["global_count"] > 20: recs.append(f" ! Only {wk['memories']} wiki pages — promote global memories")
#279	if not recs: recs.append(" All systems healthy.")
#280	for r in recs: print(r)
#281
#282	print(f"\n{'='*W}")
#283
#284	# ─── Snapshot / Trend Tracking ────────────────────────────────────────────────
#285
#286	def save_snapshot(stats):
#287	"""Save a timestamped snapshot for trend tracking."""
#288	SNAPSHOT_DIR.mkdir(parents=True, exist_ok=True)
#289	ts = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
#290	snap_file = SNAPSHOT_DIR / f"snap_{ts}.json"
#291	snap = {
#292	"timestamp": datetime.now().isoformat(),
#293	"db_size_mb": stats.get("db_size_mb", 0),
#294	"wm_total": stats["working_memory"]["total"],
#295	"wm_noise_pct": stats["working_memory"]["noise_pct"],
#296	"wm_global_count": stats["working_memory"]["global_count"],
#297	"wm_never_recalled": stats["working_memory"]["never_recalled"],
#298	"ep_total": stats["episodic"]["total"],
#299	"triples_total": stats["triples"]["total"],
#300	"consolidation_events": stats["consolidation"]["events"],
#301	"consolidation_items": stats["consolidation"]["items"],
#302	"dreamer_runs": stats["dreamer"]["runs"],
#303	"dreamer_proposals": stats["dreamer"]["proposals"],
#304	"embeddings": stats["embeddings"],
#305	"wiki_pages": stats["wiki"]["total"],
#306	"wiki_memories": stats["wiki"]["memories"],
#307	"quality_score": stats["quality_score"],
#308	}
#309	with open(snap_file, "w") as f:
#310	json.dump(snap, f, indent=2)
#311	return snap_file
#312
#313	def load_trends():
#314	"""Load recent snapshots and compute deltas."""
#315	if not SNAPSHOT_DIR.exists():
#316	return None
#317	snaps = sorted(SNAPSHOT_DIR.glob("snap_*.json"))
#318	if len(snaps) < 2:
#319	return None
#320	# Find two valid snapshots (skip corrupted ones)
#321	valid_snaps = []
#322	for snap in snaps:
#323	try:
#324	with open(snap) as f:
#325	data = json.load(f)
#326	valid_snaps.append(data)
#327	if len(valid_snaps) >= 2:
#328	break
#329	except (json.JSONDecodeError, IOError):
#330	continue
#331	if len(valid_snaps) < 2:
#332	return None
#333	prev, curr = valid_snaps[-2], valid_snaps[-1]
#334	deltas = {}
#335	for key in curr:
#336	if key == "timestamp": continue
#337	if key in prev and isinstance(curr[key], (int, float)):
#338	delta = curr[key] - prev[key]
#339	deltas[key] = {
#340	"prev": prev[key], "curr": curr[key], "delta": delta,
#341	"pct_change": round(delta / prev[key] * 100, 1) if prev[key] != 0 else 0,
#342	}
#343	return {"prev_time": prev.get("timestamp"), "curr_time": curr.get("timestamp"),
#344	"snapshots_total": len(snaps), "deltas": deltas}
#345
#346	def show_trends(trends):
#347	"""Display trend data."""
#348	if not trends:
#349	print(f"\n No trend data yet (need 2+ snapshots)")
#350	print(f" Snapshots dir: {SNAPSHOT_DIR}")
#351	print(f" Run: python3 mnemosyne-stats.py --save-snapshot")
#352	return
#353	print(f"\n{'─' * W}")
#354	print(f" TRENDS (last 2 snapshots)")
#355	print(f"{'─' * W}")
#356	print(f" Previous: {trends['prev_time'][:19]}")
#357	print(f" Current: {trends['curr_time'][:19]}")
#358	print(f" Total snapshots: {trends['snapshots_total']}")
#359	print()
#360	trend_metrics = [
#361	("wm_total", "Working Memory"), ("wm_noise_pct", "Noise %"),
#362	("wm_never_recalled", "Never Recalled"), ("wm_global_count", "Global Memories"),
#363	("ep_total", "Episodic Memory"), ("triples_total", "Knowledge Triples"),
#364	("consolidation_events", "Consolidation Events"), ("dreamer_proposals", "Dreamer Proposals"),
#365	("embeddings", "Embeddings"), ("wiki_memories", "Wiki Memories"),
#366	("quality_score", "Quality Score"),
#367	]
#368	for key, label in trend_metrics:
#369	if key in trends["deltas"]:
#370	d = trends["deltas"][key]
#371	arrow = "↑" if d["delta"] > 0 else "↓" if d["delta"] < 0 else "="
#372	print(f" {label:20s} {d['prev']:>8} → {d['curr']:>8} {arrow} {d['delta']:+.1f} ({d['pct_change']:+.1f}%)")
#373
#374	def main():
#375	args = sys.argv[1:]
#376	compact = "--compact" in args
#377	s = collect()
#378	if "--json" in args:
#379	print(json.dumps(s, indent=2, default=str))
#380	elif "--save-snapshot" in args:
#381	snap_file = save_snapshot(s)
#382	print(f"Snapshot saved: {snap_file}")
#383	trends = load_trends()
#384	if trends: show_trends(trends)
#385	elif "--trends" in args:
#386	show_trends(load_trends())
#387	else:
#388	show(s, compact=compact)
#389	save_snapshot(s)
#390
#391	if __name__ == "__main__":
#392	main()
#393

z6Mkq5mY3JWtxoxUobWcfNHm7AkRubgSWEZTkBVqZXJviFZ5/my-project-public