fd454c4d79
- Upload progress bar with percentage, file count, speed, ETA - Detects active upload from upload_*.log files automatically - Last 10 upload lines shown with ✓/✗ color coding - Combined log panel shows both setup.log and upload logs - Upload folder distribution in API response Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
649 lines
26 KiB
Python
Executable File
649 lines
26 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
AnythingLLM Persona RAG Monitor
|
|
|
|
Usage:
|
|
python3 monitor.py # CLI one-shot
|
|
python3 monitor.py --watch # CLI auto-refresh (2s)
|
|
python3 monitor.py --web # Web dashboard on :8899
|
|
python3 monitor.py --web 9000 # Custom port
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from http.server import HTTPServer, BaseHTTPRequestHandler
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
try:
|
|
import requests
|
|
except ImportError:
|
|
requests = None
|
|
|
|
CONFIG_PATH = Path(__file__).parent / "config.yaml"
|
|
PROGRESS_PATH = Path(__file__).parent / "upload_progress.json"
|
|
LOG_PATH = Path(__file__).parent / "setup.log"
|
|
LANCEDB_PATH = Path.home() / ".config/anythingllm-desktop/storage/lancedb"
|
|
DOCS_PATH = Path.home() / ".config/anythingllm-desktop/storage/documents"
|
|
VCACHE_PATH = Path.home() / ".config/anythingllm-desktop/storage/vector-cache"
|
|
|
|
|
|
def load_config():
|
|
with open(CONFIG_PATH) as f:
|
|
return yaml.safe_load(f)
|
|
|
|
|
|
def load_progress():
|
|
if PROGRESS_PATH.exists():
|
|
with open(PROGRESS_PATH) as f:
|
|
return json.load(f)
|
|
return {}
|
|
|
|
|
|
def dir_size_mb(path):
|
|
if not path.exists():
|
|
return 0
|
|
try:
|
|
return sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) / (1024 * 1024)
|
|
except Exception:
|
|
return 0
|
|
|
|
|
|
def get_lance_workspaces():
|
|
if not LANCEDB_PATH.exists():
|
|
return set()
|
|
return {d.name.replace(".lance", "") for d in LANCEDB_PATH.iterdir()
|
|
if d.is_dir() and d.name.endswith(".lance")}
|
|
|
|
|
|
def get_lance_sizes():
|
|
sizes = {}
|
|
if not LANCEDB_PATH.exists():
|
|
return sizes
|
|
for d in LANCEDB_PATH.iterdir():
|
|
if d.is_dir() and d.name.endswith(".lance"):
|
|
slug = d.name.replace(".lance", "")
|
|
sizes[slug] = sum(f.stat().st_size for f in d.rglob("*") if f.is_file()) / (1024 * 1024)
|
|
return sizes
|
|
|
|
|
|
def check_api(config):
|
|
if not requests:
|
|
return None
|
|
try:
|
|
url = f"{config['anythingllm']['base_url']}/auth"
|
|
headers = {"Authorization": f"Bearer {config['anythingllm']['api_key']}"}
|
|
resp = requests.get(url, headers=headers, timeout=3)
|
|
return resp.status_code == 200
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def check_script_running():
|
|
try:
|
|
import subprocess
|
|
result = subprocess.run(
|
|
["pgrep", "-af", "setup.py"],
|
|
capture_output=True, text=True
|
|
)
|
|
# Filter out monitor.py and grep itself
|
|
for line in result.stdout.strip().split("\n"):
|
|
if line and "monitor" not in line and "pgrep" not in line:
|
|
return True
|
|
return False
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def collect_status():
|
|
config = load_config()
|
|
progress = load_progress()
|
|
|
|
workspaces = config.get("workspaces", {})
|
|
ws_docs = progress.get("workspace_docs", {})
|
|
uploaded = progress.get("uploaded_files", {})
|
|
lance_ws = get_lance_workspaces()
|
|
lance_sizes = get_lance_sizes()
|
|
|
|
# Get real slugs from API
|
|
api_slugs = {} # workspace name → slug
|
|
if requests:
|
|
try:
|
|
url = f"{config['anythingllm']['base_url']}/workspaces"
|
|
headers = {"Authorization": f"Bearer {config['anythingllm']['api_key']}"}
|
|
resp = requests.get(url, headers=headers, timeout=5)
|
|
if resp.status_code == 200:
|
|
for ws in resp.json().get("workspaces", []):
|
|
api_slugs[ws["name"]] = ws["slug"]
|
|
except Exception:
|
|
pass
|
|
|
|
# Build expected doc counts per persona
|
|
folder_counts = {}
|
|
for fpath, info in uploaded.items():
|
|
f = info.get("folder", "")
|
|
if f:
|
|
folder_counts[f] = folder_counts.get(f, 0) + 1
|
|
|
|
personas = []
|
|
for codename, ws_cfg in workspaces.items():
|
|
# Use real slug from API, fallback to name-based guess
|
|
slug = api_slugs.get(ws_cfg["name"], codename)
|
|
|
|
# Expected docs from mapped folders
|
|
expected = 0
|
|
for entry in ws_cfg.get("folders", []):
|
|
fn = entry["path"].replace("/", "_")
|
|
expected += folder_counts.get(fn, 0)
|
|
|
|
assigned = len(ws_docs.get(codename, []))
|
|
has_vectors = any(slug in lw for lw in lance_ws)
|
|
vector_size = 0
|
|
for lw, sz in lance_sizes.items():
|
|
if slug in lw:
|
|
vector_size = sz
|
|
break
|
|
|
|
personas.append({
|
|
"codename": codename,
|
|
"name": ws_cfg["name"],
|
|
"slug": slug,
|
|
"expected": expected,
|
|
"assigned": assigned,
|
|
"has_vectors": has_vectors,
|
|
"vector_size_mb": vector_size,
|
|
})
|
|
|
|
# Cluster grouping
|
|
clusters = {
|
|
"intel": ["frodo", "echo", "ghost", "oracle", "wraith", "scribe", "polyglot"],
|
|
"cyber": ["neo", "bastion", "sentinel", "specter", "phantom", "cipher", "vortex"],
|
|
"military": ["marshal", "centurion", "corsair", "warden", "medic"],
|
|
"humanities": ["chronos", "tribune", "arbiter", "ledger", "sage", "herald", "scholar", "gambit"],
|
|
"engineering": ["forge", "architect"],
|
|
}
|
|
|
|
api_ok = check_api(config)
|
|
script_running = check_script_running()
|
|
|
|
# Upload progress from upload_progress.json
|
|
upload_total = len(uploaded)
|
|
upload_folders = {}
|
|
for info in uploaded.values():
|
|
f = info.get("folder", "?")
|
|
upload_folders[f] = upload_folders.get(f, 0) + 1
|
|
|
|
# Detect active upload from log files (upload_cia_en.log or similar)
|
|
upload_active = None
|
|
upload_current = 0
|
|
upload_target = 0
|
|
upload_speed = None
|
|
upload_log_lines = []
|
|
for logfile in sorted(Path(__file__).parent.glob("upload_*.log"), key=lambda p: p.stat().st_mtime, reverse=True):
|
|
try:
|
|
with open(logfile, "r", encoding="utf-8", errors="ignore") as f:
|
|
ulines = f.readlines()
|
|
upload_log_lines = [l.rstrip() for l in ulines[-10:]]
|
|
import re as _re
|
|
for line in reversed(ulines):
|
|
m = _re.search(r'\[(\d+)/(\d+)\]', line)
|
|
if m:
|
|
upload_current = int(m.group(1))
|
|
upload_target = int(m.group(2))
|
|
upload_active = logfile.name
|
|
break
|
|
# Speed: count ✓ lines with timestamps in last 20
|
|
ok_times = []
|
|
for line in ulines:
|
|
if '✓' in line or '✗' in line:
|
|
m2 = _re.search(r'\[(\d+)/\d+\]', line)
|
|
if m2:
|
|
ok_times.append(int(m2.group(1)))
|
|
if len(ok_times) >= 2 and upload_target > 0:
|
|
# Rough speed from file modification time
|
|
age = time.time() - logfile.stat().st_mtime
|
|
if age < 300 and upload_current > 0: # active in last 5 min
|
|
elapsed = time.time() - logfile.stat().st_ctime
|
|
if elapsed > 0 and upload_current > 0:
|
|
upload_speed = elapsed / upload_current # seconds per file
|
|
if upload_active:
|
|
break
|
|
except Exception:
|
|
pass
|
|
|
|
# Read last N lines from setup.log
|
|
log_lines = []
|
|
batch_times = []
|
|
if LOG_PATH.exists():
|
|
try:
|
|
with open(LOG_PATH, "r", encoding="utf-8") as f:
|
|
all_lines = f.readlines()
|
|
log_lines = [l.rstrip() for l in all_lines[-15:]]
|
|
|
|
# Parse batch timestamps to calculate ETA
|
|
import re
|
|
for line in all_lines:
|
|
m = re.match(r'^(\d{2}:\d{2}:\d{2}) \[INFO\]\s+✓\s+\w+ batch \d+/\d+:', line)
|
|
if m:
|
|
batch_times.append(m.group(1))
|
|
except Exception:
|
|
pass
|
|
|
|
# Calculate ETA from batch speed
|
|
total_expected = sum(p["expected"] for p in personas)
|
|
total_assigned = sum(len(v) for v in ws_docs.values())
|
|
remaining = max(0, total_expected - total_assigned)
|
|
|
|
eta_seconds = None
|
|
avg_batch_seconds = None
|
|
if len(batch_times) >= 2:
|
|
try:
|
|
from datetime import datetime
|
|
times = []
|
|
for t in batch_times[-20:]: # last 20 batches for avg
|
|
dt = datetime.strptime(t, "%H:%M:%S")
|
|
times.append(dt.hour * 3600 + dt.minute * 60 + dt.second)
|
|
if len(times) >= 2:
|
|
deltas = [times[i+1] - times[i] for i in range(len(times)-1)]
|
|
# Handle midnight wrap
|
|
deltas = [d if d > 0 else d + 86400 for d in deltas]
|
|
avg_batch_seconds = sum(deltas) / len(deltas)
|
|
remaining_batches = remaining / 5 # batch size = 5
|
|
eta_seconds = int(remaining_batches * avg_batch_seconds)
|
|
except Exception:
|
|
pass
|
|
|
|
# Format ETA
|
|
eta_str = None
|
|
if eta_seconds is not None:
|
|
hours = eta_seconds // 3600
|
|
minutes = (eta_seconds % 3600) // 60
|
|
if hours > 0:
|
|
eta_str = f"{hours}h {minutes}m"
|
|
else:
|
|
eta_str = f"{minutes}m"
|
|
|
|
# Upload ETA
|
|
upload_eta = None
|
|
upload_remaining = max(0, upload_target - upload_current) if upload_target else 0
|
|
if upload_speed and upload_remaining > 0:
|
|
eta_s = int(upload_remaining * upload_speed)
|
|
h, m = eta_s // 3600, (eta_s % 3600) // 60
|
|
upload_eta = f"{h}h {m}m" if h > 0 else f"{m}m"
|
|
|
|
return {
|
|
"personas": personas,
|
|
"clusters": clusters,
|
|
"total_uploaded": len(uploaded),
|
|
"total_assigned": total_assigned,
|
|
"total_expected": total_expected,
|
|
"upload_active": upload_active,
|
|
"upload_current": upload_current,
|
|
"upload_target": upload_target,
|
|
"upload_speed": round(upload_speed, 1) if upload_speed else None,
|
|
"upload_eta": upload_eta,
|
|
"upload_remaining": upload_remaining,
|
|
"upload_folders": dict(sorted(upload_folders.items(), key=lambda x: -x[1])[:10]),
|
|
"upload_log_lines": upload_log_lines,
|
|
"total_personas": len(workspaces),
|
|
"personas_with_vectors": sum(1 for p in personas if p["has_vectors"]),
|
|
"lancedb_size_mb": dir_size_mb(LANCEDB_PATH),
|
|
"docs_size_mb": dir_size_mb(DOCS_PATH),
|
|
"vcache_size_mb": dir_size_mb(VCACHE_PATH),
|
|
"api_online": api_ok,
|
|
"script_running": script_running,
|
|
"timestamp": time.strftime("%H:%M:%S"),
|
|
"log_tail": log_lines,
|
|
"eta": eta_str,
|
|
"avg_batch_seconds": round(avg_batch_seconds, 1) if avg_batch_seconds else None,
|
|
"remaining_docs": remaining,
|
|
}
|
|
|
|
|
|
# ──────────────────────────────────────────────────
|
|
# CLI OUTPUT
|
|
# ──────────────────────────────────────────────────
|
|
|
|
CLUSTER_COLORS = {
|
|
"intel": "\033[34m", # blue
|
|
"cyber": "\033[31m", # red
|
|
"military": "\033[33m", # yellow
|
|
"humanities": "\033[35m", # magenta
|
|
"engineering": "\033[36m",# cyan
|
|
}
|
|
RESET = "\033[0m"
|
|
BOLD = "\033[1m"
|
|
DIM = "\033[2m"
|
|
GREEN = "\033[32m"
|
|
RED = "\033[31m"
|
|
|
|
|
|
def progress_bar(current, total, width=20):
|
|
if total == 0:
|
|
return f"{'░' * width}"
|
|
filled = int(width * min(current, total) / total)
|
|
return f"{'█' * filled}{'░' * (width - filled)}"
|
|
|
|
|
|
def cli_output(status):
|
|
lines = []
|
|
lines.append(f"{BOLD}═══ AnythingLLM Persona Monitor ═══{RESET} {DIM}{status['timestamp']}{RESET}")
|
|
lines.append("")
|
|
|
|
# System status
|
|
api = f"{GREEN}●{RESET}" if status["api_online"] else f"{RED}●{RESET}"
|
|
script = f"{GREEN}● running{RESET}" if status["script_running"] else f"{DIM}○ idle{RESET}"
|
|
lines.append(f" API: {api} Script: {script} "
|
|
f"LanceDB: {status['lancedb_size_mb']:.0f}MB "
|
|
f"Docs: {status['docs_size_mb']:.0f}MB")
|
|
eta_str = ""
|
|
if status.get("eta"):
|
|
eta_str = f" ETA: {GREEN}{status['eta']}{RESET} ({status['remaining_docs']} left, {status['avg_batch_seconds']}s/batch)"
|
|
lines.append(f" Uploaded: {status['total_uploaded']} "
|
|
f"Assigned: {status['total_assigned']}/{status.get('total_expected', '?')} "
|
|
f"Vectors: {status['personas_with_vectors']}/{status['total_personas']}")
|
|
if eta_str:
|
|
lines.append(eta_str)
|
|
lines.append("")
|
|
|
|
# Per-cluster persona table
|
|
persona_map = {p["codename"]: p for p in status["personas"]}
|
|
|
|
for cluster_name, members in status["clusters"].items():
|
|
color = CLUSTER_COLORS.get(cluster_name, "")
|
|
lines.append(f" {color}{BOLD}{cluster_name.upper()}{RESET}")
|
|
|
|
for codename in members:
|
|
p = persona_map.get(codename)
|
|
if not p:
|
|
continue
|
|
|
|
vec_icon = f"{GREEN}✓{RESET}" if p["has_vectors"] else f"{DIM}○{RESET}"
|
|
bar = progress_bar(p["assigned"], p["expected"])
|
|
pct = (p["assigned"] / p["expected"] * 100) if p["expected"] > 0 else 0
|
|
size_str = f"{p['vector_size_mb']:.0f}MB" if p["vector_size_mb"] > 0 else ""
|
|
|
|
lines.append(f" {vec_icon} {codename:<12} {bar} {p['assigned']:>5}/{p['expected']:<5} "
|
|
f"{pct:>5.0f}% {size_str}")
|
|
|
|
lines.append("")
|
|
|
|
# Log tail
|
|
log_tail = status.get("log_tail", [])
|
|
if log_tail:
|
|
lines.append(f" {BOLD}── Log (setup.log) ──{RESET}")
|
|
for ll in log_tail:
|
|
# Colorize log levels
|
|
if "[ERROR]" in ll:
|
|
lines.append(f" {RED}{ll}{RESET}")
|
|
elif "[WARNING]" in ll:
|
|
lines.append(f" \033[33m{ll}{RESET}")
|
|
elif "✓" in ll:
|
|
lines.append(f" {GREEN}{ll}{RESET}")
|
|
else:
|
|
lines.append(f" {DIM}{ll}{RESET}")
|
|
lines.append("")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def cli_mode(watch=False):
|
|
while True:
|
|
status = collect_status()
|
|
if watch:
|
|
os.system("clear")
|
|
print(cli_output(status))
|
|
if not watch:
|
|
break
|
|
time.sleep(2)
|
|
|
|
|
|
# ──────────────────────────────────────────────────
|
|
# WEB DASHBOARD
|
|
# ──────────────────────────────────────────────────
|
|
|
|
HTML_TEMPLATE = """<!DOCTYPE html>
|
|
<html lang="tr">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<title>AnythingLLM Monitor</title>
|
|
<style>
|
|
* { margin: 0; padding: 0; box-sizing: border-box; }
|
|
body { background: #0a0a0f; color: #e0e0e0; font-family: 'JetBrains Mono', 'Fira Code', monospace; font-size: 14px; padding: 24px; }
|
|
h1 { color: #7aa2f7; font-size: 18px; margin-bottom: 4px; }
|
|
.subtitle { color: #565f89; font-size: 12px; margin-bottom: 20px; }
|
|
.status-bar { display: flex; gap: 24px; margin-bottom: 20px; padding: 12px 16px; background: #13131a; border-radius: 8px; border: 1px solid #1a1b26; }
|
|
.status-item { display: flex; align-items: center; gap: 6px; }
|
|
.dot { width: 8px; height: 8px; border-radius: 50%; }
|
|
.dot.green { background: #9ece6a; box-shadow: 0 0 6px #9ece6a44; }
|
|
.dot.red { background: #f7768e; box-shadow: 0 0 6px #f7768e44; }
|
|
.dot.dim { background: #565f89; }
|
|
.stat { color: #7aa2f7; font-weight: bold; }
|
|
.cluster { margin-bottom: 16px; }
|
|
.cluster-name { font-size: 13px; font-weight: bold; padding: 6px 12px; border-radius: 4px 4px 0 0; display: inline-block; margin-bottom: 2px; }
|
|
.cluster-intel .cluster-name { background: #1a1b3a; color: #7aa2f7; }
|
|
.cluster-cyber .cluster-name { background: #2a1a1a; color: #f7768e; }
|
|
.cluster-military .cluster-name { background: #2a2a1a; color: #e0af68; }
|
|
.cluster-humanities .cluster-name { background: #2a1a2a; color: #bb9af7; }
|
|
.cluster-engineering .cluster-name { background: #1a2a2a; color: #73daca; }
|
|
.persona-table { width: 100%; border-collapse: collapse; }
|
|
.persona-table td { padding: 5px 10px; border-bottom: 1px solid #1a1b26; }
|
|
.persona-table tr:hover { background: #13131a; }
|
|
.persona-name { color: #c0caf5; min-width: 120px; }
|
|
.progress-wrap { width: 200px; }
|
|
.progress-bg { background: #1a1b26; border-radius: 3px; height: 14px; overflow: hidden; position: relative; }
|
|
.progress-fill { height: 100%; border-radius: 3px; transition: width 0.5s ease; }
|
|
.fill-intel { background: linear-gradient(90deg, #3d59a1, #7aa2f7); }
|
|
.fill-cyber { background: linear-gradient(90deg, #a1304d, #f7768e); }
|
|
.fill-military { background: linear-gradient(90deg, #8a6d2e, #e0af68); }
|
|
.fill-humanities { background: linear-gradient(90deg, #6e3da1, #bb9af7); }
|
|
.fill-engineering { background: linear-gradient(90deg, #2e8a6d, #73daca); }
|
|
.progress-pct { position: absolute; right: 6px; top: 0; font-size: 10px; line-height: 14px; color: #fff; text-shadow: 0 0 4px rgba(0,0,0,0.8); }
|
|
.counts { color: #565f89; font-size: 12px; min-width: 100px; text-align: right; }
|
|
.vec-icon { font-size: 14px; min-width: 20px; text-align: center; }
|
|
.vec-ok { color: #9ece6a; }
|
|
.vec-no { color: #565f89; }
|
|
.size { color: #565f89; font-size: 11px; min-width: 60px; text-align: right; }
|
|
.summary { display: flex; gap: 16px; margin-bottom: 20px; flex-wrap: wrap; }
|
|
.summary-card { background: #13131a; border: 1px solid #1a1b26; border-radius: 8px; padding: 12px 16px; min-width: 120px; }
|
|
.summary-card .label { color: #565f89; font-size: 11px; text-transform: uppercase; }
|
|
.summary-card .value { color: #7aa2f7; font-size: 20px; font-weight: bold; margin-top: 2px; }
|
|
.summary-card .unit { color: #565f89; font-size: 12px; }
|
|
.upload-panel { background: #13131a; border: 1px solid #1a1b26; border-radius: 8px; padding: 12px 16px; margin-top: 16px; }
|
|
.upload-panel h3 { color: #e0af68; font-size: 13px; margin-bottom: 8px; }
|
|
.upload-bar { background: #1a1b26; border-radius: 3px; height: 20px; overflow: hidden; position: relative; margin: 8px 0; }
|
|
.upload-fill { height: 100%; border-radius: 3px; background: linear-gradient(90deg, #8a6d2e, #e0af68); transition: width 0.5s ease; }
|
|
.upload-pct { position: absolute; right: 8px; top: 2px; font-size: 12px; color: #fff; text-shadow: 0 0 4px rgba(0,0,0,0.8); }
|
|
.upload-stats { display: flex; gap: 16px; font-size: 12px; color: #565f89; margin-top: 4px; }
|
|
.upload-stats span { color: #e0af68; }
|
|
.upload-log { font-size: 11px; color: #565f89; margin-top: 8px; max-height: 120px; overflow-y: auto; }
|
|
.upload-log div { white-space: pre-wrap; line-height: 1.4; }
|
|
.upload-log .ok { color: #9ece6a; }
|
|
.upload-log .fail { color: #f7768e; }
|
|
.log-panel { background: #0d0d12; border: 1px solid #1a1b26; border-radius: 8px; padding: 12px 16px; margin-top: 20px; max-height: 300px; overflow-y: auto; }
|
|
.log-panel h3 { color: #565f89; font-size: 12px; text-transform: uppercase; margin-bottom: 8px; }
|
|
.log-line { font-size: 12px; line-height: 1.6; color: #565f89; white-space: pre-wrap; word-break: break-all; }
|
|
.log-line.error { color: #f7768e; }
|
|
.log-line.warning { color: #e0af68; }
|
|
.log-line.success { color: #9ece6a; }
|
|
.log-line.info { color: #7aa2f7; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>AnythingLLM Persona Monitor</h1>
|
|
<div class="subtitle" id="timestamp">Loading...</div>
|
|
|
|
<div class="summary" id="summary"></div>
|
|
<div class="status-bar" id="statusbar"></div>
|
|
<div id="uploadpanel"></div>
|
|
<div id="clusters"></div>
|
|
<div class="log-panel" id="logpanel"><h3>Log (setup.log)</h3><div id="loglines">No log data</div></div>
|
|
|
|
<script>
|
|
const CLUSTER_ORDER = ['intel', 'cyber', 'military', 'humanities', 'engineering'];
|
|
|
|
function render(data) {
|
|
document.getElementById('timestamp').textContent = 'Updated: ' + data.timestamp + ' — auto-refresh 2s';
|
|
|
|
// Summary cards
|
|
const pctVec = Math.round(data.personas_with_vectors / data.total_personas * 100);
|
|
document.getElementById('summary').innerHTML = `
|
|
<div class="summary-card"><div class="label">Uploaded</div><div class="value">${data.total_uploaded}</div><div class="unit">files</div></div>
|
|
<div class="summary-card"><div class="label">Assigned</div><div class="value">${data.total_assigned}</div><div class="unit">docs</div></div>
|
|
<div class="summary-card"><div class="label">Vectors</div><div class="value">${data.personas_with_vectors}<span class="unit">/${data.total_personas}</span></div><div class="unit">${pctVec}%</div></div>
|
|
<div class="summary-card"><div class="label">LanceDB</div><div class="value">${data.lancedb_size_mb < 1 ? Math.round(data.lancedb_size_mb * 1024) + '<span class="unit">KB</span>' : Math.round(data.lancedb_size_mb) + '<span class="unit">MB</span>'}</div></div>
|
|
<div class="summary-card"><div class="label">Documents</div><div class="value">${Math.round(data.docs_size_mb)}<span class="unit">MB</span></div></div>
|
|
${data.eta ? `<div class="summary-card"><div class="label">ETA</div><div class="value">${data.eta}</div><div class="unit">${data.remaining_docs} docs left · ${data.avg_batch_seconds}s/batch</div></div>` : ''}
|
|
`;
|
|
|
|
// Status bar
|
|
const apiDot = data.api_online ? 'green' : 'red';
|
|
const scriptDot = data.script_running ? 'green' : 'dim';
|
|
const scriptText = data.script_running ? 'running' : 'idle';
|
|
document.getElementById('statusbar').innerHTML = `
|
|
<div class="status-item"><span class="dot ${apiDot}"></span>API</div>
|
|
<div class="status-item"><span class="dot ${scriptDot}"></span>Script ${scriptText}</div>
|
|
`;
|
|
|
|
// Clusters
|
|
const personaMap = {};
|
|
data.personas.forEach(p => personaMap[p.codename] = p);
|
|
|
|
let html = '';
|
|
CLUSTER_ORDER.forEach(cl => {
|
|
const members = data.clusters[cl] || [];
|
|
html += `<div class="cluster cluster-${cl}"><span class="cluster-name">${cl.toUpperCase()}</span><table class="persona-table">`;
|
|
members.forEach(code => {
|
|
const p = personaMap[code];
|
|
if (!p) return;
|
|
const pct = p.expected > 0 ? Math.round(p.assigned / p.expected * 100) : 0;
|
|
const vecClass = p.has_vectors ? 'vec-ok' : 'vec-no';
|
|
const vecIcon = p.has_vectors ? '✓' : '○';
|
|
const sizeStr = p.vector_size_mb > 0 ? (p.vector_size_mb < 1 ? Math.round(p.vector_size_mb * 1024) + 'KB' : Math.round(p.vector_size_mb) + 'MB') : '';
|
|
html += `<tr>
|
|
<td class="vec-icon ${vecClass}">${vecIcon}</td>
|
|
<td class="persona-name">${code}</td>
|
|
<td class="progress-wrap"><div class="progress-bg"><div class="progress-fill fill-${cl}" style="width:${Math.min(pct,100)}%"></div><span class="progress-pct">${pct}%</span></div></td>
|
|
<td class="counts">${p.assigned} / ${p.expected}</td>
|
|
<td class="size">${sizeStr}</td>
|
|
</tr>`;
|
|
});
|
|
html += '</table></div>';
|
|
});
|
|
document.getElementById('clusters').innerHTML = html;
|
|
|
|
// Upload panel
|
|
const upEl = document.getElementById('uploadpanel');
|
|
if (data.upload_active && data.upload_target > 0) {
|
|
const upPct = Math.round(data.upload_current / data.upload_target * 100);
|
|
let upStats = `<span>${data.upload_current}</span> / ${data.upload_target} files`;
|
|
if (data.upload_speed) upStats += ` · <span>${data.upload_speed}s</span>/file`;
|
|
if (data.upload_eta) upStats += ` · ETA <span>${data.upload_eta}</span>`;
|
|
if (data.upload_remaining) upStats += ` · ${data.upload_remaining} remaining`;
|
|
|
|
let upLog = '';
|
|
(data.upload_log_lines || []).forEach(line => {
|
|
let cls = '';
|
|
if (line.includes('✓')) cls = 'ok';
|
|
else if (line.includes('✗')) cls = 'fail';
|
|
upLog += `<div class="${cls}">${line.replace(/</g,'<')}</div>`;
|
|
});
|
|
|
|
upEl.innerHTML = `<div class="upload-panel">
|
|
<h3>Upload: ${data.upload_active}</h3>
|
|
<div class="upload-bar"><div class="upload-fill" style="width:${upPct}%"></div><span class="upload-pct">${upPct}%</span></div>
|
|
<div class="upload-stats">${upStats}</div>
|
|
<div class="upload-log">${upLog}</div>
|
|
</div>`;
|
|
} else if (data.total_uploaded > 0) {
|
|
upEl.innerHTML = `<div class="upload-panel"><h3>Upload: idle</h3><div class="upload-stats">Total uploaded: <span>${data.total_uploaded}</span> files</div></div>`;
|
|
} else {
|
|
upEl.innerHTML = '';
|
|
}
|
|
|
|
// Log panel — show both setup.log and upload logs
|
|
const logLines = data.log_tail || [];
|
|
const uploadLogLines = data.upload_log_lines || [];
|
|
const allLogs = [...logLines, ...uploadLogLines.map(l => '[UPLOAD] ' + l)];
|
|
if (allLogs.length > 0) {
|
|
let logHtml = '';
|
|
allLogs.forEach(line => {
|
|
let cls = '';
|
|
if (line.includes('[ERROR]') || line.includes('✗')) cls = 'error';
|
|
else if (line.includes('[WARNING]')) cls = 'warning';
|
|
else if (line.includes('✓')) cls = 'success';
|
|
else if (line.includes('[INFO]') || line.includes('[UPLOAD]')) cls = 'info';
|
|
logHtml += `<div class="log-line ${cls}">${line.replace(/</g,'<')}</div>`;
|
|
});
|
|
document.getElementById('loglines').innerHTML = logHtml;
|
|
const panel = document.getElementById('logpanel');
|
|
panel.scrollTop = panel.scrollHeight;
|
|
}
|
|
}
|
|
|
|
async function poll() {
|
|
try {
|
|
const resp = await fetch('/api/status');
|
|
const data = await resp.json();
|
|
render(data);
|
|
} catch(e) {
|
|
document.getElementById('timestamp').textContent = 'Connection lost — retrying...';
|
|
}
|
|
setTimeout(poll, 2000);
|
|
}
|
|
poll();
|
|
</script>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
class MonitorHandler(BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
if self.path == "/api/status":
|
|
status = collect_status()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Access-Control-Allow-Origin", "*")
|
|
self.end_headers()
|
|
self.wfile.write(json.dumps(status).encode())
|
|
elif self.path in ("/", "/index.html"):
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/html; charset=utf-8")
|
|
self.end_headers()
|
|
self.wfile.write(HTML_TEMPLATE.encode())
|
|
else:
|
|
self.send_response(404)
|
|
self.end_headers()
|
|
|
|
def log_message(self, format, *args):
|
|
pass # quiet
|
|
|
|
|
|
def web_mode(port=8899):
|
|
server = HTTPServer(("0.0.0.0", port), MonitorHandler)
|
|
print(f" AnythingLLM Monitor → http://localhost:{port}")
|
|
print(f" API endpoint → http://localhost:{port}/api/status")
|
|
print(f" Press Ctrl+C to stop\n")
|
|
try:
|
|
server.serve_forever()
|
|
except KeyboardInterrupt:
|
|
print("\n Stopped.")
|
|
server.server_close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
args = sys.argv[1:]
|
|
|
|
if "--web" in args:
|
|
idx = args.index("--web")
|
|
port = int(args[idx + 1]) if idx + 1 < len(args) and args[idx + 1].isdigit() else 8899
|
|
web_mode(port)
|
|
elif "--watch" in args:
|
|
cli_mode(watch=True)
|
|
else:
|
|
cli_mode(watch=False)
|