salvacybersec/anything-llm-rag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add structured logging + log panel to monitor

Browse Source 
- setup.py: logging module with file (setup.log) + console output
  - Line-buffered output (fixes background execution buffering)
  - API calls with timeout (300s), retry (3x), debug logging
  - Per-batch progress: [1/29] persona batch 1/20 (20 docs)
  - --verbose flag for debug-level console
- monitor.py: log tail in CLI + web dashboard
  - CLI: colorized last 15 log lines
  - Web: scrollable log panel with level-based colors
- Smaller embed batches (20 instead of 50) for reliability

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
salvacybersec
2026-04-07 00:30:29 +03:00
parent 9105c03b4b
commit 1028d11507
3 changed files with 177 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
monitor.py
View File

@@ -25,6 +25,7 @@ except ImportError:
CONFIG_PATH = Path(__file__).parent / "config.yaml"
PROGRESS_PATH = Path(__file__).parent / "upload_progress.json"
LOG_PATH = Path(__file__).parent / "setup.log"
LANCEDB_PATH = Path.home() / ".config/anythingllm-desktop/storage/lancedb"
DOCS_PATH = Path.home() / ".config/anythingllm-desktop/storage/documents"
VCACHE_PATH = Path.home() / ".config/anythingllm-desktop/storage/vector-cache"
@@ -152,6 +153,16 @@ def collect_status():
api_ok = check_api(config)
script_running = check_script_running()
# Read last N lines from setup.log
log_lines = []
if LOG_PATH.exists():
try:
with open(LOG_PATH, "r", encoding="utf-8") as f:
all_lines = f.readlines()
log_lines = [l.rstrip() for l in all_lines[-15:]]
except Exception:
pass
return {
"personas": personas,
"clusters": clusters,
@@ -165,6 +176,7 @@ def collect_status():
"api_online": api_ok,
"script_running": script_running,
"timestamp": time.strftime("%H:%M:%S"),
"log_tail": log_lines,
}
@@ -231,6 +243,22 @@ def cli_output(status):
lines.append("")
# Log tail
log_tail = status.get("log_tail", [])
if log_tail:
lines.append(f" {BOLD}── Log (setup.log) ──{RESET}")
for ll in log_tail:
# Colorize log levels
if "[ERROR]" in ll:
lines.append(f" {RED}{ll}{RESET}")
elif "[WARNING]" in ll:
lines.append(f" \033[33m{ll}{RESET}")
elif "✓" in ll:
lines.append(f" {GREEN}{ll}{RESET}")
else:
lines.append(f" {DIM}{ll}{RESET}")
lines.append("")
return "\n".join(lines)
@@ -297,6 +325,13 @@ HTML_TEMPLATE = """<!DOCTYPE html>
.summary-card .label { color: #565f89; font-size: 11px; text-transform: uppercase; }
.summary-card .value { color: #7aa2f7; font-size: 20px; font-weight: bold; margin-top: 2px; }
.summary-card .unit { color: #565f89; font-size: 12px; }
.log-panel { background: #0d0d12; border: 1px solid #1a1b26; border-radius: 8px; padding: 12px 16px; margin-top: 20px; max-height: 300px; overflow-y: auto; }
.log-panel h3 { color: #565f89; font-size: 12px; text-transform: uppercase; margin-bottom: 8px; }
.log-line { font-size: 12px; line-height: 1.6; color: #565f89; white-space: pre-wrap; word-break: break-all; }
.log-line.error { color: #f7768e; }
.log-line.warning { color: #e0af68; }
.log-line.success { color: #9ece6a; }
.log-line.info { color: #7aa2f7; }
</style>
</head>
<body>
@@ -306,6 +341,7 @@ HTML_TEMPLATE = """<!DOCTYPE html>
<div class="summary" id="summary"></div>
<div class="status-bar" id="statusbar"></div>
<div id="clusters"></div>
<div class="log-panel" id="logpanel"><h3>Log (setup.log)</h3><div id="loglines">No log data</div></div>
<script>
const CLUSTER_ORDER = ['intel', 'cyber', 'military', 'humanities', 'engineering'];
@@ -358,6 +394,23 @@ function render(data) {
html += '</table></div>';
});
document.getElementById('clusters').innerHTML = html;
// Log panel
const logLines = data.log_tail || [];
if (logLines.length > 0) {
let logHtml = '';
logLines.forEach(line => {
let cls = '';
if (line.includes('[ERROR]')) cls = 'error';
else if (line.includes('[WARNING]')) cls = 'warning';
else if (line.includes('✓')) cls = 'success';
else if (line.includes('[INFO]')) cls = 'info';
logHtml += `<div class="log-line ${cls}">${line.replace(/</g,'&lt;')}</div>`;
});
document.getElementById('loglines').innerHTML = logHtml;
const panel = document.getElementById('logpanel');
panel.scrollTop = panel.scrollHeight;
}
}
async function poll() {