From 1028d11507f58c4f0a9e5df317fdc7dc6f4280b5 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Tue, 7 Apr 2026 00:30:29 +0300 Subject: [PATCH] Add structured logging + log panel to monitor - setup.py: logging module with file (setup.log) + console output - Line-buffered output (fixes background execution buffering) - API calls with timeout (300s), retry (3x), debug logging - Per-batch progress: [1/29] persona batch 1/20 (20 docs) - --verbose flag for debug-level console - monitor.py: log tail in CLI + web dashboard - CLI: colorized last 15 log lines - Web: scrollable log panel with level-based colors - Smaller embed batches (20 instead of 50) for reliability Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 1 + monitor.py | 53 +++++++++++++++++++ setup.py | 149 +++++++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 177 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index ebb715b..f1cc37c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # State files (machine-specific, regenerated by script) upload_progress.json +setup.log # OCR output (large binary files) ocr_output/ diff --git a/monitor.py b/monitor.py index 1d09bdc..5c5f3ec 100755 --- a/monitor.py +++ b/monitor.py @@ -25,6 +25,7 @@ except ImportError: CONFIG_PATH = Path(__file__).parent / "config.yaml" PROGRESS_PATH = Path(__file__).parent / "upload_progress.json" +LOG_PATH = Path(__file__).parent / "setup.log" LANCEDB_PATH = Path.home() / ".config/anythingllm-desktop/storage/lancedb" DOCS_PATH = Path.home() / ".config/anythingllm-desktop/storage/documents" VCACHE_PATH = Path.home() / ".config/anythingllm-desktop/storage/vector-cache" @@ -152,6 +153,16 @@ def collect_status(): api_ok = check_api(config) script_running = check_script_running() + # Read last N lines from setup.log + log_lines = [] + if LOG_PATH.exists(): + try: + with open(LOG_PATH, "r", encoding="utf-8") as f: + all_lines = f.readlines() + log_lines = [l.rstrip() for l in all_lines[-15:]] + except Exception: + pass + return { "personas": personas, "clusters": clusters, @@ -165,6 +176,7 @@ def collect_status(): "api_online": api_ok, "script_running": script_running, "timestamp": time.strftime("%H:%M:%S"), + "log_tail": log_lines, } @@ -231,6 +243,22 @@ def cli_output(status): lines.append("") + # Log tail + log_tail = status.get("log_tail", []) + if log_tail: + lines.append(f" {BOLD}── Log (setup.log) ──{RESET}") + for ll in log_tail: + # Colorize log levels + if "[ERROR]" in ll: + lines.append(f" {RED}{ll}{RESET}") + elif "[WARNING]" in ll: + lines.append(f" \033[33m{ll}{RESET}") + elif "✓" in ll: + lines.append(f" {GREEN}{ll}{RESET}") + else: + lines.append(f" {DIM}{ll}{RESET}") + lines.append("") + return "\n".join(lines) @@ -297,6 +325,13 @@ HTML_TEMPLATE = """ .summary-card .label { color: #565f89; font-size: 11px; text-transform: uppercase; } .summary-card .value { color: #7aa2f7; font-size: 20px; font-weight: bold; margin-top: 2px; } .summary-card .unit { color: #565f89; font-size: 12px; } + .log-panel { background: #0d0d12; border: 1px solid #1a1b26; border-radius: 8px; padding: 12px 16px; margin-top: 20px; max-height: 300px; overflow-y: auto; } + .log-panel h3 { color: #565f89; font-size: 12px; text-transform: uppercase; margin-bottom: 8px; } + .log-line { font-size: 12px; line-height: 1.6; color: #565f89; white-space: pre-wrap; word-break: break-all; } + .log-line.error { color: #f7768e; } + .log-line.warning { color: #e0af68; } + .log-line.success { color: #9ece6a; } + .log-line.info { color: #7aa2f7; } @@ -306,6 +341,7 @@ HTML_TEMPLATE = """
+

Log (setup.log)

No log data