From fd454c4d79011d974661934b5b884ed285f3ec0c Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Wed, 8 Apr 2026 00:48:22 +0300 Subject: [PATCH] Add upload progress panel to web monitor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Upload progress bar with percentage, file count, speed, ETA - Detects active upload from upload_*.log files automatically - Last 10 upload lines shown with ✓/✗ color coding - Combined log panel shows both setup.log and upload logs - Upload folder distribution in API response Co-Authored-By: Claude Opus 4.6 (1M context) --- monitor.py | 119 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 111 insertions(+), 8 deletions(-) diff --git a/monitor.py b/monitor.py index 4b50387..72015fc 100755 --- a/monitor.py +++ b/monitor.py @@ -169,6 +169,51 @@ def collect_status(): api_ok = check_api(config) script_running = check_script_running() + # Upload progress from upload_progress.json + upload_total = len(uploaded) + upload_folders = {} + for info in uploaded.values(): + f = info.get("folder", "?") + upload_folders[f] = upload_folders.get(f, 0) + 1 + + # Detect active upload from log files (upload_cia_en.log or similar) + upload_active = None + upload_current = 0 + upload_target = 0 + upload_speed = None + upload_log_lines = [] + for logfile in sorted(Path(__file__).parent.glob("upload_*.log"), key=lambda p: p.stat().st_mtime, reverse=True): + try: + with open(logfile, "r", encoding="utf-8", errors="ignore") as f: + ulines = f.readlines() + upload_log_lines = [l.rstrip() for l in ulines[-10:]] + import re as _re + for line in reversed(ulines): + m = _re.search(r'\[(\d+)/(\d+)\]', line) + if m: + upload_current = int(m.group(1)) + upload_target = int(m.group(2)) + upload_active = logfile.name + break + # Speed: count ✓ lines with timestamps in last 20 + ok_times = [] + for line in ulines: + if '✓' in line or '✗' in line: + m2 = _re.search(r'\[(\d+)/\d+\]', line) + if m2: + ok_times.append(int(m2.group(1))) + if len(ok_times) >= 2 and upload_target > 0: + # Rough speed from file modification time + age = time.time() - logfile.stat().st_mtime + if age < 300 and upload_current > 0: # active in last 5 min + elapsed = time.time() - logfile.stat().st_ctime + if elapsed > 0 and upload_current > 0: + upload_speed = elapsed / upload_current # seconds per file + if upload_active: + break + except Exception: + pass + # Read last N lines from setup.log log_lines = [] batch_times = [] @@ -179,7 +224,6 @@ def collect_status(): log_lines = [l.rstrip() for l in all_lines[-15:]] # Parse batch timestamps to calculate ETA - # Format: "01:32:34 [INFO] ✓ arbiter batch 1/80: 5 embedded (5/396)" import re for line in all_lines: m = re.match(r'^(\d{2}:\d{2}:\d{2}) \[INFO\]\s+✓\s+\w+ batch \d+/\d+:', line) @@ -222,12 +266,28 @@ def collect_status(): else: eta_str = f"{minutes}m" + # Upload ETA + upload_eta = None + upload_remaining = max(0, upload_target - upload_current) if upload_target else 0 + if upload_speed and upload_remaining > 0: + eta_s = int(upload_remaining * upload_speed) + h, m = eta_s // 3600, (eta_s % 3600) // 60 + upload_eta = f"{h}h {m}m" if h > 0 else f"{m}m" + return { "personas": personas, "clusters": clusters, "total_uploaded": len(uploaded), "total_assigned": total_assigned, "total_expected": total_expected, + "upload_active": upload_active, + "upload_current": upload_current, + "upload_target": upload_target, + "upload_speed": round(upload_speed, 1) if upload_speed else None, + "upload_eta": upload_eta, + "upload_remaining": upload_remaining, + "upload_folders": dict(sorted(upload_folders.items(), key=lambda x: -x[1])[:10]), + "upload_log_lines": upload_log_lines, "total_personas": len(workspaces), "personas_with_vectors": sum(1 for p in personas if p["has_vectors"]), "lancedb_size_mb": dir_size_mb(LANCEDB_PATH), @@ -393,6 +453,17 @@ HTML_TEMPLATE = """ .summary-card .label { color: #565f89; font-size: 11px; text-transform: uppercase; } .summary-card .value { color: #7aa2f7; font-size: 20px; font-weight: bold; margin-top: 2px; } .summary-card .unit { color: #565f89; font-size: 12px; } + .upload-panel { background: #13131a; border: 1px solid #1a1b26; border-radius: 8px; padding: 12px 16px; margin-top: 16px; } + .upload-panel h3 { color: #e0af68; font-size: 13px; margin-bottom: 8px; } + .upload-bar { background: #1a1b26; border-radius: 3px; height: 20px; overflow: hidden; position: relative; margin: 8px 0; } + .upload-fill { height: 100%; border-radius: 3px; background: linear-gradient(90deg, #8a6d2e, #e0af68); transition: width 0.5s ease; } + .upload-pct { position: absolute; right: 8px; top: 2px; font-size: 12px; color: #fff; text-shadow: 0 0 4px rgba(0,0,0,0.8); } + .upload-stats { display: flex; gap: 16px; font-size: 12px; color: #565f89; margin-top: 4px; } + .upload-stats span { color: #e0af68; } + .upload-log { font-size: 11px; color: #565f89; margin-top: 8px; max-height: 120px; overflow-y: auto; } + .upload-log div { white-space: pre-wrap; line-height: 1.4; } + .upload-log .ok { color: #9ece6a; } + .upload-log .fail { color: #f7768e; } .log-panel { background: #0d0d12; border: 1px solid #1a1b26; border-radius: 8px; padding: 12px 16px; margin-top: 20px; max-height: 300px; overflow-y: auto; } .log-panel h3 { color: #565f89; font-size: 12px; text-transform: uppercase; margin-bottom: 8px; } .log-line { font-size: 12px; line-height: 1.6; color: #565f89; white-space: pre-wrap; word-break: break-all; } @@ -408,6 +479,7 @@ HTML_TEMPLATE = """
+

Log (setup.log)

No log data
@@ -464,16 +536,47 @@ function render(data) { }); document.getElementById('clusters').innerHTML = html; - // Log panel - const logLines = data.log_tail || []; - if (logLines.length > 0) { - let logHtml = ''; - logLines.forEach(line => { + // Upload panel + const upEl = document.getElementById('uploadpanel'); + if (data.upload_active && data.upload_target > 0) { + const upPct = Math.round(data.upload_current / data.upload_target * 100); + let upStats = `${data.upload_current} / ${data.upload_target} files`; + if (data.upload_speed) upStats += ` · ${data.upload_speed}s/file`; + if (data.upload_eta) upStats += ` · ETA ${data.upload_eta}`; + if (data.upload_remaining) upStats += ` · ${data.upload_remaining} remaining`; + + let upLog = ''; + (data.upload_log_lines || []).forEach(line => { let cls = ''; - if (line.includes('[ERROR]')) cls = 'error'; + if (line.includes('✓')) cls = 'ok'; + else if (line.includes('✗')) cls = 'fail'; + upLog += `
${line.replace(/`; + }); + + upEl.innerHTML = `
+

Upload: ${data.upload_active}

+
${upPct}%
+
${upStats}
+
${upLog}
+
`; + } else if (data.total_uploaded > 0) { + upEl.innerHTML = `

Upload: idle

Total uploaded: ${data.total_uploaded} files
`; + } else { + upEl.innerHTML = ''; + } + + // Log panel — show both setup.log and upload logs + const logLines = data.log_tail || []; + const uploadLogLines = data.upload_log_lines || []; + const allLogs = [...logLines, ...uploadLogLines.map(l => '[UPLOAD] ' + l)]; + if (allLogs.length > 0) { + let logHtml = ''; + allLogs.forEach(line => { + let cls = ''; + if (line.includes('[ERROR]') || line.includes('✗')) cls = 'error'; else if (line.includes('[WARNING]')) cls = 'warning'; else if (line.includes('✓')) cls = 'success'; - else if (line.includes('[INFO]')) cls = 'info'; + else if (line.includes('[INFO]') || line.includes('[UPLOAD]')) cls = 'info'; logHtml += `
${line.replace(/`; }); document.getElementById('loglines').innerHTML = logHtml;