Add upload progress panel to web monitor

- Upload progress bar with percentage, file count, speed, ETA
- Detects active upload from upload_*.log files automatically
- Last 10 upload lines shown with ✓/✗ color coding
- Combined log panel shows both setup.log and upload logs
- Upload folder distribution in API response

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

monitor.py

@@ -169,6 +169,51 @@ def collect_status():
     api_ok = check_api(config)
     script_running = check_script_running()
 
+    # Upload progress from upload_progress.json
+    upload_total = len(uploaded)
+    upload_folders = {}
+    for info in uploaded.values():
+        f = info.get("folder", "?")
+        upload_folders[f] = upload_folders.get(f, 0) + 1
+
+    # Detect active upload from log files (upload_cia_en.log or similar)
+    upload_active = None
+    upload_current = 0
+    upload_target = 0
+    upload_speed = None
+    upload_log_lines = []
+    for logfile in sorted(Path(__file__).parent.glob("upload_*.log"), key=lambda p: p.stat().st_mtime, reverse=True):
+        try:
+            with open(logfile, "r", encoding="utf-8", errors="ignore") as f:
+                ulines = f.readlines()
+            upload_log_lines = [l.rstrip() for l in ulines[-10:]]
+            import re as _re
+            for line in reversed(ulines):
+                m = _re.search(r'\[(\d+)/(\d+)\]', line)
+                if m:
+                    upload_current = int(m.group(1))
+                    upload_target = int(m.group(2))
+                    upload_active = logfile.name
+                    break
+            # Speed: count ✓ lines with timestamps in last 20
+            ok_times = []
+            for line in ulines:
+                if '✓' in line or '✗' in line:
+                    m2 = _re.search(r'\[(\d+)/\d+\]', line)
+                    if m2:
+                        ok_times.append(int(m2.group(1)))
+            if len(ok_times) >= 2 and upload_target > 0:
+                # Rough speed from file modification time
+                age = time.time() - logfile.stat().st_mtime
+                if age < 300 and upload_current > 0:  # active in last 5 min
+                    elapsed = time.time() - logfile.stat().st_ctime
+                    if elapsed > 0 and upload_current > 0:
+                        upload_speed = elapsed / upload_current  # seconds per file
+            if upload_active:
+                break
+        except Exception:
+            pass
+
     # Read last N lines from setup.log
     log_lines = []
     batch_times = []
@@ -179,7 +224,6 @@ def collect_status():
                 log_lines = [l.rstrip() for l in all_lines[-15:]]
 
             # Parse batch timestamps to calculate ETA
-            # Format: "01:32:34 [INFO]   ✓ arbiter batch 1/80: 5 embedded (5/396)"
             import re
             for line in all_lines:
                 m = re.match(r'^(\d{2}:\d{2}:\d{2}) \[INFO\]\s+✓\s+\w+ batch \d+/\d+:', line)
@@ -222,12 +266,28 @@ def collect_status():
         else:
             eta_str = f"{minutes}m"
 
+    # Upload ETA
+    upload_eta = None
+    upload_remaining = max(0, upload_target - upload_current) if upload_target else 0
+    if upload_speed and upload_remaining > 0:
+        eta_s = int(upload_remaining * upload_speed)
+        h, m = eta_s // 3600, (eta_s % 3600) // 60
+        upload_eta = f"{h}h {m}m" if h > 0 else f"{m}m"
+
     return {
         "personas": personas,
         "clusters": clusters,
         "total_uploaded": len(uploaded),
         "total_assigned": total_assigned,
         "total_expected": total_expected,
+        "upload_active": upload_active,
+        "upload_current": upload_current,
+        "upload_target": upload_target,
+        "upload_speed": round(upload_speed, 1) if upload_speed else None,
+        "upload_eta": upload_eta,
+        "upload_remaining": upload_remaining,
+        "upload_folders": dict(sorted(upload_folders.items(), key=lambda x: -x[1])[:10]),
+        "upload_log_lines": upload_log_lines,
         "total_personas": len(workspaces),
         "personas_with_vectors": sum(1 for p in personas if p["has_vectors"]),
         "lancedb_size_mb": dir_size_mb(LANCEDB_PATH),
@@ -393,6 +453,17 @@ HTML_TEMPLATE = """<!DOCTYPE html>
   .summary-card .label { color: #565f89; font-size: 11px; text-transform: uppercase; }
   .summary-card .value { color: #7aa2f7; font-size: 20px; font-weight: bold; margin-top: 2px; }
   .summary-card .unit { color: #565f89; font-size: 12px; }
+  .upload-panel { background: #13131a; border: 1px solid #1a1b26; border-radius: 8px; padding: 12px 16px; margin-top: 16px; }
+  .upload-panel h3 { color: #e0af68; font-size: 13px; margin-bottom: 8px; }
+  .upload-bar { background: #1a1b26; border-radius: 3px; height: 20px; overflow: hidden; position: relative; margin: 8px 0; }
+  .upload-fill { height: 100%; border-radius: 3px; background: linear-gradient(90deg, #8a6d2e, #e0af68); transition: width 0.5s ease; }
+  .upload-pct { position: absolute; right: 8px; top: 2px; font-size: 12px; color: #fff; text-shadow: 0 0 4px rgba(0,0,0,0.8); }
+  .upload-stats { display: flex; gap: 16px; font-size: 12px; color: #565f89; margin-top: 4px; }
+  .upload-stats span { color: #e0af68; }
+  .upload-log { font-size: 11px; color: #565f89; margin-top: 8px; max-height: 120px; overflow-y: auto; }
+  .upload-log div { white-space: pre-wrap; line-height: 1.4; }
+  .upload-log .ok { color: #9ece6a; }
+  .upload-log .fail { color: #f7768e; }
   .log-panel { background: #0d0d12; border: 1px solid #1a1b26; border-radius: 8px; padding: 12px 16px; margin-top: 20px; max-height: 300px; overflow-y: auto; }
   .log-panel h3 { color: #565f89; font-size: 12px; text-transform: uppercase; margin-bottom: 8px; }
   .log-line { font-size: 12px; line-height: 1.6; color: #565f89; white-space: pre-wrap; word-break: break-all; }
@@ -408,6 +479,7 @@ HTML_TEMPLATE = """<!DOCTYPE html>
 
 <div class="summary" id="summary"></div>
 <div class="status-bar" id="statusbar"></div>
+<div id="uploadpanel"></div>
 <div id="clusters"></div>
 <div class="log-panel" id="logpanel"><h3>Log (setup.log)</h3><div id="loglines">No log data</div></div>
 
@@ -464,16 +536,47 @@ function render(data) {
   });
   document.getElementById('clusters').innerHTML = html;
 
-  // Log panel
+  // Upload panel
-  const logLines = data.log_tail || [];
+  const upEl = document.getElementById('uploadpanel');
-  if (logLines.length > 0) {
+  if (data.upload_active && data.upload_target > 0) {
-    let logHtml = '';
+    const upPct = Math.round(data.upload_current / data.upload_target * 100);
-    logLines.forEach(line => {
+    let upStats = `<span>${data.upload_current}</span> / ${data.upload_target} files`;
+    if (data.upload_speed) upStats += ` · <span>${data.upload_speed}s</span>/file`;
+    if (data.upload_eta) upStats += ` · ETA <span>${data.upload_eta}</span>`;
+    if (data.upload_remaining) upStats += ` · ${data.upload_remaining} remaining`;
+
+    let upLog = '';
+    (data.upload_log_lines || []).forEach(line => {
       let cls = '';
-      if (line.includes('[ERROR]')) cls = 'error';
+      if (line.includes('✓')) cls = 'ok';
+      else if (line.includes('✗')) cls = 'fail';
+      upLog += `<div class="${cls}">${line.replace(/</g,'&lt;')}</div>`;
+    });
+
+    upEl.innerHTML = `<div class="upload-panel">
+      <h3>Upload: ${data.upload_active}</h3>
+      <div class="upload-bar"><div class="upload-fill" style="width:${upPct}%"></div><span class="upload-pct">${upPct}%</span></div>
+      <div class="upload-stats">${upStats}</div>
+      <div class="upload-log">${upLog}</div>
+    </div>`;
+  } else if (data.total_uploaded > 0) {
+    upEl.innerHTML = `<div class="upload-panel"><h3>Upload: idle</h3><div class="upload-stats">Total uploaded: <span>${data.total_uploaded}</span> files</div></div>`;
+  } else {
+    upEl.innerHTML = '';
+  }
+
+  // Log panel — show both setup.log and upload logs
+  const logLines = data.log_tail || [];
+  const uploadLogLines = data.upload_log_lines || [];
+  const allLogs = [...logLines, ...uploadLogLines.map(l => '[UPLOAD] ' + l)];
+  if (allLogs.length > 0) {
+    let logHtml = '';
+    allLogs.forEach(line => {
+      let cls = '';
+      if (line.includes('[ERROR]') || line.includes('✗')) cls = 'error';
       else if (line.includes('[WARNING]')) cls = 'warning';
       else if (line.includes('✓')) cls = 'success';
-      else if (line.includes('[INFO]')) cls = 'info';
+      else if (line.includes('[INFO]') || line.includes('[UPLOAD]')) cls = 'info';
       logHtml += `<div class="log-line ${cls}">${line.replace(/</g,'&lt;')}</div>`;
     });
     document.getElementById('loglines').innerHTML = logHtml;