From aec559d2aa969417facc5d2ddc041bcb62461060 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Sun, 5 Apr 2026 15:42:53 +0300 Subject: [PATCH] feat(05-01): migrate findings schema with verify_* columns - schema.sql: new findings columns verified, verify_status, verify_http_code, verify_metadata_json - db.go: migrateFindingsVerifyColumns runs on Open() for legacy DBs using PRAGMA table_info + ALTER TABLE - findings.go: Finding struct gains Verified/VerifyStatus/VerifyHTTPCode/VerifyMetadata - SaveFinding serializes verify metadata as JSON (NULL when nil) - ListFindings round-trips all verify fields --- pkg/storage/db.go | 55 +++++++++++++++++++++++++++++++++++++ pkg/storage/findings.go | 61 +++++++++++++++++++++++++++++++++++------ pkg/storage/schema.sql | 24 +++++++++------- 3 files changed, 121 insertions(+), 19 deletions(-) diff --git a/pkg/storage/db.go b/pkg/storage/db.go index 4bb905d..7875602 100644 --- a/pkg/storage/db.go +++ b/pkg/storage/db.go @@ -43,9 +43,64 @@ func Open(path string) (*DB, error) { return nil, fmt.Errorf("running schema migrations: %w", err) } + // Idempotent in-place migration for pre-Phase-5 databases that created + // the findings table before the verify_* columns existed. + if err := migrateFindingsVerifyColumns(sqlDB); err != nil { + sqlDB.Close() + return nil, fmt.Errorf("migrating findings verify columns: %w", err) + } + return &DB{sql: sqlDB}, nil } +// migrateFindingsVerifyColumns adds the Phase 5 verify_* columns to an +// existing findings table when they are missing. Uses PRAGMA table_info to +// detect the current column set (works on SQLite versions that lack +// ADD COLUMN IF NOT EXISTS). +func migrateFindingsVerifyColumns(sqlDB *sql.DB) error { + rows, err := sqlDB.Query("PRAGMA table_info(findings)") + if err != nil { + return fmt.Errorf("reading findings schema: %w", err) + } + existing := map[string]bool{} + for rows.Next() { + var cid int + var name, ctype string + var notnull, pk int + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { + rows.Close() + return fmt.Errorf("scanning findings schema row: %w", err) + } + existing[name] = true + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + rows.Close() + + type colDef struct { + name string + ddl string + } + wanted := []colDef{ + {"verified", "ALTER TABLE findings ADD COLUMN verified INTEGER NOT NULL DEFAULT 0"}, + {"verify_status", "ALTER TABLE findings ADD COLUMN verify_status TEXT NOT NULL DEFAULT ''"}, + {"verify_http_code", "ALTER TABLE findings ADD COLUMN verify_http_code INTEGER NOT NULL DEFAULT 0"}, + {"verify_metadata_json", "ALTER TABLE findings ADD COLUMN verify_metadata_json TEXT"}, + } + for _, c := range wanted { + if existing[c.name] { + continue + } + if _, err := sqlDB.Exec(c.ddl); err != nil { + return fmt.Errorf("adding column %s: %w", c.name, err) + } + } + return nil +} + // Close closes the underlying database connection. func (db *DB) Close() error { return db.sql.Close() diff --git a/pkg/storage/findings.go b/pkg/storage/findings.go index 172b700..7f5d3f2 100644 --- a/pkg/storage/findings.go +++ b/pkg/storage/findings.go @@ -2,6 +2,7 @@ package storage import ( "database/sql" + "encoding/json" "fmt" "time" ) @@ -19,6 +20,13 @@ type Finding struct { SourceType string LineNumber int CreatedAt time.Time + + // Verification fields populated by the Phase 5 verifier. Zero values mean + // the finding has not been verified. + Verified bool + VerifyStatus string // "live", "dead", "rate_limited", "error", "unknown" + VerifyHTTPCode int + VerifyMetadata map[string]string } // MaskKey returns the masked form of a key: first 8 chars + "..." + last 4 chars. @@ -51,10 +59,32 @@ func (db *DB) SaveFinding(f Finding, encKey []byte) (int64, error) { scanID = sql.NullInt64{} } + // Serialize verify metadata as JSON (NULL when nil) to match schema. + var metaJSON interface{} + if f.VerifyMetadata != nil { + b, err := json.Marshal(f.VerifyMetadata) + if err != nil { + return 0, fmt.Errorf("marshaling verify metadata: %w", err) + } + metaJSON = string(b) + } else { + metaJSON = sql.NullString{} + } + + verifiedInt := 0 + if f.Verified { + verifiedInt = 1 + } + res, err := db.sql.Exec( - `INSERT INTO findings (scan_id, provider_name, key_value, key_masked, confidence, source_path, source_type, line_number) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - scanID, f.ProviderName, encrypted, masked, f.Confidence, f.SourcePath, f.SourceType, f.LineNumber, + `INSERT INTO findings ( + scan_id, provider_name, key_value, key_masked, confidence, + source_path, source_type, line_number, + verified, verify_status, verify_http_code, verify_metadata_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + scanID, f.ProviderName, encrypted, masked, f.Confidence, + f.SourcePath, f.SourceType, f.LineNumber, + verifiedInt, f.VerifyStatus, f.VerifyHTTPCode, metaJSON, ) if err != nil { return 0, fmt.Errorf("inserting finding: %w", err) @@ -67,7 +97,9 @@ func (db *DB) SaveFinding(f Finding, encKey []byte) (int64, error) { func (db *DB) ListFindings(encKey []byte) ([]Finding, error) { rows, err := db.sql.Query( `SELECT id, scan_id, provider_name, key_value, key_masked, confidence, - source_path, source_type, line_number, created_at + source_path, source_type, line_number, + verified, verify_status, verify_http_code, verify_metadata_json, + created_at FROM findings ORDER BY created_at DESC`, ) if err != nil { @@ -81,15 +113,26 @@ func (db *DB) ListFindings(encKey []byte) ([]Finding, error) { var encrypted []byte var createdAt string var scanID sql.NullInt64 - err := rows.Scan( + var verifiedInt int + var metaJSON sql.NullString + if err := rows.Scan( &f.ID, &scanID, &f.ProviderName, &encrypted, &f.KeyMasked, - &f.Confidence, &f.SourcePath, &f.SourceType, &f.LineNumber, &createdAt, - ) + &f.Confidence, &f.SourcePath, &f.SourceType, &f.LineNumber, + &verifiedInt, &f.VerifyStatus, &f.VerifyHTTPCode, &metaJSON, + &createdAt, + ); err != nil { + return nil, fmt.Errorf("scanning finding row: %w", err) + } if scanID.Valid { f.ScanID = scanID.Int64 } - if err != nil { - return nil, fmt.Errorf("scanning finding row: %w", err) + f.Verified = verifiedInt != 0 + if metaJSON.Valid && metaJSON.String != "" { + m := map[string]string{} + if err := json.Unmarshal([]byte(metaJSON.String), &m); err != nil { + return nil, fmt.Errorf("unmarshaling verify metadata for finding %d: %w", f.ID, err) + } + f.VerifyMetadata = m } plain, err := Decrypt(encrypted, encKey) if err != nil { diff --git a/pkg/storage/schema.sql b/pkg/storage/schema.sql index 8817d8c..79b3a86 100644 --- a/pkg/storage/schema.sql +++ b/pkg/storage/schema.sql @@ -11,16 +11,20 @@ CREATE TABLE IF NOT EXISTS scans ( ); CREATE TABLE IF NOT EXISTS findings ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - scan_id INTEGER REFERENCES scans(id), - provider_name TEXT NOT NULL, - key_value BLOB NOT NULL, - key_masked TEXT NOT NULL, - confidence TEXT NOT NULL, - source_path TEXT, - source_type TEXT, - line_number INTEGER, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP + id INTEGER PRIMARY KEY AUTOINCREMENT, + scan_id INTEGER REFERENCES scans(id), + provider_name TEXT NOT NULL, + key_value BLOB NOT NULL, + key_masked TEXT NOT NULL, + confidence TEXT NOT NULL, + source_path TEXT, + source_type TEXT, + line_number INTEGER, + verified INTEGER NOT NULL DEFAULT 0, + verify_status TEXT NOT NULL DEFAULT '', + verify_http_code INTEGER NOT NULL DEFAULT 0, + verify_metadata_json TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP ); CREATE TABLE IF NOT EXISTS settings (