From 2ef54f71960c30f96a9efa1c67bd2a6f7c598464 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Sun, 5 Apr 2026 00:04:06 +0300 Subject: [PATCH 1/4] test(01-foundation-03): add failing tests for storage layer - Tests for AES-256-GCM encrypt/decrypt roundtrip - Tests for Argon2id key derivation determinism - Tests for SQLite open with schema tables - Tests for SaveFinding/ListFindings with encryption contract - Tests verify raw BLOB does not contain plaintext key --- go.mod | 22 +++++++ go.sum | 33 +++++++++++ pkg/storage/db_test.go | 127 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 pkg/storage/db_test.go diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..47c4011 --- /dev/null +++ b/go.mod @@ -0,0 +1,22 @@ +module github.com/salvacybersec/keyhunter + +go 1.26.1 + +require github.com/stretchr/testify v1.11.1 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/crypto v0.49.0 // indirect + golang.org/x/sys v0.42.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.70.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.48.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..26560fd --- /dev/null +++ b/go.sum @@ -0,0 +1,33 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.70.0 h1:U58NawXqXbgpZ/dcdS9kMshu08aiA6b7gusEusqzNkw= +modernc.org/libc v1.70.0/go.mod h1:OVmxFGP1CI/Z4L3E0Q3Mf1PDE0BucwMkcXjjLntvHJo= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.48.1 h1:S85iToyU6cgeojybE2XJlSbcsvcWkQ6qqNXJHtW5hWA= +modernc.org/sqlite v1.48.1/go.mod h1:hWjRO6Tj/5Ik8ieqxQybiEOUXy0NJFNp2tpvVpKlvig= diff --git a/pkg/storage/db_test.go b/pkg/storage/db_test.go new file mode 100644 index 0000000..d3b6114 --- /dev/null +++ b/pkg/storage/db_test.go @@ -0,0 +1,127 @@ +package storage_test + +import ( + "bytes" + "testing" + + "github.com/salvacybersec/keyhunter/pkg/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDBOpen(t *testing.T) { + db, err := storage.Open(":memory:") + require.NoError(t, err) + defer db.Close() + + // Verify schema tables exist + rows, err := db.SQL().Query("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name") + require.NoError(t, err) + defer rows.Close() + + var tables []string + for rows.Next() { + var name string + require.NoError(t, rows.Scan(&name)) + tables = append(tables, name) + } + assert.Contains(t, tables, "findings") + assert.Contains(t, tables, "scans") + assert.Contains(t, tables, "settings") +} + +func TestEncryptDecryptRoundtrip(t *testing.T) { + key := make([]byte, 32) // all-zero key for test + for i := range key { + key[i] = byte(i) + } + plaintext := []byte("sk-proj-supersecretapikey1234") + + ciphertext, err := storage.Encrypt(plaintext, key) + require.NoError(t, err) + assert.Greater(t, len(ciphertext), len(plaintext), "ciphertext should be longer than plaintext") + + recovered, err := storage.Decrypt(ciphertext, key) + require.NoError(t, err) + assert.Equal(t, plaintext, recovered) +} + +func TestEncryptNonDeterministic(t *testing.T) { + key := make([]byte, 32) + plain := []byte("test-key") + ct1, err1 := storage.Encrypt(plain, key) + ct2, err2 := storage.Encrypt(plain, key) + require.NoError(t, err1) + require.NoError(t, err2) + assert.NotEqual(t, ct1, ct2, "same plaintext encrypted twice should produce different ciphertext") +} + +func TestDecryptWrongKey(t *testing.T) { + key1 := make([]byte, 32) + key2 := make([]byte, 32) + key2[0] = 0xFF + + ct, err := storage.Encrypt([]byte("secret"), key1) + require.NoError(t, err) + + _, err = storage.Decrypt(ct, key2) + assert.Error(t, err, "decryption with wrong key should fail") +} + +func TestArgon2KeyDerivation(t *testing.T) { + passphrase := []byte("my-secure-passphrase") + salt := []byte("1234567890abcdef") // 16 bytes + + key1 := storage.DeriveKey(passphrase, salt) + key2 := storage.DeriveKey(passphrase, salt) + + assert.Equal(t, 32, len(key1), "derived key must be 32 bytes") + assert.Equal(t, key1, key2, "same passphrase+salt must produce same key") +} + +func TestNewSalt(t *testing.T) { + salt1, err1 := storage.NewSalt() + salt2, err2 := storage.NewSalt() + require.NoError(t, err1) + require.NoError(t, err2) + assert.Equal(t, 16, len(salt1)) + assert.NotEqual(t, salt1, salt2, "two salts should differ") +} + +func TestSaveFindingEncrypted(t *testing.T) { + db, err := storage.Open(":memory:") + require.NoError(t, err) + defer db.Close() + + // Derive a test key + key := storage.DeriveKey([]byte("testpassphrase"), []byte("testsalt1234xxxx")) + + plainKey := "sk-proj-test1234567890abcdefghijklmnopqr" + f := storage.Finding{ + ProviderName: "openai", + KeyValue: plainKey, + Confidence: "high", + SourcePath: "/test/file.env", + SourceType: "file", + LineNumber: 42, + } + + id, err := db.SaveFinding(f, key) + require.NoError(t, err) + assert.Greater(t, id, int64(0)) + + findings, err := db.ListFindings(key) + require.NoError(t, err) + require.Len(t, findings, 1) + assert.Equal(t, plainKey, findings[0].KeyValue) + assert.Equal(t, "openai", findings[0].ProviderName) + // Verify masking + assert.Equal(t, "sk-proj-...opqr", findings[0].KeyMasked) + + // Verify encryption contract: raw BLOB bytes in the database must NOT contain the plaintext key. + // This confirms Encrypt() was called before INSERT, not that the key was stored verbatim. + var rawBlob []byte + require.NoError(t, db.SQL().QueryRow("SELECT key_value FROM findings WHERE id = ?", id).Scan(&rawBlob)) + assert.False(t, bytes.Contains(rawBlob, []byte(plainKey)), + "raw database BLOB must not contain plaintext key — encryption was not applied") +} From 239e2c214c698e776326f74899f3c2f7eb95f812 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Sun, 5 Apr 2026 00:04:33 +0300 Subject: [PATCH 2/4] feat(01-foundation-03): implement AES-256-GCM encryption and Argon2id key derivation - Encrypt/Decrypt using AES-256-GCM with random nonce prepended to ciphertext - ErrCiphertextTooShort sentinel error for malformed ciphertext - DeriveKey using Argon2id RFC 9106 params (time=1, mem=64MB, threads=4, keyLen=32) - NewSalt generates cryptographically random 16-byte salt --- pkg/storage/crypto.go | 32 ++++++++++++++++++++++++++ pkg/storage/encrypt.go | 52 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 pkg/storage/crypto.go create mode 100644 pkg/storage/encrypt.go diff --git a/pkg/storage/crypto.go b/pkg/storage/crypto.go new file mode 100644 index 0000000..fd1d5a4 --- /dev/null +++ b/pkg/storage/crypto.go @@ -0,0 +1,32 @@ +package storage + +import ( + "crypto/rand" + + "golang.org/x/crypto/argon2" +) + +const ( + argon2Time uint32 = 1 + argon2Memory uint32 = 64 * 1024 // 64 MB — RFC 9106 Section 7.3 + argon2Threads uint8 = 4 + argon2KeyLen uint32 = 32 // AES-256 key length + saltSize = 16 +) + +// DeriveKey produces a 32-byte AES-256 key from a passphrase and salt using Argon2id. +// Uses RFC 9106 Section 7.3 recommended parameters. +// Given the same passphrase and salt, always returns the same key. +func DeriveKey(passphrase []byte, salt []byte) []byte { + return argon2.IDKey(passphrase, salt, argon2Time, argon2Memory, argon2Threads, argon2KeyLen) +} + +// NewSalt generates a cryptographically random 16-byte salt. +// Store alongside the database and reuse on each key derivation. +func NewSalt() ([]byte, error) { + salt := make([]byte, saltSize) + if _, err := rand.Read(salt); err != nil { + return nil, err + } + return salt, nil +} diff --git a/pkg/storage/encrypt.go b/pkg/storage/encrypt.go new file mode 100644 index 0000000..fec367d --- /dev/null +++ b/pkg/storage/encrypt.go @@ -0,0 +1,52 @@ +package storage + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "errors" + "io" +) + +// ErrCiphertextTooShort is returned when ciphertext is shorter than the GCM nonce size. +var ErrCiphertextTooShort = errors.New("ciphertext too short") + +// Encrypt encrypts plaintext using AES-256-GCM with a random nonce. +// The nonce is prepended to the returned ciphertext. +// key must be exactly 32 bytes (AES-256). +func Encrypt(plaintext []byte, key []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + // Seal appends encrypted data to nonce, so nonce is prepended + ciphertext := gcm.Seal(nonce, nonce, plaintext, nil) + return ciphertext, nil +} + +// Decrypt decrypts ciphertext produced by Encrypt. +// Expects the nonce to be prepended to the ciphertext. +func Decrypt(ciphertext []byte, key []byte) ([]byte, error) { + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + nonceSize := gcm.NonceSize() + if len(ciphertext) < nonceSize { + return nil, ErrCiphertextTooShort + } + nonce, ciphertext := ciphertext[:nonceSize], ciphertext[nonceSize:] + return gcm.Open(nil, nonce, ciphertext, nil) +} From 3334633867a2b3d9114950fe2bb49c2d086874c9 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Sun, 5 Apr 2026 00:05:54 +0300 Subject: [PATCH 3/4] feat(01-foundation-03): implement SQLite storage with Finding CRUD and encryption - schema.sql: CREATE TABLE for findings, scans, settings with indexes - db.go: Open() with WAL mode, foreign keys, embedded schema migration - findings.go: SaveFinding encrypts key_value before INSERT, ListFindings decrypts after SELECT - MaskKey: first8...last4 masking helper - Fix: NULL scan_id handling for findings without parent scan --- pkg/storage/db.go | 57 ++++++++++++++++++++++ pkg/storage/findings.go | 103 ++++++++++++++++++++++++++++++++++++++++ pkg/storage/schema.sql | 35 ++++++++++++++ 3 files changed, 195 insertions(+) create mode 100644 pkg/storage/db.go create mode 100644 pkg/storage/findings.go create mode 100644 pkg/storage/schema.sql diff --git a/pkg/storage/db.go b/pkg/storage/db.go new file mode 100644 index 0000000..4bb905d --- /dev/null +++ b/pkg/storage/db.go @@ -0,0 +1,57 @@ +package storage + +import ( + "database/sql" + _ "embed" + "fmt" + + _ "modernc.org/sqlite" +) + +//go:embed schema.sql +var schemaSQLBytes []byte + +// DB wraps the sql.DB connection with KeyHunter-specific behavior. +type DB struct { + sql *sql.DB +} + +// Open opens or creates a SQLite database at path, runs embedded schema migrations, +// and enables WAL mode for better concurrent read performance. +// Use ":memory:" for tests. +func Open(path string) (*DB, error) { + sqlDB, err := sql.Open("sqlite", path) + if err != nil { + return nil, fmt.Errorf("opening database: %w", err) + } + + // Enable WAL mode for concurrent reads + if _, err := sqlDB.Exec("PRAGMA journal_mode=WAL"); err != nil { + sqlDB.Close() + return nil, fmt.Errorf("enabling WAL mode: %w", err) + } + + // Enable foreign keys + if _, err := sqlDB.Exec("PRAGMA foreign_keys=ON"); err != nil { + sqlDB.Close() + return nil, fmt.Errorf("enabling foreign keys: %w", err) + } + + // Run schema migrations + if _, err := sqlDB.Exec(string(schemaSQLBytes)); err != nil { + sqlDB.Close() + return nil, fmt.Errorf("running schema migrations: %w", err) + } + + return &DB{sql: sqlDB}, nil +} + +// Close closes the underlying database connection. +func (db *DB) Close() error { + return db.sql.Close() +} + +// SQL returns the underlying sql.DB for advanced use cases. +func (db *DB) SQL() *sql.DB { + return db.sql +} diff --git a/pkg/storage/findings.go b/pkg/storage/findings.go new file mode 100644 index 0000000..172b700 --- /dev/null +++ b/pkg/storage/findings.go @@ -0,0 +1,103 @@ +package storage + +import ( + "database/sql" + "fmt" + "time" +) + +// Finding represents a detected API key with metadata. +// KeyValue is always plaintext in this struct — encryption happens at the storage boundary. +type Finding struct { + ID int64 + ScanID int64 + ProviderName string + KeyValue string // plaintext — encrypted before storage, decrypted after retrieval + KeyMasked string // first8...last4, stored plaintext + Confidence string + SourcePath string + SourceType string + LineNumber int + CreatedAt time.Time +} + +// MaskKey returns the masked form of a key: first 8 chars + "..." + last 4 chars. +// If the key is too short (< 12 chars), returns the full key masked with asterisks. +func MaskKey(key string) string { + if len(key) < 12 { + return "****" + } + return key[:8] + "..." + key[len(key)-4:] +} + +// SaveFinding encrypts the finding's KeyValue and persists the finding to the database. +// encKey must be a 32-byte AES-256 key (from DeriveKey). +func (db *DB) SaveFinding(f Finding, encKey []byte) (int64, error) { + encrypted, err := Encrypt([]byte(f.KeyValue), encKey) + if err != nil { + return 0, fmt.Errorf("encrypting key value: %w", err) + } + + masked := f.KeyMasked + if masked == "" { + masked = MaskKey(f.KeyValue) + } + + // Use NULL for scan_id when not set (zero value) to satisfy FK constraint + var scanID interface{} + if f.ScanID != 0 { + scanID = f.ScanID + } else { + scanID = sql.NullInt64{} + } + + res, err := db.sql.Exec( + `INSERT INTO findings (scan_id, provider_name, key_value, key_masked, confidence, source_path, source_type, line_number) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + scanID, f.ProviderName, encrypted, masked, f.Confidence, f.SourcePath, f.SourceType, f.LineNumber, + ) + if err != nil { + return 0, fmt.Errorf("inserting finding: %w", err) + } + return res.LastInsertId() +} + +// ListFindings retrieves all findings, decrypting key values using encKey. +// encKey must be the same 32-byte key used during SaveFinding. +func (db *DB) ListFindings(encKey []byte) ([]Finding, error) { + rows, err := db.sql.Query( + `SELECT id, scan_id, provider_name, key_value, key_masked, confidence, + source_path, source_type, line_number, created_at + FROM findings ORDER BY created_at DESC`, + ) + if err != nil { + return nil, fmt.Errorf("querying findings: %w", err) + } + defer rows.Close() + + var findings []Finding + for rows.Next() { + var f Finding + var encrypted []byte + var createdAt string + var scanID sql.NullInt64 + err := rows.Scan( + &f.ID, &scanID, &f.ProviderName, &encrypted, &f.KeyMasked, + &f.Confidence, &f.SourcePath, &f.SourceType, &f.LineNumber, &createdAt, + ) + if scanID.Valid { + f.ScanID = scanID.Int64 + } + if err != nil { + return nil, fmt.Errorf("scanning finding row: %w", err) + } + plain, err := Decrypt(encrypted, encKey) + if err != nil { + return nil, fmt.Errorf("decrypting finding %d: %w", f.ID, err) + } + f.KeyValue = string(plain) + f.CreatedAt, _ = time.Parse("2006-01-02 15:04:05", createdAt) + findings = append(findings, f) + } + return findings, rows.Err() +} diff --git a/pkg/storage/schema.sql b/pkg/storage/schema.sql new file mode 100644 index 0000000..8817d8c --- /dev/null +++ b/pkg/storage/schema.sql @@ -0,0 +1,35 @@ +-- KeyHunter database schema +-- Version: 1 + +CREATE TABLE IF NOT EXISTS scans ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + started_at DATETIME NOT NULL, + finished_at DATETIME, + source_path TEXT, + finding_count INTEGER DEFAULT 0, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS findings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + scan_id INTEGER REFERENCES scans(id), + provider_name TEXT NOT NULL, + key_value BLOB NOT NULL, + key_masked TEXT NOT NULL, + confidence TEXT NOT NULL, + source_path TEXT, + source_type TEXT, + line_number INTEGER, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- Indexes for common queries +CREATE INDEX IF NOT EXISTS idx_findings_scan_id ON findings(scan_id); +CREATE INDEX IF NOT EXISTS idx_findings_provider ON findings(provider_name); +CREATE INDEX IF NOT EXISTS idx_findings_created ON findings(created_at DESC); From 43aeb8985d722d20027e57c3305f57eee16a7691 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Sun, 5 Apr 2026 00:07:24 +0300 Subject: [PATCH 4/4] =?UTF-8?q?docs(01-foundation-03):=20complete=20storag?= =?UTF-8?q?e=20layer=20plan=20=E2=80=94=20SUMMARY,=20STATE,=20ROADMAP,=20R?= =?UTF-8?q?EQUIREMENTS=20updated?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 01-03-SUMMARY.md: AES-256-GCM + Argon2id + SQLite CRUD layer complete - STATE.md: progress 20%, decisions logged, session updated - ROADMAP.md: Phase 1 In Progress (1/5 summaries) - REQUIREMENTS.md: STOR-01, STOR-02, STOR-03 marked complete --- .planning/REQUIREMENTS.md | 6 +- .planning/ROADMAP.md | 4 +- .planning/STATE.md | 27 +++- .../phases/01-foundation/01-03-SUMMARY.md | 139 ++++++++++++++++++ 4 files changed, 168 insertions(+), 8 deletions(-) create mode 100644 .planning/phases/01-foundation/01-03-SUMMARY.md diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md index 32296ba..262ad00 100644 --- a/.planning/REQUIREMENTS.md +++ b/.planning/REQUIREMENTS.md @@ -74,9 +74,9 @@ Requirements for initial release. Each maps to roadmap phases. ### Storage -- [ ] **STOR-01**: SQLite database for persisting scan results, keys, recon history -- [ ] **STOR-02**: Application-level AES-256 encryption for stored keys and sensitive config -- [ ] **STOR-03**: Encryption key derived from user passphrase via Argon2 +- [x] **STOR-01**: SQLite database for persisting scan results, keys, recon history +- [x] **STOR-02**: Application-level AES-256 encryption for stored keys and sensitive config +- [x] **STOR-03**: Encryption key derived from user passphrase via Argon2 ### CLI diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 53c9549..ccd920a 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -48,7 +48,7 @@ Decimal phases appear between their surrounding integers in numeric order. Plans: - [ ] 01-01-PLAN.md — Go module init, dependency installation, test scaffolding and testdata fixtures - [ ] 01-02-PLAN.md — Provider registry: YAML schema, embed loader, Aho-Corasick automaton, Registry struct -- [ ] 01-03-PLAN.md — Storage layer: AES-256-GCM encryption, Argon2id key derivation, SQLite + Finding CRUD +- [x] 01-03-PLAN.md — Storage layer: AES-256-GCM encryption, Argon2id key derivation, SQLite + Finding CRUD - [ ] 01-04-PLAN.md — Scan engine pipeline: keyword pre-filter, regex+entropy detector, FileSource, ants worker pool - [ ] 01-05-PLAN.md — CLI wiring: scan, providers list/info/stats, config init/set/get, output table @@ -255,7 +255,7 @@ Phases execute in numeric order: 1 → 2 → 3 → ... → 18 | Phase | Plans Complete | Status | Completed | |-------|----------------|--------|-----------| -| 1. Foundation | 0/5 | Planning complete | - | +| 1. Foundation | 1/5 | In Progress| | | 2. Tier 1-2 Providers | 0/? | Not started | - | | 3. Tier 3-9 Providers | 0/? | Not started | - | | 4. Input Sources | 0/? | Not started | - | diff --git a/.planning/STATE.md b/.planning/STATE.md index cde2084..2f9bdb8 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -1,3 +1,19 @@ +--- +gsd_state_version: 1.0 +milestone: v1.0 +milestone_name: milestone +status: planning +stopped_at: Completed 01-foundation-03-PLAN.md +last_updated: "2026-04-04T21:07:04.658Z" +last_activity: 2026-04-04 — Roadmap created, 18 phases defined covering 146 v1 requirements +progress: + total_phases: 18 + completed_phases: 0 + total_plans: 5 + completed_plans: 1 + percent: 20 +--- + # Project State ## Project Reference @@ -14,11 +30,12 @@ Plan: 0 of ? in current phase Status: Ready to plan Last activity: 2026-04-04 — Roadmap created, 18 phases defined covering 146 v1 requirements -Progress: [░░░░░░░░░░░░░░░░░░░░] 0% +Progress: [██░░░░░░░░] 20% ## Performance Metrics **Velocity:** + - Total plans completed: 0 - Average duration: — - Total execution time: 0 hours @@ -30,10 +47,12 @@ Progress: [░░░░░░░░░░░░░░░░░░░░] 0% | - | - | - | - | **Recent Trend:** + - Last 5 plans: — - Trend: — *Updated after each plan completion* +| Phase 01-foundation P03 | 3 | 2 tasks | 7 files | ## Accumulated Context @@ -46,6 +65,8 @@ Recent decisions affecting current work: - Roadmap: Per-source rate limiter architecture (Phase 9) must precede all OSINT source modules (Phases 10-16) - Roadmap: AES-256 encryption added in Phase 1, not post-hoc — avoids migration complexity - Roadmap: Verification (Phase 5) requires consent prompt + LEGAL.md — not optional polish +- [Phase 01-foundation]: Storage 01-03: Argon2id selected over PBKDF2 — memory-hard RFC 9106 params, resolves STATE.md blocker +- [Phase 01-foundation]: Storage 01-03: AES-256-GCM nonce prepended to ciphertext in single BLOB column — no separate nonce column needed ### Pending Todos @@ -60,6 +81,6 @@ None yet. ## Session Continuity -Last session: 2026-04-04 -Stopped at: Roadmap written to .planning/ROADMAP.md; ready to begin Phase 1 planning +Last session: 2026-04-04T21:07:04.654Z +Stopped at: Completed 01-foundation-03-PLAN.md Resume file: None diff --git a/.planning/phases/01-foundation/01-03-SUMMARY.md b/.planning/phases/01-foundation/01-03-SUMMARY.md new file mode 100644 index 0000000..5418f34 --- /dev/null +++ b/.planning/phases/01-foundation/01-03-SUMMARY.md @@ -0,0 +1,139 @@ +--- +phase: 01-foundation +plan: 03 +subsystem: database +tags: [sqlite, aes-256-gcm, argon2id, encryption, storage, modernc-sqlite] + +# Dependency graph +requires: + - phase: 01-foundation-01 + provides: go.mod with modernc.org/sqlite and golang.org/x/crypto dependencies + +provides: + - AES-256-GCM column encryption (Encrypt/Decrypt) with random nonce prepended + - Argon2id key derivation (DeriveKey/NewSalt) using RFC 9106 parameters + - SQLite database Open() with WAL mode and embedded schema migration + - Finding CRUD: SaveFinding encrypts key_value at boundary, ListFindings decrypts transparently + - MaskKey helper: first8...last4 display format + - schema.sql with findings, scans, settings tables and performance indexes + +affects: [01-04-scanner, 01-05-cli, 17-dashboard, 18-telegram] + +# Tech tracking +tech-stack: + added: + - modernc.org/sqlite v1.48.1 (pure Go SQLite, CGO-free) + - golang.org/x/crypto (argon2.IDKey for key derivation) + - crypto/aes + crypto/cipher (stdlib AES-256-GCM) + patterns: + - "Encrypt-at-boundary: SaveFinding encrypts, ListFindings decrypts — storage layer handles all crypto transparently" + - "go:embed schema.sql — schema migrated on Open(), idempotent via CREATE TABLE IF NOT EXISTS" + - "WAL mode enabled on every Open() for concurrent read performance" + - "NULL scan_id: zero-value ScanID stored as SQL NULL to satisfy FK constraint" + +key-files: + created: + - pkg/storage/encrypt.go + - pkg/storage/crypto.go + - pkg/storage/db.go + - pkg/storage/findings.go + - pkg/storage/schema.sql + - pkg/storage/db_test.go + modified: [] + +key-decisions: + - "Argon2id over PBKDF2: RFC 9106 recommended, memory-hard, resolves blocker from STATE.md" + - "NULL scan_id for findings without parent scan — FK constraint satisfied without mandatory scan creation" + - "Nonce prepended to ciphertext in single []byte — simplifies storage (no separate column needed)" + - "MaskKey returns first8...last4 — consistent with plan spec, 12-char minimum before masking" + +patterns-established: + - "Pattern: Encrypt-at-boundary — pkg/storage is the only layer that sees encrypted bytes" + - "Pattern: sql.NullInt64 for nullable FK columns in scan results" + - "Pattern: go:embed for all embedded assets — schema.sql embedded in db.go" + +requirements-completed: [STOR-01, STOR-02, STOR-03] + +# Metrics +duration: 3min +completed: 2026-04-04 +--- + +# Phase 1 Plan 3: Storage Layer Summary + +**AES-256-GCM column encryption with Argon2id key derivation and SQLite CRUD — raw BLOB verified to contain no plaintext key data** + +## Performance + +- **Duration:** 3 min +- **Started:** 2026-04-04T21:02:00Z +- **Completed:** 2026-04-04T21:06:06Z +- **Tasks:** 2 +- **Files modified:** 7 (5 created, go.mod + go.sum updated) + +## Accomplishments + +- AES-256-GCM Encrypt/Decrypt with prepended random nonce — non-deterministic, wrong-key fails with GCM auth error +- Argon2id DeriveKey using RFC 9106 Section 7.3 params (time=1, memory=64MB, threads=4, keyLen=32) — resolves the Argon2 vs PBKDF2 blocker from STATE.md +- SQLite opens with WAL mode, foreign keys, and embedded schema.sql migration — works with `:memory:` for tests +- SaveFinding/ListFindings transparently encrypt/decrypt key_value at storage boundary +- All 7 tests pass including raw-BLOB assertion confirming plaintext is not stored + +## Task Commits + +Each task was committed atomically: + +1. **TDD RED: Failing test suite** - `2ef54f7` (test) +2. **Task 1: AES-256-GCM + Argon2id** - `239e2c2` (feat) +3. **Task 2: SQLite DB + schema + CRUD** - `3334633` (feat) + +## Files Created/Modified + +- `pkg/storage/encrypt.go` - Encrypt(plaintext, key) and Decrypt(ciphertext, key) using AES-256-GCM +- `pkg/storage/crypto.go` - DeriveKey(passphrase, salt) using Argon2id RFC 9106, NewSalt() 16-byte random +- `pkg/storage/db.go` - DB struct with Open(), Close(), SQL() — WAL mode, FK, embedded schema migration +- `pkg/storage/findings.go` - Finding struct, SaveFinding, ListFindings, MaskKey helper +- `pkg/storage/schema.sql` - CREATE TABLE for findings, scans, settings + 3 indexes +- `pkg/storage/db_test.go` - 7 tests including raw-BLOB encryption verification + +## Decisions Made + +- Argon2id selected over PBKDF2 (resolves STATE.md blocker) — memory-hard, RFC 9106 recommended +- NULL scan_id: zero-value ScanID stored as SQL NULL so findings can exist without a parent scan +- Single []byte for nonce+ciphertext — no separate nonce column needed, simplifies schema + +## Deviations from Plan + +### Auto-fixed Issues + +**1. [Rule 1 - Bug] FK constraint failed when ScanID = 0** +- **Found during:** Task 2 (TestSaveFindingEncrypted) +- **Issue:** Go zero value `ScanID: 0` sent as integer 0 to SQLite, failing FK constraint (no scan with id=0) +- **Fix:** SaveFinding converts zero ScanID to sql.NullInt64{} (NULL), ListFindings uses sql.NullInt64 for scan +- **Files modified:** pkg/storage/findings.go +- **Verification:** TestSaveFindingEncrypted passes after fix +- **Committed in:** 3334633 (Task 2 commit) + +--- + +**Total deviations:** 1 auto-fixed (Rule 1 - Bug) +**Impact on plan:** Necessary correctness fix — plan spec allows NULL scan_id (no NOT NULL in schema). No scope creep. + +## Issues Encountered + +None beyond the FK constraint bug documented above. + +## User Setup Required + +None - no external service configuration required. + +## Next Phase Readiness + +- Storage layer fully functional with transparent encryption +- pkg/storage exports: Encrypt, Decrypt, DeriveKey, NewSalt, Open, DB, Finding, SaveFinding, ListFindings, MaskKey +- Scanner (Plan 04) can call SaveFinding to persist findings +- CLI (Plan 05) can call ListFindings to display findings + +--- +*Phase: 01-foundation* +*Completed: 2026-04-04*