fix(01-foundation): address all checker blockers and warnings in phase plans

2026-04-04 23:57:01 +03:00
parent 684b67cb73
commit fb8a1f002b
4 changed files with 337 additions and 154 deletions
--- a/.planning/phases/01-foundation/01-05-PLAN.md
+++ b/.planning/phases/01-foundation/01-05-PLAN.md
@@ -9,9 +9,10 @@ files_modified:
  - cmd/scan.go
  - cmd/providers.go
  - cmd/config.go
+  - cmd/stubs.go
  - pkg/config/config.go
  - pkg/output/table.go
-autonomous: false
+autonomous: true
 requirements: [CLI-01, CLI-02, CLI-03, CLI-04, CLI-05]

 must_haves:
@@ -21,14 +22,19 @@ must_haves:
    - "`keyhunter providers info openai` prints OpenAI provider details"
    - "`keyhunter config init` creates ~/.keyhunter.yaml without error"
    - "`keyhunter config set workers 16` persists the value to ~/.keyhunter.yaml"
-    - "`keyhunter --help` shows all top-level commands: scan, providers, config"
+    - "`keyhunter --help` shows all top-level commands: scan, verify, import, recon, keys, serve, dorks, hook, schedule, providers, config"
+    - "Findings are stored with a per-installation salt loaded from the settings table — not a hardcoded salt"
+    - "Raw sqlite3 query on the database file does NOT return plaintext key values"
  artifacts:
    - path: "cmd/root.go"
      provides: "Cobra root command with PersistentPreRunE config loading"
      contains: "cobra.Command"
    - path: "cmd/scan.go"
-      provides: "scan command wiring Engine + FileSource + output table"
+      provides: "scan command wiring Engine + FileSource + output table + salt from settings"
      exports: ["scanCmd"]
+    - path: "cmd/stubs.go"
+      provides: "stub commands for verify, import, recon, keys, serve, dorks, hook, schedule"
+      exports: ["verifyCmd", "importCmd", "reconCmd", "keysCmd", "serveCmd", "dorksCmd", "hookCmd", "scheduleCmd"]
    - path: "cmd/providers.go"
      provides: "providers list/info/stats subcommands using Registry"
      exports: ["providersCmd"]
@@ -50,6 +56,10 @@ must_haves:
      to: "pkg/storage/db.go"
      via: "storage.Open() called, SaveFinding for each result"
      pattern: "storage\\.Open"
+    - from: "cmd/scan.go"
+      to: "pkg/storage/crypto.go"
+      via: "loadOrCreateSalt() reads salt from settings table via storage.GetSetting/SetSetting, then calls storage.DeriveKey"
+      pattern: "DeriveKey|GetSetting|SetSetting"
    - from: "cmd/root.go"
      to: "github.com/spf13/viper"
      via: "viper.SetConfigFile in PersistentPreRunE"
@@ -61,10 +71,10 @@ must_haves:
 ---

 <objective>
-Wire all subsystems together through the Cobra CLI: scan command (engine + storage + output), providers list/info/stats commands, and config init/set/get commands. This is the integration layer — all business logic lives in pkg/, cmd/ only wires.
+Wire all subsystems together through the Cobra CLI: scan command (engine + storage + output), providers list/info/stats commands, config init/set/get commands, and 8 stub commands for future phases. This is the integration layer — all business logic lives in pkg/, cmd/ only wires.

 Purpose: Satisfies all Phase 1 CLI requirements and delivers the first working `keyhunter scan` command that completes the end-to-end success criteria.
-Output: cmd/{root,scan,providers,config}.go, pkg/config/config.go, pkg/output/table.go.
+Output: cmd/{root,scan,providers,config,stubs}.go, pkg/config/config.go, pkg/output/table.go.
 </objective>

 <execution_context>
@@ -106,6 +116,10 @@ func (db *DB) SaveFinding(f Finding, encKey []byte) (int64, error)
 func DeriveKey(passphrase []byte, salt []byte) []byte
 func NewSalt() ([]byte, error)

+<!-- Storage settings CRUD — must be added to pkg/storage/db.go in this plan -->
+func (db *DB) GetSetting(key string) (string, bool, error)
+func (db *DB) SetSetting(key string, value string) error
+
 <!-- Registry (from Plan 02) -->
 package providers
 func NewRegistry() (*Registry, error)
@@ -127,17 +141,24 @@ Passphrase:  (prompt if not in env KEYHUNTER_PASSPHRASE — Phase 1: use empty s
 <!-- lipgloss table output -->
 Columns: PROVIDER | MASKED KEY | CONFIDENCE | SOURCE | LINE
 Colors: use lipgloss.NewStyle().Foreground() for confidence: high=green, medium=yellow, low=red
+
+<!-- Salt wiring strategy (BLOCKER 4 fix) -->
+On first scan, call storage.NewSalt(), hex-encode it, store in settings table with key "encryption.salt".
+On subsequent scans, read the salt from the settings table.
+This ensures all users have a unique per-installation salt instead of a shared hardcoded salt.
+The helper function loadOrCreateSalt(db *storage.DB) ([]byte, error) handles both cases.
 </interfaces>
 </context>

 <tasks>

 <task type="auto" tdd="false">
-  <name>Task 1: Config package, output table, and root command</name>
-  <files>pkg/config/config.go, pkg/output/table.go, cmd/root.go</files>
+  <name>Task 1: Config package, output table, root command, and settings helpers</name>
+  <files>pkg/config/config.go, pkg/output/table.go, cmd/root.go, pkg/storage/settings.go</files>
  <read_first>
    - /home/salva/Documents/apikey/.planning/phases/01-foundation/01-RESEARCH.md (CLI-01, CLI-02, CLI-03 rows, Standard Stack: cobra v1.10.2 + viper v1.21.0)
    - /home/salva/Documents/apikey/pkg/engine/finding.go (Finding struct fields for output)
+    - /home/salva/Documents/apikey/pkg/storage/db.go (DB struct, to add GetSetting/SetSetting)
  </read_first>
  <action>
 Create **pkg/config/config.go**:
@@ -172,6 +193,43 @@ func Load() Config {
 }
 ```

+Create **pkg/storage/settings.go** — adds GetSetting/SetSetting to the storage package:
+```go
+package storage
+
+import (
+    "database/sql"
+    "fmt"
+)
+
+// GetSetting retrieves a value from the settings table.
+// Returns (value, true, nil) if found, ("", false, nil) if not found, ("", false, err) on error.
+func (db *DB) GetSetting(key string) (string, bool, error) {
+    var value string
+    err := db.sql.QueryRow("SELECT value FROM settings WHERE key = ?", key).Scan(&value)
+    if err == sql.ErrNoRows {
+        return "", false, nil
+    }
+    if err != nil {
+        return "", false, fmt.Errorf("getting setting %q: %w", key, err)
+    }
+    return value, true, nil
+}
+
+// SetSetting inserts or updates a key-value pair in the settings table.
+func (db *DB) SetSetting(key, value string) error {
+    _, err := db.sql.Exec(
+        `INSERT INTO settings (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)
+         ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = CURRENT_TIMESTAMP`,
+        key, value,
+    )
+    if err != nil {
+        return fmt.Errorf("setting %q: %w", key, err)
+    }
+    return nil
+}
+```
+
 Create **pkg/output/table.go**:
 ```go
 package output
@@ -281,6 +339,15 @@ func init() {
    rootCmd.AddCommand(scanCmd)
    rootCmd.AddCommand(providersCmd)
    rootCmd.AddCommand(configCmd)
+    // Stub commands for future phases (per CLI-01 requirement of 11 commands)
+    rootCmd.AddCommand(verifyCmd)
+    rootCmd.AddCommand(importCmd)
+    rootCmd.AddCommand(reconCmd)
+    rootCmd.AddCommand(keysCmd)
+    rootCmd.AddCommand(serveCmd)
+    rootCmd.AddCommand(dorksCmd)
+    rootCmd.AddCommand(hookCmd)
+    rootCmd.AddCommand(scheduleCmd)
 }

 func initConfig() {
@@ -316,28 +383,30 @@ func mustHomeDir() string {
 ```
  </action>
  <verify>
-    <automated>cd /home/salva/Documents/apikey && go build ./... && ./keyhunter --help 2>&1 | grep -E "scan|providers|config" && echo "HELP OK"</automated>
+    <automated>cd /home/salva/Documents/apikey && go build ./... && ./keyhunter --help 2>&1 | grep -E "scan|providers|config|verify|recon|keys|serve|dorks|hook|schedule" && echo "HELP OK"</automated>
  </verify>
  <acceptance_criteria>
    - `go build ./...` exits 0
-    - `./keyhunter --help` shows "scan", "providers", and "config" in command list
+    - `./keyhunter --help` shows scan, providers, config, verify, import, recon, keys, serve, dorks, hook, schedule in command list
    - pkg/config/config.go exports Config and Load
    - pkg/output/table.go exports PrintFindings
-    - cmd/root.go declares rootCmd, Execute(), scanCmd, providersCmd, configCmd referenced
+    - pkg/storage/settings.go exports GetSetting and SetSetting
+    - cmd/root.go declares rootCmd, Execute(), and adds all 11 subcommands
    - `grep -q 'viper\.SetConfigFile\|viper\.SetConfigName' cmd/root.go` exits 0
    - lipgloss used for header and confidence coloring
  </acceptance_criteria>
-  <done>Root command, config package, and output table exist. `keyhunter --help` shows the three top-level commands.</done>
+  <done>Root command registers all 11 CLI commands. Config package, output table, and settings helpers exist. `keyhunter --help` shows all commands.</done>
 </task>

 <task type="auto" tdd="false">
-  <name>Task 2: scan, providers, and config subcommands</name>
-  <files>cmd/scan.go, cmd/providers.go, cmd/config.go</files>
+  <name>Task 2: scan, providers, config subcommands, and stub commands</name>
+  <files>cmd/scan.go, cmd/providers.go, cmd/config.go, cmd/stubs.go</files>
  <read_first>
    - /home/salva/Documents/apikey/.planning/phases/01-foundation/01-RESEARCH.md (CLI-04, CLI-05 rows, Pattern 2 pipeline usage)
    - /home/salva/Documents/apikey/cmd/root.go (rootCmd, viper setup)
    - /home/salva/Documents/apikey/pkg/engine/engine.go (Engine.Scan, ScanConfig)
    - /home/salva/Documents/apikey/pkg/storage/db.go (Open, SaveFinding)
+    - /home/salva/Documents/apikey/pkg/storage/settings.go (GetSetting, SetSetting)
    - /home/salva/Documents/apikey/pkg/providers/registry.go (NewRegistry, List, Get, Stats)
  </read_first>
  <action>
@@ -347,6 +416,8 @@ package cmd

 import (
    "context"
+    "encoding/hex"
+    "encoding/json"
    "fmt"
    "os"
    "path/filepath"
@@ -422,9 +493,13 @@ var scanCmd = &cobra.Command{
        }
        defer db.Close()

-        // Derive encryption key (Phase 1: empty passphrase with fixed dev salt)
-        salt := []byte("keyhunter-dev-s0") // Phase 1 placeholder — Phase 6 replaces with proper salt storage
-        encKey := storage.DeriveKey([]byte(cfg.Passphrase), salt)
+        // Derive encryption key using a per-installation salt stored in settings table.
+        // On first run, NewSalt() generates a random salt and stores it.
+        // On subsequent runs, the same salt is loaded — ensuring consistent encryption.
+        encKey, err := loadOrCreateEncKey(db, cfg.Passphrase)
+        if err != nil {
+            return fmt.Errorf("preparing encryption key: %w", err)
+        }

        // Run scan
        ch, err := eng.Scan(context.Background(), src, scanCfg)
@@ -453,8 +528,29 @@ var scanCmd = &cobra.Command{
        // Output
        switch flagOutput {
        case "json":
-            // Phase 6 — basic JSON for now
-            fmt.Printf("[]  # JSON output: Phase 6\n")
+            // Return valid empty JSON array when no findings; full JSON in Phase 6.
+            enc := json.NewEncoder(os.Stdout)
+            enc.SetIndent("", "  ")
+            type jsonFinding struct {
+                Provider   string `json:"provider"`
+                KeyMasked  string `json:"key_masked"`
+                Confidence string `json:"confidence"`
+                Source     string `json:"source"`
+                Line       int    `json:"line"`
+            }
+            out := make([]jsonFinding, 0, len(findings))
+            for _, f := range findings {
+                out = append(out, jsonFinding{
+                    Provider:   f.ProviderName,
+                    KeyMasked:  f.KeyMasked,
+                    Confidence: f.Confidence,
+                    Source:     f.Source,
+                    Line:       f.LineNumber,
+                })
+            }
+            if err := enc.Encode(out); err != nil {
+                return fmt.Errorf("encoding JSON output: %w", err)
+            }
        default:
            output.PrintFindings(findings, flagUnmask)
        }
@@ -467,11 +563,39 @@ var scanCmd = &cobra.Command{
    },
 }

+// loadOrCreateEncKey loads the per-installation salt from the settings table.
+// On first run it generates a new random salt with storage.NewSalt() and persists it.
+// The salt is hex-encoded in the settings table under key "encryption.salt".
+func loadOrCreateEncKey(db *storage.DB, passphrase string) ([]byte, error) {
+    const saltKey = "encryption.salt"
+    saltHex, found, err := db.GetSetting(saltKey)
+    if err != nil {
+        return nil, fmt.Errorf("reading salt from settings: %w", err)
+    }
+    var salt []byte
+    if !found {
+        // First run: generate and persist a new random salt.
+        salt, err = storage.NewSalt()
+        if err != nil {
+            return nil, fmt.Errorf("generating salt: %w", err)
+        }
+        if err := db.SetSetting(saltKey, hex.EncodeToString(salt)); err != nil {
+            return nil, fmt.Errorf("storing salt: %w", err)
+        }
+    } else {
+        salt, err = hex.DecodeString(saltHex)
+        if err != nil {
+            return nil, fmt.Errorf("decoding stored salt: %w", err)
+        }
+    }
+    return storage.DeriveKey([]byte(passphrase), salt), nil
+}
+
 func init() {
    scanCmd.Flags().IntVar(&flagWorkers, "workers", 0, "number of worker goroutines (default: CPU*8)")
    scanCmd.Flags().BoolVar(&flagVerify, "verify", false, "actively verify found keys (opt-in, Phase 5)")
    scanCmd.Flags().BoolVar(&flagUnmask, "unmask", false, "show full key values (default: masked)")
-    scanCmd.Flags().StringVar(&flagOutput, "output", "table", "output format: table, json (more in Phase 6)")
+    scanCmd.Flags().StringVar(&flagOutput, "output", "table", "output format: table, json (full JSON output in Phase 6)")
    scanCmd.Flags().StringSliceVar(&flagExclude, "exclude", nil, "glob patterns to exclude (e.g. *.min.js)")
    viper.BindPFlag("scan.workers", scanCmd.Flags().Lookup("workers"))
 }
@@ -658,6 +782,77 @@ func init() {
    configCmd.AddCommand(configSetCmd)
    configCmd.AddCommand(configGetCmd)
 }
+```
+
+Create **cmd/stubs.go** — stub commands for the 8 phases not yet implemented.
+These satisfy CLI-01 (11 commands) and print a clear "not implemented" message so users
+know the command exists but is pending a future phase.
+
+```go
+package cmd
+
+import (
+    "fmt"
+
+    "github.com/spf13/cobra"
+)
+
+// notImplemented returns a RunE function that prints a "not yet implemented" message.
+// Each stub command is registered in root.go and satisfies CLI-01's 11-command requirement.
+func notImplemented(name, phase string) func(cmd *cobra.Command, args []string) error {
+    return func(cmd *cobra.Command, args []string) error {
+        fmt.Printf("%s: not implemented in this phase (coming in %s)\n", name, phase)
+        return nil
+    }
+}
+
+var verifyCmd = &cobra.Command{
+    Use:   "verify",
+    Short: "Actively verify found API keys (Phase 5)",
+    RunE:  notImplemented("verify", "Phase 5"),
+}
+
+var importCmd = &cobra.Command{
+    Use:   "import",
+    Short: "Import findings from TruffleHog or Gitleaks output (Phase 7)",
+    RunE:  notImplemented("import", "Phase 7"),
+}
+
+var reconCmd = &cobra.Command{
+    Use:   "recon",
+    Short: "Run OSINT recon across internet sources (Phase 9+)",
+    RunE:  notImplemented("recon", "Phase 9"),
+}
+
+var keysCmd = &cobra.Command{
+    Use:   "keys",
+    Short: "Manage stored keys (list, export, delete) (Phase 6)",
+    RunE:  notImplemented("keys", "Phase 6"),
+}
+
+var serveCmd = &cobra.Command{
+    Use:   "serve",
+    Short: "Start the web dashboard (Phase 18)",
+    RunE:  notImplemented("serve", "Phase 18"),
+}
+
+var dorksCmd = &cobra.Command{
+    Use:   "dorks",
+    Short: "Manage and run dork queries (Phase 8)",
+    RunE:  notImplemented("dorks", "Phase 8"),
+}
+
+var hookCmd = &cobra.Command{
+    Use:   "hook",
+    Short: "Install or manage git pre-commit hooks (Phase 7)",
+    RunE:  notImplemented("hook", "Phase 7"),
+}
+
+var scheduleCmd = &cobra.Command{
+    Use:   "schedule",
+    Short: "Manage scheduled recurring scans (Phase 17)",
+    RunE:  notImplemented("schedule", "Phase 17"),
+}
 ```
  </action>
  <verify>
@@ -665,26 +860,32 @@ func init() {
  </verify>
  <acceptance_criteria>
    - `go build -o keyhunter .` exits 0
-    - `./keyhunter --help` shows scan, providers, config commands
+    - `./keyhunter --help` shows all 11 commands: scan, verify, import, recon, keys, serve, dorks, hook, schedule, providers, config
    - `./keyhunter providers list` prints table with >= 3 rows including "openai"
    - `./keyhunter providers info openai` prints Name, Tier, Keywords, Patterns, Verify URL
    - `./keyhunter providers stats` prints "Total providers: 3" or more
    - `./keyhunter config init` creates or updates ~/.keyhunter.yaml
    - `./keyhunter config set scan.workers 16` exits 0
+    - `./keyhunter verify` prints "not implemented in this phase"
+    - `./keyhunter recon` prints "not implemented in this phase"
    - `./keyhunter scan testdata/samples/openai_key.txt` exits 1 (keys found) and prints a table row with "openai"
    - `./keyhunter scan testdata/samples/no_keys.txt` exits 0 and prints "No API keys found."
+    - `./keyhunter scan --output json testdata/samples/no_keys.txt` exits 0 and prints `[]` (valid JSON)
+    - Second run of `./keyhunter scan testdata/samples/openai_key.txt` uses the SAME salt (loaded from settings table)
    - `grep -q 'viper\.BindPFlag' cmd/scan.go` exits 0
+    - `grep -q 'loadOrCreateEncKey' cmd/scan.go` exits 0
  </acceptance_criteria>
-  <done>Full CLI works: scan finds and persists keys, providers list/info/stats work, config init/set/get work. Phase 1 success criteria all met.</done>
+  <done>Full CLI works: scan finds and persists keys with per-installation salt, providers list/info/stats work, config init/set/get work, 8 stub commands registered and respond. Phase 1 success criteria all met.</done>
 </task>

 <task type="checkpoint:human-verify" gate="blocking">
  <what-built>
 Complete Phase 1 implementation:
 - Provider registry with 3 YAML definitions, Aho-Corasick automaton, schema validation
- Storage layer with AES-256-GCM encryption, Argon2id key derivation, SQLite WAL mode
+- Storage layer with AES-256-GCM encryption, Argon2id key derivation, SQLite WAL mode, per-installation salt
 - Three-stage scan engine: keyword pre-filter → regex + entropy detector → finding channel
 - CLI: keyhunter scan, providers list/info/stats, config init/set/get
+- 8 stub commands for future phases (verify, import, recon, keys, serve, dorks, hook, schedule)
  </what-built>
  <how-to-verify>
 Run these commands from the project root and confirm each expected output:
@@ -698,23 +899,29 @@ Run these commands from the project root and confirm each expected output:
 3. `./keyhunter scan testdata/samples/no_keys.txt`
   Expected: Exit code 0, "No API keys found." printed

-4. `./keyhunter providers list`
+4. `./keyhunter scan --output json testdata/samples/no_keys.txt`
+   Expected: Exit code 0, valid JSON printed: `[]`
+
+5. `./keyhunter providers list`
   Expected: Table with openai, anthropic, huggingface rows

-5. `./keyhunter providers info openai`
+6. `./keyhunter providers info openai`
   Expected: Name, Tier 1, Keywords including "sk-proj-", Pattern regex shown

-6. `./keyhunter config init`
+7. `./keyhunter config init`
   Expected: "Config initialized: ~/.keyhunter.yaml" and the file exists

-7. `./keyhunter config set scan.workers 16 && ./keyhunter config get scan.workers`
+8. `./keyhunter config set scan.workers 16 && ./keyhunter config get scan.workers`
   Expected: "Set scan.workers = 16" then "16"

-8. Build the binary with production flags:
-   `CGO_ENABLED=0 go build -ldflags="-s -w" -o keyhunter-prod .`
-   Expected: Builds without error, binary produced
+9. `./keyhunter verify`
+   Expected: "verify: not implemented in this phase (coming in Phase 5)"
+
+10. Build the binary with production flags:
+    `CGO_ENABLED=0 go build -ldflags="-s -w" -o keyhunter-prod .`
+    Expected: Builds without error, binary produced
  </how-to-verify>
-  <resume-signal>Type "approved" if all 8 checks pass, or describe which check failed and what output you saw.</resume-signal>
+  <resume-signal>Type "approved" if all 10 checks pass, or describe which check failed and what output you saw.</resume-signal>
 </task>

 </tasks>
@@ -724,20 +931,24 @@ Full Phase 1 integration check:
 - `go test ./... -count=1` exits 0
 - `./keyhunter scan testdata/samples/openai_key.txt` exits 1 with findings table
 - `./keyhunter scan testdata/samples/no_keys.txt` exits 0 with "No API keys found."
+- `./keyhunter scan --output json testdata/samples/no_keys.txt` prints valid JSON `[]`
 - `./keyhunter providers list` shows 3+ providers
 - `./keyhunter config init` creates ~/.keyhunter.yaml
+- `./keyhunter verify` prints "not implemented in this phase"
 - `CGO_ENABLED=0 go build -ldflags="-s -w" -o keyhunter-prod .` exits 0
 </verification>

 <success_criteria>
- Cobra CLI with scan, providers, config commands (CLI-01)
+- Cobra CLI with all 11 commands: scan, verify, import, recon, keys, serve, dorks, hook, schedule, providers, config (CLI-01)
 - `keyhunter config init` creates ~/.keyhunter.yaml (CLI-02)
 - `keyhunter config set key value` persists (CLI-03)
 - `keyhunter providers list/info/stats` work (CLI-04)
 - scan flags: --workers, --verify, --unmask, --output, --exclude (CLI-05)
+- Per-installation salt stored in settings table; no hardcoded salt in production code
+- JSON output returns valid JSON (not a comment string)
 - All Phase 1 success criteria from ROADMAP.md satisfied:
  1. `keyhunter scan ./somefile` runs three-stage pipeline and returns findings with provider names
-  2. Findings persisted to SQLite with AES-256 encrypted key_value
+  2. Findings persisted to SQLite with AES-256 encrypted key_value; raw db does not contain plaintext
  3. `keyhunter config init` and `config set` work
  4. `keyhunter providers list/info` return provider metadata from YAML
  5. Provider YAML has format_version and last_verified, validated at load time