merge: plan 01-02 provider registry
This commit is contained in:
salvacybersec
@@ -9,13 +9,13 @@ Requirements for initial release. Each maps to roadmap phases.
|
||||
|
||||
### Core Engine
|
||||
|
||||
- [x] **CORE-01**: Scanner engine detects API keys using keyword pre-filtering + regex matching pipeline
|
||||
- [ ] **CORE-01**: Scanner engine detects API keys using keyword pre-filtering + regex matching pipeline
|
||||
- [x] **CORE-02**: Provider definitions loaded from YAML files embedded at compile time via Go embed
|
||||
- [x] **CORE-03**: Provider registry manages 108+ provider definitions with pattern, keyword, confidence, and verify metadata
|
||||
- [x] **CORE-04**: Entropy analysis as secondary signal for low-confidence providers (generic key formats)
|
||||
- [x] **CORE-05**: Worker pool parallelism with configurable worker count (default: CPU count)
|
||||
- [ ] **CORE-04**: Entropy analysis as secondary signal for low-confidence providers (generic key formats)
|
||||
- [ ] **CORE-05**: Worker pool parallelism with configurable worker count (default: CPU count)
|
||||
- [x] **CORE-06**: Aho-Corasick keyword pre-filter runs before regex for 10x performance on large files
|
||||
- [x] **CORE-07**: mmap-based large file reading for memory efficiency
|
||||
- [ ] **CORE-07**: mmap-based large file reading for memory efficiency
|
||||
|
||||
### Providers
|
||||
|
||||
@@ -28,7 +28,7 @@ Requirements for initial release. Each maps to roadmap phases.
|
||||
- [ ] **PROV-07**: 10 Tier 7 Code/Dev Tools provider definitions (GitHub Copilot, Cursor, Tabnine, Codeium, Sourcegraph, CodeWhisperer, Replit AI, Codestral, watsonx, Oracle AI)
|
||||
- [ ] **PROV-08**: 10 Tier 8 Self-Hosted provider definitions (Ollama, vLLM, LocalAI, LM Studio, llama.cpp, GPT4All, text-gen-webui, TensorRT-LLM, Triton, Jan AI)
|
||||
- [ ] **PROV-09**: 8 Tier 9 Enterprise provider definitions (Salesforce Einstein, ServiceNow, SAP AI Core, Palantir, Databricks, Snowflake, Oracle GenAI, HPE GreenLake)
|
||||
- [ ] **PROV-10**: Provider YAML schema includes format_version and last_verified date for pattern health tracking
|
||||
- [x] **PROV-10**: Provider YAML schema includes format_version and last_verified date for pattern health tracking
|
||||
|
||||
### Input Sources
|
||||
|
||||
@@ -74,13 +74,13 @@ Requirements for initial release. Each maps to roadmap phases.
|
||||
|
||||
### Storage
|
||||
|
||||
- [x] **STOR-01**: SQLite database for persisting scan results, keys, recon history
|
||||
- [x] **STOR-02**: Application-level AES-256 encryption for stored keys and sensitive config
|
||||
- [x] **STOR-03**: Encryption key derived from user passphrase via Argon2
|
||||
- [ ] **STOR-01**: SQLite database for persisting scan results, keys, recon history
|
||||
- [ ] **STOR-02**: Application-level AES-256 encryption for stored keys and sensitive config
|
||||
- [ ] **STOR-03**: Encryption key derived from user passphrase via Argon2
|
||||
|
||||
### CLI
|
||||
|
||||
- [x] **CLI-01**: Cobra-based CLI with commands: scan, verify, import, recon, keys, serve, dorks, providers, config, hook, schedule
|
||||
- [ ] **CLI-01**: Cobra-based CLI with commands: scan, verify, import, recon, keys, serve, dorks, providers, config, hook, schedule
|
||||
- [ ] **CLI-02**: keyhunter config init creates ~/.keyhunter.yaml
|
||||
- [ ] **CLI-03**: keyhunter config set <key> <value> for all configuration
|
||||
- [ ] **CLI-04**: keyhunter providers list/info/stats for provider management
|
||||
@@ -288,7 +288,7 @@ Requirements for initial release. Each maps to roadmap phases.
|
||||
| CORE-01, CORE-02, CORE-03, CORE-04, CORE-05, CORE-06, CORE-07 | Phase 1 | Pending |
|
||||
| STOR-01, STOR-02, STOR-03 | Phase 1 | Pending |
|
||||
| CLI-01, CLI-02, CLI-03, CLI-04, CLI-05 | Phase 1 | Pending |
|
||||
| PROV-10 | Phase 1 | Pending |
|
||||
| PROV-10 | Phase 1 | Complete |
|
||||
| PROV-01, PROV-02 | Phase 2 | Pending |
|
||||
| PROV-03, PROV-04, PROV-05, PROV-06, PROV-07, PROV-08, PROV-09 | Phase 3 | Pending |
|
||||
| INPUT-01, INPUT-02, INPUT-03, INPUT-04, INPUT-05, INPUT-06 | Phase 4 | Pending |
|
||||
|
||||
@@ -46,9 +46,9 @@ Decimal phases appear between their surrounding integers in numeric order.
|
||||
**Plans**: 5 plans
|
||||
|
||||
Plans:
|
||||
- [x] 01-01-PLAN.md — Go module init, dependency installation, test scaffolding and testdata fixtures
|
||||
- [ ] 01-02-PLAN.md — Provider registry: YAML schema, embed loader, Aho-Corasick automaton, Registry struct
|
||||
- [x] 01-03-PLAN.md — Storage layer: AES-256-GCM encryption, Argon2id key derivation, SQLite + Finding CRUD
|
||||
- [ ] 01-01-PLAN.md — Go module init, dependency installation, test scaffolding and testdata fixtures
|
||||
- [x] 01-02-PLAN.md — Provider registry: YAML schema, embed loader, Aho-Corasick automaton, Registry struct
|
||||
- [ ] 01-03-PLAN.md — Storage layer: AES-256-GCM encryption, Argon2id key derivation, SQLite + Finding CRUD
|
||||
- [ ] 01-04-PLAN.md — Scan engine pipeline: keyword pre-filter, regex+entropy detector, FileSource, ants worker pool
|
||||
- [ ] 01-05-PLAN.md — CLI wiring: scan, providers list/info/stats, config init/set/get, output table
|
||||
|
||||
@@ -255,7 +255,7 @@ Phases execute in numeric order: 1 → 2 → 3 → ... → 18
|
||||
|
||||
| Phase | Plans Complete | Status | Completed |
|
||||
|-------|----------------|--------|-----------|
|
||||
| 1. Foundation | 1/5 | In Progress| |
|
||||
| 1. Foundation | 0/5 | Planning complete | - |
|
||||
| 2. Tier 1-2 Providers | 0/? | Not started | - |
|
||||
| 3. Tier 3-9 Providers | 0/? | Not started | - |
|
||||
| 4. Input Sources | 0/? | Not started | - |
|
||||
|
||||
@@ -3,8 +3,8 @@ gsd_state_version: 1.0
|
||||
milestone: v1.0
|
||||
milestone_name: milestone
|
||||
status: planning
|
||||
stopped_at: Completed 01-foundation-03-PLAN.md
|
||||
last_updated: "2026-04-04T21:07:04.658Z"
|
||||
stopped_at: Completed 01-foundation 01-02-PLAN.md
|
||||
last_updated: "2026-04-04T21:12:49.099Z"
|
||||
last_activity: 2026-04-04 — Roadmap created, 18 phases defined covering 146 v1 requirements
|
||||
progress:
|
||||
total_phases: 18
|
||||
@@ -52,7 +52,7 @@ Progress: [██░░░░░░░░] 20%
|
||||
- Trend: —
|
||||
|
||||
*Updated after each plan completion*
|
||||
| Phase 01-foundation P03 | 3 | 2 tasks | 7 files |
|
||||
| Phase 01-foundation P02 | 9 | 2 tasks | 11 files |
|
||||
|
||||
## Accumulated Context
|
||||
|
||||
@@ -65,8 +65,8 @@ Recent decisions affecting current work:
|
||||
- Roadmap: Per-source rate limiter architecture (Phase 9) must precede all OSINT source modules (Phases 10-16)
|
||||
- Roadmap: AES-256 encryption added in Phase 1, not post-hoc — avoids migration complexity
|
||||
- Roadmap: Verification (Phase 5) requires consent prompt + LEGAL.md — not optional polish
|
||||
- [Phase 01-foundation]: Storage 01-03: Argon2id selected over PBKDF2 — memory-hard RFC 9106 params, resolves STATE.md blocker
|
||||
- [Phase 01-foundation]: Storage 01-03: AES-256-GCM nonce prepended to ciphertext in single BLOB column — no separate nonce column needed
|
||||
- [Phase 01-foundation]: Provider YAML in dual locations: providers/ (user-visible) and pkg/providers/definitions/ (embed) — Go embed cannot use '..' paths
|
||||
- [Phase 01-foundation]: Aho-Corasick built with DFA=true at NewRegistry() for O(n) keyword pre-filtering across all providers
|
||||
|
||||
### Pending Todos
|
||||
|
||||
@@ -81,6 +81,6 @@ None yet.
|
||||
|
||||
## Session Continuity
|
||||
|
||||
Last session: 2026-04-04T21:07:04.654Z
|
||||
Stopped at: Completed 01-foundation-03-PLAN.md
|
||||
Last session: 2026-04-04T21:12:49.095Z
|
||||
Stopped at: Completed 01-foundation 01-02-PLAN.md
|
||||
Resume file: None
|
||||
|
||||
157
.planning/phases/01-foundation/01-02-SUMMARY.md
Normal file
157
.planning/phases/01-foundation/01-02-SUMMARY.md
Normal file
@@ -0,0 +1,157 @@
|
||||
---
|
||||
phase: 01-foundation
|
||||
plan: 02
|
||||
subsystem: providers
|
||||
tags: [yaml, embed, aho-corasick, registry, go-embed, gopkg.in/yaml.v3]
|
||||
|
||||
# Dependency graph
|
||||
requires:
|
||||
- phase: 01-01
|
||||
provides: go.mod with all Phase 1 dependencies, test scaffolding, cmd/root.go stub
|
||||
provides:
|
||||
- Provider, Pattern, VerifySpec, RegistryStats Go structs with YAML validation
|
||||
- Registry with List(), Get(), Stats(), AC() methods
|
||||
- Aho-Corasick automaton built from all provider keywords at NewRegistry()
|
||||
- Three reference provider YAML definitions (openai, anthropic, huggingface)
|
||||
- Compile-time embed of provider YAML via pkg/providers/definitions/
|
||||
affects:
|
||||
- scan-engine
|
||||
- cli-providers-command
|
||||
- verification-engine
|
||||
- storage-layer
|
||||
|
||||
# Tech tracking
|
||||
tech-stack:
|
||||
added:
|
||||
- gopkg.in/yaml.v3 (UnmarshalYAML custom validation)
|
||||
- github.com/petar-dambovaliev/aho-corasick (keyword pre-filter automaton)
|
||||
- embed (stdlib) for compile-time YAML embedding
|
||||
patterns:
|
||||
- Provider YAML at providers/ (user-visible) + pkg/providers/definitions/ (embed location)
|
||||
- Type alias pattern for custom UnmarshalYAML without infinite recursion
|
||||
- Registry injected via constructor (NewRegistry), not global singleton
|
||||
|
||||
key-files:
|
||||
created:
|
||||
- pkg/providers/schema.go
|
||||
- pkg/providers/loader.go
|
||||
- pkg/providers/registry.go
|
||||
- pkg/providers/registry_test.go
|
||||
- pkg/providers/definitions/openai.yaml
|
||||
- pkg/providers/definitions/anthropic.yaml
|
||||
- pkg/providers/definitions/huggingface.yaml
|
||||
- providers/openai.yaml
|
||||
- providers/anthropic.yaml
|
||||
- providers/huggingface.yaml
|
||||
modified: []
|
||||
|
||||
key-decisions:
|
||||
- "Provider YAML kept in dual locations: providers/ (user-visible) and pkg/providers/definitions/ (embedded) — Go embed cannot use '..' paths, so definitions/ subdirectory is canonical embed source"
|
||||
- "UnmarshalYAML validates format_version >= 1 and non-empty last_verified at parse time, not at registry use time — fail fast on malformed definitions"
|
||||
- "Aho-Corasick automaton built with DFA=true for deterministic performance — trades memory for guaranteed O(n) matching"
|
||||
- "Registry is value-safe for concurrent reads — no mutex needed since providers slice is written once at NewRegistry and never mutated"
|
||||
|
||||
patterns-established:
|
||||
- "Pattern 1: Type alias in UnmarshalYAML to avoid infinite recursion: `type ProviderAlias Provider`"
|
||||
- "Pattern 2: embed path convention — YAML at pkg/providers/definitions/, user docs at providers/"
|
||||
- "Pattern 3: Registry constructor NewRegistry() loads+validates+indexes+builds AC in one call"
|
||||
|
||||
requirements-completed: [CORE-02, CORE-03, CORE-06, PROV-10]
|
||||
|
||||
# Metrics
|
||||
duration: 9min
|
||||
completed: 2026-04-04
|
||||
---
|
||||
|
||||
# Phase 01 Plan 02: Provider Registry Summary
|
||||
|
||||
**YAML schema structs with UnmarshalYAML validation, embed.FS loader, and Aho-Corasick registry serving List/Get/Stats/AC to all downstream subsystems**
|
||||
|
||||
## Performance
|
||||
|
||||
- **Duration:** ~9 min
|
||||
- **Started:** 2026-04-04T21:02:31Z
|
||||
- **Completed:** 2026-04-04T21:11:41Z
|
||||
- **Tasks:** 2 (both TDD)
|
||||
- **Files modified:** 10 created, 1 updated (registry_test.go)
|
||||
|
||||
## Accomplishments
|
||||
|
||||
- Provider YAML schema with compile-time validation (format_version >= 1, last_verified required, confidence enum)
|
||||
- Registry loads 3 providers from embedded YAML at startup, builds Aho-Corasick automaton over all keywords
|
||||
- Three reference provider YAML definitions with full verify specs (OpenAI, Anthropic, HuggingFace)
|
||||
- All 5 provider tests pass: TestRegistryLoad, TestRegistryGet, TestRegistryStats, TestAhoCorasickBuild, TestProviderSchemaValidation
|
||||
|
||||
## Task Commits
|
||||
|
||||
Each task was committed atomically:
|
||||
|
||||
1. **TDD RED - Failing tests for schema and registry** - `ebaf7d7` (test)
|
||||
2. **Task 1: Provider schema structs and reference YAMLs** - `4fcdc42` (feat)
|
||||
3. **Task 2: Embed loader, registry with AC, filled test stubs** - `a9859b3` (feat)
|
||||
|
||||
_Note: Bootstrap (go.mod, main.go, test stubs) was included in the RED commit since Plan 01-01 runs in parallel._
|
||||
|
||||
## Files Created/Modified
|
||||
|
||||
- `pkg/providers/schema.go` - Provider, Pattern, VerifySpec, RegistryStats structs with UnmarshalYAML validation
|
||||
- `pkg/providers/loader.go` - embed.FS declaration with //go:embed definitions/*.yaml and fs.WalkDir loader
|
||||
- `pkg/providers/registry.go` - Registry struct with List(), Get(), Stats(), AC() methods and NewRegistry() constructor
|
||||
- `pkg/providers/registry_test.go` - Full test implementation (replaced stub from Plan 01)
|
||||
- `pkg/providers/definitions/openai.yaml` - Embedded OpenAI provider definition
|
||||
- `pkg/providers/definitions/anthropic.yaml` - Embedded Anthropic provider definition
|
||||
- `pkg/providers/definitions/huggingface.yaml` - Embedded HuggingFace provider definition
|
||||
- `providers/openai.yaml` - User-visible OpenAI reference definition
|
||||
- `providers/anthropic.yaml` - User-visible Anthropic reference definition
|
||||
- `providers/huggingface.yaml` - User-visible HuggingFace reference definition
|
||||
|
||||
## Decisions Made
|
||||
|
||||
- **Dual YAML location:** providers/ for user reference, pkg/providers/definitions/ for embed — Go's embed package cannot traverse `..` paths, so definitions/ inside the package is the only valid embed location.
|
||||
- **DFA mode for Aho-Corasick:** `Opts{DFA: true}` chosen for guaranteed O(n) matching at cost of higher upfront build time — appropriate for a scanner tool that pays build cost once and scans many files.
|
||||
- **Constructor injection over globals:** NewRegistry() returns a value; callers inject it. No package-level `var Registry` global — avoids init order issues and enables testing.
|
||||
|
||||
## Deviations from Plan
|
||||
|
||||
### Auto-fixed Issues
|
||||
|
||||
**1. [Rule 3 - Blocking] Bootstrapped Plan 01-01 prerequisites in this worktree**
|
||||
- **Found during:** Pre-task setup
|
||||
- **Issue:** Plan 01-02 depends on Plan 01-01 (go.mod, main.go, test stubs) which runs in parallel in a different worktree. This worktree had no go.mod.
|
||||
- **Fix:** Executed Plan 01-01 bootstrap (go mod init, go get all 10 deps, main.go, cmd/root.go, testdata fixtures, test stub files) before starting Plan 01-02 tasks.
|
||||
- **Files modified:** go.mod, go.sum, main.go, cmd/root.go, testdata/samples/*.txt, pkg/*/stub_test.go files
|
||||
- **Verification:** `go build ./...` succeeded before Plan 01-02 task execution
|
||||
- **Committed in:** ebaf7d7 (RED phase commit includes bootstrap)
|
||||
|
||||
**2. [Rule 3 - Blocking] go mod tidy required after adding production packages**
|
||||
- **Found during:** Task 2 GREEN phase
|
||||
- **Issue:** `go test` failed with "no required module provides package github.com/petar-dambovaliev/aho-corasick" even though it was in go.mod — tidy hadn't propagated it for non-test code.
|
||||
- **Fix:** Ran `go mod tidy` which resolved the module graph.
|
||||
- **Files modified:** go.mod, go.sum
|
||||
- **Verification:** `go test ./pkg/providers/...` passed after tidy
|
||||
|
||||
---
|
||||
|
||||
**Total deviations:** 2 auto-fixed (2 blocking/infrastructure)
|
||||
**Impact on plan:** Both deviations were infrastructure setup, not scope changes. Plan objectives met exactly.
|
||||
|
||||
## Issues Encountered
|
||||
|
||||
- Go embed `..` path restriction required dual YAML directory strategy (documented in plan's context, confirmed during implementation)
|
||||
- aho-corasick package name is `aho_corasick` (underscore) not `ahocorasick` — used import alias `ahocorasick` for cleaner code
|
||||
|
||||
## User Setup Required
|
||||
|
||||
None - no external service configuration required.
|
||||
|
||||
## Next Phase Readiness
|
||||
|
||||
- Registry interface is stable: NewRegistry(), List(), Get(), Stats(), AC() — downstream plans can depend on these signatures
|
||||
- Phase 03 (Storage Layer) can proceed immediately — no registry dependency
|
||||
- Phase 04 (Scan Engine) can now wire AC() for keyword pre-filtering
|
||||
- Phase 05 (CLI) can call Registry.List() for `keyhunter providers list`
|
||||
- Known: only 3 reference providers embedded; Phase 02-03 will add all 108
|
||||
|
||||
---
|
||||
*Phase: 01-foundation*
|
||||
*Completed: 2026-04-04*
|
||||
18
go.mod
18
go.mod
@@ -2,21 +2,13 @@ module github.com/salvacybersec/keyhunter
|
||||
|
||||
go 1.26.1
|
||||
|
||||
require github.com/stretchr/testify v1.11.1
|
||||
require (
|
||||
github.com/petar-dambovaliev/aho-corasick v0.0.0-20250424160509-463d218d4745
|
||||
github.com/stretchr/testify v1.11.1
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
golang.org/x/crypto v0.49.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/libc v1.70.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.48.1 // indirect
|
||||
)
|
||||
|
||||
25
go.sum
25
go.sum
@@ -1,33 +1,12 @@
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/petar-dambovaliev/aho-corasick v0.0.0-20250424160509-463d218d4745 h1:Vpr4VgAizEgEZsaMohpw6JYDP+i9Of9dmdY4ufNP6HI=
|
||||
github.com/petar-dambovaliev/aho-corasick v0.0.0-20250424160509-463d218d4745/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
|
||||
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
modernc.org/libc v1.70.0 h1:U58NawXqXbgpZ/dcdS9kMshu08aiA6b7gusEusqzNkw=
|
||||
modernc.org/libc v1.70.0/go.mod h1:OVmxFGP1CI/Z4L3E0Q3Mf1PDE0BucwMkcXjjLntvHJo=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/sqlite v1.48.1 h1:S85iToyU6cgeojybE2XJlSbcsvcWkQ6qqNXJHtW5hWA=
|
||||
modernc.org/sqlite v1.48.1/go.mod h1:hWjRO6Tj/5Ik8ieqxQybiEOUXy0NJFNp2tpvVpKlvig=
|
||||
|
||||
20
pkg/providers/definitions/anthropic.yaml
Normal file
20
pkg/providers/definitions/anthropic.yaml
Normal file
@@ -0,0 +1,20 @@
|
||||
format_version: 1
|
||||
name: anthropic
|
||||
display_name: Anthropic
|
||||
tier: 1
|
||||
last_verified: "2026-04-04"
|
||||
keywords:
|
||||
- "sk-ant-api03-"
|
||||
- "anthropic"
|
||||
patterns:
|
||||
- regex: 'sk-ant-api03-[A-Za-z0-9_\-]{93,}'
|
||||
entropy_min: 3.5
|
||||
confidence: high
|
||||
verify:
|
||||
method: GET
|
||||
url: https://api.anthropic.com/v1/models
|
||||
headers:
|
||||
x-api-key: "{KEY}"
|
||||
anthropic-version: "2023-06-01"
|
||||
valid_status: [200]
|
||||
invalid_status: [401, 403]
|
||||
19
pkg/providers/definitions/huggingface.yaml
Normal file
19
pkg/providers/definitions/huggingface.yaml
Normal file
@@ -0,0 +1,19 @@
|
||||
format_version: 1
|
||||
name: huggingface
|
||||
display_name: HuggingFace
|
||||
tier: 3
|
||||
last_verified: "2026-04-04"
|
||||
keywords:
|
||||
- "hf_"
|
||||
- "huggingface"
|
||||
patterns:
|
||||
- regex: 'hf_[A-Za-z0-9]{34,}'
|
||||
entropy_min: 3.5
|
||||
confidence: high
|
||||
verify:
|
||||
method: GET
|
||||
url: https://huggingface.co/api/whoami-v2
|
||||
headers:
|
||||
Authorization: "Bearer {KEY}"
|
||||
valid_status: [200]
|
||||
invalid_status: [401, 403]
|
||||
19
pkg/providers/definitions/openai.yaml
Normal file
19
pkg/providers/definitions/openai.yaml
Normal file
@@ -0,0 +1,19 @@
|
||||
format_version: 1
|
||||
name: openai
|
||||
display_name: OpenAI
|
||||
tier: 1
|
||||
last_verified: "2026-04-04"
|
||||
keywords:
|
||||
- "sk-proj-"
|
||||
- "openai"
|
||||
patterns:
|
||||
- regex: 'sk-proj-[A-Za-z0-9_\-]{48,}'
|
||||
entropy_min: 3.5
|
||||
confidence: high
|
||||
verify:
|
||||
method: GET
|
||||
url: https://api.openai.com/v1/models
|
||||
headers:
|
||||
Authorization: "Bearer {KEY}"
|
||||
valid_status: [200]
|
||||
invalid_status: [401, 403]
|
||||
37
pkg/providers/loader.go
Normal file
37
pkg/providers/loader.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
//go:embed definitions/*.yaml
|
||||
var definitionsFS embed.FS
|
||||
|
||||
// loadProviders reads all YAML files from the embedded definitions FS.
|
||||
func loadProviders() ([]Provider, error) {
|
||||
var providers []Provider
|
||||
err := fs.WalkDir(definitionsFS, "definitions", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() || filepath.Ext(path) != ".yaml" {
|
||||
return nil
|
||||
}
|
||||
data, err := definitionsFS.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading provider file %s: %w", path, err)
|
||||
}
|
||||
var p Provider
|
||||
if err := yaml.Unmarshal(data, &p); err != nil {
|
||||
return fmt.Errorf("parsing provider %s: %w", path, err)
|
||||
}
|
||||
providers = append(providers, p)
|
||||
return nil
|
||||
})
|
||||
return providers, err
|
||||
}
|
||||
75
pkg/providers/registry.go
Normal file
75
pkg/providers/registry.go
Normal file
@@ -0,0 +1,75 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
ahocorasick "github.com/petar-dambovaliev/aho-corasick"
|
||||
)
|
||||
|
||||
// Registry is the in-memory store of all loaded provider definitions.
|
||||
// It is initialized once at startup and is safe for concurrent reads.
|
||||
type Registry struct {
|
||||
providers []Provider
|
||||
index map[string]int // name -> slice index
|
||||
ac ahocorasick.AhoCorasick // pre-built automaton for keyword pre-filter
|
||||
}
|
||||
|
||||
// NewRegistry loads all embedded provider YAML files, validates them, builds the
|
||||
// Aho-Corasick automaton from all provider keywords, and returns the Registry.
|
||||
func NewRegistry() (*Registry, error) {
|
||||
providers, err := loadProviders()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("loading providers: %w", err)
|
||||
}
|
||||
|
||||
index := make(map[string]int, len(providers))
|
||||
var keywords []string
|
||||
for i, p := range providers {
|
||||
index[p.Name] = i
|
||||
keywords = append(keywords, p.Keywords...)
|
||||
}
|
||||
|
||||
builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{DFA: true})
|
||||
ac := builder.Build(keywords)
|
||||
|
||||
return &Registry{
|
||||
providers: providers,
|
||||
index: index,
|
||||
ac: ac,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// List returns all loaded providers.
|
||||
func (r *Registry) List() []Provider {
|
||||
return r.providers
|
||||
}
|
||||
|
||||
// Get returns a provider by name and a boolean indicating whether it was found.
|
||||
func (r *Registry) Get(name string) (Provider, bool) {
|
||||
idx, ok := r.index[name]
|
||||
if !ok {
|
||||
return Provider{}, false
|
||||
}
|
||||
return r.providers[idx], true
|
||||
}
|
||||
|
||||
// Stats returns aggregate statistics about the loaded providers.
|
||||
func (r *Registry) Stats() RegistryStats {
|
||||
stats := RegistryStats{
|
||||
Total: len(r.providers),
|
||||
ByTier: make(map[int]int),
|
||||
ByConfidence: make(map[string]int),
|
||||
}
|
||||
for _, p := range r.providers {
|
||||
stats.ByTier[p.Tier]++
|
||||
for _, pat := range p.Patterns {
|
||||
stats.ByConfidence[pat.Confidence]++
|
||||
}
|
||||
}
|
||||
return stats
|
||||
}
|
||||
|
||||
// AC returns the pre-built Aho-Corasick automaton for keyword pre-filtering.
|
||||
func (r *Registry) AC() ahocorasick.AhoCorasick {
|
||||
return r.ac
|
||||
}
|
||||
@@ -2,22 +2,57 @@ package providers_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// TestRegistryLoad verifies that provider YAML files are loaded from embed.FS.
|
||||
// Stub: will be implemented when registry.go exists (Plan 02).
|
||||
func TestRegistryLoad(t *testing.T) {
|
||||
t.Skip("stub — implement after registry.go exists")
|
||||
reg, err := providers.NewRegistry()
|
||||
require.NoError(t, err)
|
||||
assert.GreaterOrEqual(t, len(reg.List()), 3, "expected at least 3 providers")
|
||||
}
|
||||
|
||||
// TestProviderSchemaValidation verifies format_version and last_verified are required.
|
||||
// Stub: will be implemented when schema.go validation exists (Plan 02).
|
||||
func TestProviderSchemaValidation(t *testing.T) {
|
||||
t.Skip("stub — implement after schema.go validation exists")
|
||||
func TestRegistryGet(t *testing.T) {
|
||||
reg, err := providers.NewRegistry()
|
||||
require.NoError(t, err)
|
||||
|
||||
p, ok := reg.Get("openai")
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "openai", p.Name)
|
||||
assert.Equal(t, 1, p.Tier)
|
||||
|
||||
_, notOk := reg.Get("nonexistent-provider")
|
||||
assert.False(t, notOk)
|
||||
}
|
||||
|
||||
func TestRegistryStats(t *testing.T) {
|
||||
reg, err := providers.NewRegistry()
|
||||
require.NoError(t, err)
|
||||
|
||||
stats := reg.Stats()
|
||||
assert.GreaterOrEqual(t, stats.Total, 3)
|
||||
assert.GreaterOrEqual(t, stats.ByTier[1], 2)
|
||||
}
|
||||
|
||||
// TestAhoCorasickBuild verifies Aho-Corasick automaton builds from provider keywords.
|
||||
// Stub: will be implemented when registry builds automaton (Plan 02).
|
||||
func TestAhoCorasickBuild(t *testing.T) {
|
||||
t.Skip("stub — implement after registry AC build exists")
|
||||
reg, err := providers.NewRegistry()
|
||||
require.NoError(t, err)
|
||||
|
||||
ac := reg.AC()
|
||||
matches := ac.FindAll("export OPENAI_API_KEY=sk-proj-abc")
|
||||
assert.NotEmpty(t, matches)
|
||||
|
||||
noMatches := ac.FindAll("hello world nothing here")
|
||||
assert.Empty(t, noMatches)
|
||||
}
|
||||
|
||||
func TestProviderSchemaValidation(t *testing.T) {
|
||||
invalid := []byte("format_version: 0\nname: invalid\nlast_verified: \"\"\n")
|
||||
var p providers.Provider
|
||||
err := yaml.Unmarshal(invalid, &p)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "format_version")
|
||||
}
|
||||
|
||||
66
pkg/providers/schema.go
Normal file
66
pkg/providers/schema.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// Provider represents a single API key provider definition loaded from YAML.
|
||||
type Provider struct {
|
||||
FormatVersion int `yaml:"format_version"`
|
||||
Name string `yaml:"name"`
|
||||
DisplayName string `yaml:"display_name"`
|
||||
Tier int `yaml:"tier"`
|
||||
LastVerified string `yaml:"last_verified"`
|
||||
Keywords []string `yaml:"keywords"`
|
||||
Patterns []Pattern `yaml:"patterns"`
|
||||
Verify VerifySpec `yaml:"verify"`
|
||||
}
|
||||
|
||||
// Pattern defines a single regex pattern for API key detection.
|
||||
type Pattern struct {
|
||||
Regex string `yaml:"regex"`
|
||||
EntropyMin float64 `yaml:"entropy_min"`
|
||||
Confidence string `yaml:"confidence"`
|
||||
}
|
||||
|
||||
// VerifySpec defines how to verify a key is live (used by Phase 5 verification engine).
|
||||
type VerifySpec struct {
|
||||
Method string `yaml:"method"`
|
||||
URL string `yaml:"url"`
|
||||
Headers map[string]string `yaml:"headers"`
|
||||
ValidStatus []int `yaml:"valid_status"`
|
||||
InvalidStatus []int `yaml:"invalid_status"`
|
||||
}
|
||||
|
||||
// RegistryStats holds aggregate statistics about loaded providers.
|
||||
type RegistryStats struct {
|
||||
Total int
|
||||
ByTier map[int]int
|
||||
ByConfidence map[string]int
|
||||
}
|
||||
|
||||
// UnmarshalYAML implements yaml.Unmarshaler with schema validation (satisfies PROV-10).
|
||||
func (p *Provider) UnmarshalYAML(value *yaml.Node) error {
|
||||
// Use a type alias to avoid infinite recursion
|
||||
type ProviderAlias Provider
|
||||
var alias ProviderAlias
|
||||
if err := value.Decode(&alias); err != nil {
|
||||
return err
|
||||
}
|
||||
if alias.FormatVersion < 1 {
|
||||
return fmt.Errorf("provider %q: format_version must be >= 1 (got %d)", alias.Name, alias.FormatVersion)
|
||||
}
|
||||
if alias.LastVerified == "" {
|
||||
return fmt.Errorf("provider %q: last_verified is required", alias.Name)
|
||||
}
|
||||
validConfidences := map[string]bool{"high": true, "medium": true, "low": true, "": true}
|
||||
for _, pat := range alias.Patterns {
|
||||
if !validConfidences[pat.Confidence] {
|
||||
return fmt.Errorf("provider %q: pattern confidence %q must be high, medium, or low", alias.Name, pat.Confidence)
|
||||
}
|
||||
}
|
||||
*p = Provider(alias)
|
||||
return nil
|
||||
}
|
||||
20
providers/anthropic.yaml
Normal file
20
providers/anthropic.yaml
Normal file
@@ -0,0 +1,20 @@
|
||||
format_version: 1
|
||||
name: anthropic
|
||||
display_name: Anthropic
|
||||
tier: 1
|
||||
last_verified: "2026-04-04"
|
||||
keywords:
|
||||
- "sk-ant-api03-"
|
||||
- "anthropic"
|
||||
patterns:
|
||||
- regex: 'sk-ant-api03-[A-Za-z0-9_\-]{93,}'
|
||||
entropy_min: 3.5
|
||||
confidence: high
|
||||
verify:
|
||||
method: GET
|
||||
url: https://api.anthropic.com/v1/models
|
||||
headers:
|
||||
x-api-key: "{KEY}"
|
||||
anthropic-version: "2023-06-01"
|
||||
valid_status: [200]
|
||||
invalid_status: [401, 403]
|
||||
19
providers/huggingface.yaml
Normal file
19
providers/huggingface.yaml
Normal file
@@ -0,0 +1,19 @@
|
||||
format_version: 1
|
||||
name: huggingface
|
||||
display_name: HuggingFace
|
||||
tier: 3
|
||||
last_verified: "2026-04-04"
|
||||
keywords:
|
||||
- "hf_"
|
||||
- "huggingface"
|
||||
patterns:
|
||||
- regex: 'hf_[A-Za-z0-9]{34,}'
|
||||
entropy_min: 3.5
|
||||
confidence: high
|
||||
verify:
|
||||
method: GET
|
||||
url: https://huggingface.co/api/whoami-v2
|
||||
headers:
|
||||
Authorization: "Bearer {KEY}"
|
||||
valid_status: [200]
|
||||
invalid_status: [401, 403]
|
||||
19
providers/openai.yaml
Normal file
19
providers/openai.yaml
Normal file
@@ -0,0 +1,19 @@
|
||||
format_version: 1
|
||||
name: openai
|
||||
display_name: OpenAI
|
||||
tier: 1
|
||||
last_verified: "2026-04-04"
|
||||
keywords:
|
||||
- "sk-proj-"
|
||||
- "openai"
|
||||
patterns:
|
||||
- regex: 'sk-proj-[A-Za-z0-9_\-]{48,}'
|
||||
entropy_min: 3.5
|
||||
confidence: high
|
||||
verify:
|
||||
method: GET
|
||||
url: https://api.openai.com/v1/models
|
||||
headers:
|
||||
Authorization: "Bearer {KEY}"
|
||||
valid_status: [200]
|
||||
invalid_status: [401, 403]
|
||||
Reference in New Issue
Block a user