From 30c0e9871bf98e9811325a73ef9655cfe8f779e3 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Sun, 5 Apr 2026 15:41:13 +0300 Subject: [PATCH] feat(05-01): extend VerifySpec and Finding, add gjson dep - VerifySpec: add SuccessCodes, FailureCodes, RateLimitCodes, MetadataPaths, Body - Preserve legacy ValidStatus/InvalidStatus for backward compat - Add EffectiveSuccessCodes/FailureCodes/RateLimitCodes fallback helpers - Add ExtractMetadata helper using gjson (skeleton for Plan 05-03) - Finding: add Verified, VerifyStatus, VerifyHTTPCode, VerifyMetadata, VerifyError - Add github.com/tidwall/gjson v1.18.0 as direct dependency --- go.mod | 3 ++ go.sum | 6 ++++ pkg/engine/finding.go | 7 ++++ pkg/providers/schema.go | 73 ++++++++++++++++++++++++++++++++++++++--- 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 703309b..2f0f746 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 + github.com/tidwall/gjson v1.18.0 golang.org/x/crypto v0.49.0 golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 golang.org/x/time v0.15.0 @@ -59,6 +60,8 @@ require ( github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect diff --git a/go.sum b/go.sum index 68088fb..df57136 100644 --- a/go.sum +++ b/go.sum @@ -135,6 +135,12 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= diff --git a/pkg/engine/finding.go b/pkg/engine/finding.go index c2861d3..c1ddc68 100644 --- a/pkg/engine/finding.go +++ b/pkg/engine/finding.go @@ -14,6 +14,13 @@ type Finding struct { LineNumber int Offset int64 DetectedAt time.Time + + // Verification fields populated when scan --verify is set (Phase 5). + Verified bool // true if verifier ran against this finding + VerifyStatus string // "live", "dead", "rate_limited", "error", "unknown" + VerifyHTTPCode int // HTTP status code returned by verify endpoint + VerifyMetadata map[string]string // extracted metadata from response (org, tier, etc.) + VerifyError string // non-empty if VerifyStatus == "error" } // MaskKey returns a masked representation: first 8 chars + "..." + last 4 chars. diff --git a/pkg/providers/schema.go b/pkg/providers/schema.go index 8a4491b..546960b 100644 --- a/pkg/providers/schema.go +++ b/pkg/providers/schema.go @@ -3,6 +3,7 @@ package providers import ( "fmt" + "github.com/tidwall/gjson" "gopkg.in/yaml.v3" ) @@ -27,11 +28,73 @@ type Pattern struct { // VerifySpec defines how to verify a key is live (used by Phase 5 verification engine). type VerifySpec struct { - Method string `yaml:"method"` - URL string `yaml:"url"` - Headers map[string]string `yaml:"headers"` - ValidStatus []int `yaml:"valid_status"` - InvalidStatus []int `yaml:"invalid_status"` + Method string `yaml:"method"` + URL string `yaml:"url"` + Headers map[string]string `yaml:"headers"` + // Body is an optional request body template; supports {{KEY}} substitution. + Body string `yaml:"body"` + // Canonical status code fields (Phase 5) + SuccessCodes []int `yaml:"success_codes"` + FailureCodes []int `yaml:"failure_codes"` + RateLimitCodes []int `yaml:"rate_limit_codes"` + // MetadataPaths maps display-name -> gjson path (e.g. "org" -> "organization.name"). + MetadataPaths map[string]string `yaml:"metadata_paths"` + // Legacy fields kept for backward compat with existing YAMLs (Phase 2-3 providers). + ValidStatus []int `yaml:"valid_status"` + InvalidStatus []int `yaml:"invalid_status"` +} + +// EffectiveSuccessCodes returns SuccessCodes if non-empty, else falls back to +// legacy ValidStatus, else the default [200]. +func (v VerifySpec) EffectiveSuccessCodes() []int { + if len(v.SuccessCodes) > 0 { + return v.SuccessCodes + } + if len(v.ValidStatus) > 0 { + return v.ValidStatus + } + return []int{200} +} + +// EffectiveFailureCodes returns FailureCodes if non-empty, else falls back to +// legacy InvalidStatus, else the default [401, 403]. +func (v VerifySpec) EffectiveFailureCodes() []int { + if len(v.FailureCodes) > 0 { + return v.FailureCodes + } + if len(v.InvalidStatus) > 0 { + return v.InvalidStatus + } + return []int{401, 403} +} + +// EffectiveRateLimitCodes returns RateLimitCodes if non-empty, else the default [429]. +func (v VerifySpec) EffectiveRateLimitCodes() []int { + if len(v.RateLimitCodes) > 0 { + return v.RateLimitCodes + } + return []int{429} +} + +// ExtractMetadata applies MetadataPaths (gjson expressions) to a JSON response +// body and returns a display-name -> value map. Paths that do not resolve are +// skipped. Returns nil if no paths are configured or the body is empty. +// Plan 05-03 may extend this with type coercion and nested extraction. +func (v VerifySpec) ExtractMetadata(jsonBody []byte) map[string]string { + if len(v.MetadataPaths) == 0 || len(jsonBody) == 0 { + return nil + } + out := make(map[string]string, len(v.MetadataPaths)) + for name, path := range v.MetadataPaths { + result := gjson.GetBytes(jsonBody, path) + if result.Exists() { + out[name] = result.String() + } + } + if len(out) == 0 { + return nil + } + return out } // RegistryStats holds aggregate statistics about loaded providers.