diff --git a/cmd/recon.go b/cmd/recon.go index 49a285b..4362af2 100644 --- a/cmd/recon.go +++ b/cmd/recon.go @@ -167,7 +167,8 @@ func buildReconEngine() *recon.Engine { FOFAAPIKey: firstNonEmpty(os.Getenv("FOFA_API_KEY"), viper.GetString("recon.fofa.api_key")), NetlasAPIKey: firstNonEmpty(os.Getenv("NETLAS_API_KEY"), viper.GetString("recon.netlas.api_key")), BinaryEdgeAPIKey: firstNonEmpty(os.Getenv("BINARYEDGE_API_KEY"), viper.GetString("recon.binaryedge.api_key")), - CircleCIToken: firstNonEmpty(os.Getenv("CIRCLECI_TOKEN"), viper.GetString("recon.circleci.token")), + CircleCIToken: firstNonEmpty(os.Getenv("CIRCLECI_TOKEN"), viper.GetString("recon.circleci.token")), + SecurityTrailsAPIKey: firstNonEmpty(os.Getenv("SECURITYTRAILS_API_KEY"), viper.GetString("recon.securitytrails.api_key")), } sources.RegisterAll(e, cfg) return e diff --git a/pkg/recon/sources/register.go b/pkg/recon/sources/register.go index 304f7d3..e6ca45f 100644 --- a/pkg/recon/sources/register.go +++ b/pkg/recon/sources/register.go @@ -52,6 +52,9 @@ type SourcesConfig struct { // Phase 14: CI/CD source tokens. CircleCIToken string + // Phase 16: DNS/threat intel source tokens. + SecurityTrailsAPIKey string + // Registry drives query generation for every source via BuildQueries. Registry *providers.Registry // Limiters is the shared per-source rate-limiter registry. @@ -61,8 +64,8 @@ type SourcesConfig struct { // RegisterAll registers every Phase 10 code-hosting, Phase 11 search engine / // paste site, Phase 12 IoT scanner / cloud storage, Phase 13 package // registry / container / IaC, Phase 14 CI/CD log / web archive / frontend -// leak, and Phase 15 forum / collaboration tool / log aggregator source on -// engine (67 sources total). +// leak, Phase 15 forum / collaboration tool / log aggregator, and Phase 16 +// mobile / DNS / threat intel source on engine (70 sources total). // // All sources are registered unconditionally so that cmd/recon.go can surface // the full catalog via `keyhunter recon list` regardless of which credentials @@ -282,4 +285,13 @@ func RegisterAll(engine *recon.Engine, cfg SourcesConfig) { engine.Register(&SplunkSource{Registry: reg, Limiters: lim}) engine.Register(&GrafanaSource{Registry: reg, Limiters: lim}) engine.Register(&SentrySource{Registry: reg, Limiters: lim}) + + // Phase 16: Mobile, DNS, and threat intel sources. + engine.Register(&APKMirrorSource{Registry: reg, Limiters: lim}) + engine.Register(&CrtShSource{Registry: reg, Limiters: lim}) + engine.Register(&SecurityTrailsSource{ + APIKey: cfg.SecurityTrailsAPIKey, + Registry: reg, + Limiters: lim, + }) } diff --git a/pkg/recon/sources/securitytrails.go b/pkg/recon/sources/securitytrails.go new file mode 100644 index 0000000..1a4ba29 --- /dev/null +++ b/pkg/recon/sources/securitytrails.go @@ -0,0 +1,189 @@ +package sources + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// SecurityTrailsSource searches SecurityTrails DNS/subdomain data for API key +// exposure. It enumerates subdomains for a target domain and probes config +// endpoints, and also checks DNS history records (TXT records may contain keys). +type SecurityTrailsSource struct { + APIKey string + BaseURL string + Registry *providers.Registry + Limiters *recon.LimiterRegistry + Client *Client + + // ProbeBaseURL overrides the scheme+host used when probing discovered + // subdomains. Tests set this to the httptest server URL. + ProbeBaseURL string +} + +var _ recon.ReconSource = (*SecurityTrailsSource)(nil) + +func (s *SecurityTrailsSource) Name() string { return "securitytrails" } +func (s *SecurityTrailsSource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) } +func (s *SecurityTrailsSource) Burst() int { return 5 } +func (s *SecurityTrailsSource) RespectsRobots() bool { return false } + +func (s *SecurityTrailsSource) Enabled(_ recon.Config) bool { + return s.APIKey != "" +} + +// securityTrailsSubdomains represents the subdomain listing API response. +type securityTrailsSubdomains struct { + Subdomains []string `json:"subdomains"` +} + +func (s *SecurityTrailsSource) Sweep(ctx context.Context, query string, out chan<- recon.Finding) error { + base := s.BaseURL + if base == "" { + base = "https://api.securitytrails.com/v1" + } + client := s.Client + if client == nil { + client = NewClient() + } + + if query == "" || !strings.Contains(query, ".") { + return nil + } + + // Phase 1: Enumerate subdomains. + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + subURL := fmt.Sprintf("%s/domain/%s/subdomains?children_only=false", base, query) + subReq, err := http.NewRequestWithContext(ctx, http.MethodGet, subURL, nil) + if err != nil { + return err + } + subReq.Header.Set("APIKEY", s.APIKey) + + subResp, err := client.Do(ctx, subReq) + if err != nil { + return nil // non-fatal + } + + subData, err := io.ReadAll(io.LimitReader(subResp.Body, 512*1024)) + _ = subResp.Body.Close() + if err != nil { + return nil + } + + var subResult securityTrailsSubdomains + if err := json.Unmarshal(subData, &subResult); err != nil { + return nil + } + + // Build FQDNs and limit to 20. + var fqdns []string + for _, sub := range subResult.Subdomains { + fqdns = append(fqdns, sub+"."+query) + if len(fqdns) >= 20 { + break + } + } + + // Probe config endpoints on each subdomain. + probeClient := &http.Client{Timeout: 5 * time.Second} + for _, fqdn := range fqdns { + if err := ctx.Err(); err != nil { + return err + } + s.probeSubdomain(ctx, probeClient, fqdn, out) + } + + // Phase 2: Check DNS history for key patterns in TXT records. + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + dnsURL := fmt.Sprintf("%s/domain/%s", base, query) + dnsReq, err := http.NewRequestWithContext(ctx, http.MethodGet, dnsURL, nil) + if err != nil { + return nil + } + dnsReq.Header.Set("APIKEY", s.APIKey) + + dnsResp, err := client.Do(ctx, dnsReq) + if err != nil { + return nil + } + + dnsData, err := io.ReadAll(io.LimitReader(dnsResp.Body, 512*1024)) + _ = dnsResp.Body.Close() + if err != nil { + return nil + } + + if ciLogKeyPattern.Match(dnsData) { + out <- recon.Finding{ + ProviderName: query, + Source: dnsURL, + SourceType: "recon:securitytrails", + Confidence: "medium", + DetectedAt: time.Now(), + } + } + + return nil +} + +// probeSubdomain checks well-known config endpoints for key patterns. +func (s *SecurityTrailsSource) probeSubdomain(ctx context.Context, probeClient *http.Client, subdomain string, out chan<- recon.Finding) { + for _, ep := range configProbeEndpoints { + if err := ctx.Err(); err != nil { + return + } + + var probeURL string + if s.ProbeBaseURL != "" { + probeURL = s.ProbeBaseURL + "/" + subdomain + ep + } else { + probeURL = "https://" + subdomain + ep + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, probeURL, nil) + if err != nil { + continue + } + + resp, err := probeClient.Do(req) + if err != nil { + continue + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, 64*1024)) + _ = resp.Body.Close() + if err != nil { + continue + } + + if resp.StatusCode == http.StatusOK && ciLogKeyPattern.Match(body) { + out <- recon.Finding{ + ProviderName: subdomain, + Source: probeURL, + SourceType: "recon:securitytrails", + Confidence: "high", + DetectedAt: time.Now(), + } + } + } +} diff --git a/pkg/recon/sources/securitytrails_test.go b/pkg/recon/sources/securitytrails_test.go new file mode 100644 index 0000000..6867076 --- /dev/null +++ b/pkg/recon/sources/securitytrails_test.go @@ -0,0 +1,180 @@ +package sources + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +func TestSecurityTrails_Name(t *testing.T) { + s := &SecurityTrailsSource{} + if s.Name() != "securitytrails" { + t.Fatalf("expected securitytrails, got %s", s.Name()) + } +} + +func TestSecurityTrails_Enabled(t *testing.T) { + s := &SecurityTrailsSource{} + if s.Enabled(recon.Config{}) { + t.Fatal("SecurityTrailsSource should be disabled without API key") + } + + s.APIKey = "test-key" + if !s.Enabled(recon.Config{}) { + t.Fatal("SecurityTrailsSource should be enabled with API key") + } +} + +func TestSecurityTrails_Sweep(t *testing.T) { + // API server mocks SecurityTrails endpoints. + apiMux := http.NewServeMux() + + // Subdomain enumeration. + apiMux.HandleFunc("/domain/example.com/subdomains", func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("APIKEY") != "test-key" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"subdomains":["api","staging"]}`)) + }) + + // DNS history. + apiMux.HandleFunc("/domain/example.com", func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("APIKEY") != "test-key" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"current_dns":{"txt":{"values":[{"value":"token = sk-proj-ABCDEF1234567890abcdef"}]}}}`)) + }) + + apiSrv := httptest.NewServer(apiMux) + defer apiSrv.Close() + + // Probe server. + probeMux := http.NewServeMux() + probeMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/.env") { + _, _ = w.Write([]byte(`SECRET_KEY = "sk-proj-ABCDEF1234567890abcdef"`)) + return + } + http.NotFound(w, r) + }) + probeSrv := httptest.NewServer(probeMux) + defer probeSrv.Close() + + s := &SecurityTrailsSource{ + APIKey: "test-key", + BaseURL: apiSrv.URL, + Client: NewClient(), + ProbeBaseURL: probeSrv.URL, + } + + out := make(chan recon.Finding, 20) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + err := s.Sweep(ctx, "example.com", out) + close(out) + if err != nil { + t.Fatalf("Sweep error: %v", err) + } + + var findings []recon.Finding + for f := range out { + findings = append(findings, f) + } + if len(findings) == 0 { + t.Fatal("expected at least one finding from SecurityTrails") + } + + // Check that we got both probe findings and DNS history findings. + var probeFound, dnsFound bool + for _, f := range findings { + if f.SourceType != "recon:securitytrails" { + t.Fatalf("expected recon:securitytrails, got %s", f.SourceType) + } + if strings.Contains(f.Source, "/.env") { + probeFound = true + } + if strings.Contains(f.Source, "/domain/example.com") && !strings.Contains(f.Source, "subdomains") { + dnsFound = true + } + } + if !probeFound { + t.Fatal("expected probe finding from SecurityTrails") + } + if !dnsFound { + t.Fatal("expected DNS history finding from SecurityTrails") + } +} + +func TestSecurityTrails_Sweep_SkipsKeywords(t *testing.T) { + s := &SecurityTrailsSource{ + APIKey: "test-key", + Client: NewClient(), + } + + out := make(chan recon.Finding, 10) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := s.Sweep(ctx, "sk-proj-", out) + close(out) + if err != nil { + t.Fatalf("Sweep error: %v", err) + } + + var findings []recon.Finding + for f := range out { + findings = append(findings, f) + } + if len(findings) != 0 { + t.Fatalf("expected no findings for keyword query, got %d", len(findings)) + } +} + +func TestSecurityTrails_Sweep_NoSubdomains(t *testing.T) { + apiMux := http.NewServeMux() + apiMux.HandleFunc("/domain/empty.example.com/subdomains", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"subdomains":[]}`)) + }) + apiMux.HandleFunc("/domain/empty.example.com", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"current_dns":{}}`)) + }) + + apiSrv := httptest.NewServer(apiMux) + defer apiSrv.Close() + + s := &SecurityTrailsSource{ + APIKey: "test-key", + BaseURL: apiSrv.URL, + Client: NewClient(), + } + + out := make(chan recon.Finding, 10) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := s.Sweep(ctx, "empty.example.com", out) + close(out) + if err != nil { + t.Fatalf("Sweep error: %v", err) + } + + var findings []recon.Finding + for f := range out { + findings = append(findings, f) + } + if len(findings) != 0 { + t.Fatalf("expected no findings, got %d", len(findings)) + } +}