From 7272e652079aa064ebd41f4b0f8f557c22c1bdff Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Mon, 6 Apr 2026 11:54:36 +0300 Subject: [PATCH] feat(11-01): add GoogleDorkSource and BingDorkSource with formatQuery updates - GoogleDorkSource uses Google Custom Search JSON API (APIKey+CX required) - BingDorkSource uses Bing Web Search API v7 (Ocp-Apim-Subscription-Key header) - formatQuery now handles google/bing/duckduckgo/yandex/brave dork syntax - Both sources follow established pattern: retry via Client, rate limit via LimiterRegistry --- pkg/recon/sources/bing.go | 155 ++++++++++++++++++++++++++++ pkg/recon/sources/bing_test.go | 146 ++++++++++++++++++++++++++ pkg/recon/sources/google.go | 172 +++++++++++++++++++++++++++++++ pkg/recon/sources/google_test.go | 158 ++++++++++++++++++++++++++++ pkg/recon/sources/queries.go | 2 + 5 files changed, 633 insertions(+) create mode 100644 pkg/recon/sources/bing.go create mode 100644 pkg/recon/sources/bing_test.go create mode 100644 pkg/recon/sources/google.go create mode 100644 pkg/recon/sources/google_test.go diff --git a/pkg/recon/sources/bing.go b/pkg/recon/sources/bing.go new file mode 100644 index 0000000..cdc7ea5 --- /dev/null +++ b/pkg/recon/sources/bing.go @@ -0,0 +1,155 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// BingDorkSource implements recon.ReconSource against the Bing Web Search +// API v7. It iterates provider keyword queries and emits a Finding per result. +// +// A missing API key disables the source without error. +type BingDorkSource struct { + APIKey string + BaseURL string + Registry *providers.Registry + Limiters *recon.LimiterRegistry + client *Client +} + +// Compile-time assertion. +var _ recon.ReconSource = (*BingDorkSource)(nil) + +// NewBingDorkSource constructs a BingDorkSource with the shared retry client. +func NewBingDorkSource(apiKey string, reg *providers.Registry, lim *recon.LimiterRegistry) *BingDorkSource { + return &BingDorkSource{ + APIKey: apiKey, + BaseURL: "https://api.bing.microsoft.com", + Registry: reg, + Limiters: lim, + client: NewClient(), + } +} + +func (s *BingDorkSource) Name() string { return "bing" } +func (s *BingDorkSource) RateLimit() rate.Limit { return rate.Every(500 * time.Millisecond) } +func (s *BingDorkSource) Burst() int { return 2 } +func (s *BingDorkSource) RespectsRobots() bool { return false } + +// Enabled returns true only when APIKey is configured. +func (s *BingDorkSource) Enabled(_ recon.Config) bool { return s.APIKey != "" } + +// Sweep issues one Bing Web Search request per provider keyword and emits a +// Finding for every webPages.value result. +func (s *BingDorkSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error { + if s.APIKey == "" { + return nil + } + base := s.BaseURL + if base == "" { + base = "https://api.bing.microsoft.com" + } + + queries := BuildQueries(s.Registry, "bing") + kwIndex := bingKeywordIndex(s.Registry) + + for _, q := range queries { + if err := ctx.Err(); err != nil { + return err + } + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + endpoint := fmt.Sprintf("%s/v7.0/search?q=%s&count=50", base, url.QueryEscape(q)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("bing: build request: %w", err) + } + req.Header.Set("Ocp-Apim-Subscription-Key", s.APIKey) + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "keyhunter-recon") + + resp, err := s.client.Do(ctx, req) + if err != nil { + if errors.Is(err, ErrUnauthorized) { + return err + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + continue + } + + var parsed bingSearchResponse + decErr := json.NewDecoder(resp.Body).Decode(&parsed) + _ = resp.Body.Close() + if decErr != nil { + continue + } + + provName := kwIndex[strings.ToLower(extractGoogleKeyword(q))] + for _, it := range parsed.WebPages.Value { + f := recon.Finding{ + ProviderName: provName, + Confidence: "low", + Source: it.URL, + SourceType: "recon:bing", + DetectedAt: time.Now(), + } + select { + case out <- f: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil +} + +type bingSearchResponse struct { + WebPages bingWebPages `json:"webPages"` +} + +type bingWebPages struct { + Value []bingWebResult `json:"value"` +} + +type bingWebResult struct { + Name string `json:"name"` + URL string `json:"url"` + Snippet string `json:"snippet"` +} + +// bingKeywordIndex maps lowercased keywords to provider names. +func bingKeywordIndex(reg *providers.Registry) map[string]string { + m := make(map[string]string) + if reg == nil { + return m + } + for _, p := range reg.List() { + for _, k := range p.Keywords { + kl := strings.ToLower(strings.TrimSpace(k)) + if kl == "" { + continue + } + if _, exists := m[kl]; !exists { + m[kl] = p.Name + } + } + } + return m +} diff --git a/pkg/recon/sources/bing_test.go b/pkg/recon/sources/bing_test.go new file mode 100644 index 0000000..9c9d7dd --- /dev/null +++ b/pkg/recon/sources/bing_test.go @@ -0,0 +1,146 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +func bingStubHandler(t *testing.T, calls *int32) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(calls, 1) + if !strings.HasPrefix(r.URL.Path, "/v7.0/search") { + t.Errorf("unexpected path: %s", r.URL.Path) + } + if got := r.Header.Get("Ocp-Apim-Subscription-Key"); got != "testkey" { + t.Errorf("missing subscription key header: %q", got) + } + body := map[string]any{ + "webPages": map[string]any{ + "value": []map[string]any{ + {"name": "result1", "url": "https://pastebin.com/xyz789", "snippet": "found"}, + {"name": "result2", "url": "https://github.com/user/repo/blob/main/.env", "snippet": "key"}, + {"name": "result3", "url": "https://example.com/leak", "snippet": "data"}, + }, + }, + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(body) + } +} + +func TestBingDorkSource_EnabledRequiresAPIKey(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + + if s := NewBingDorkSource("", reg, lim); s.Enabled(recon.Config{}) { + t.Error("expected Enabled=false with empty key") + } + if s := NewBingDorkSource("key", reg, lim); !s.Enabled(recon.Config{}) { + t.Error("expected Enabled=true with key") + } +} + +func TestBingDorkSource_SweepEmptyKeyReturnsNil(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + s := NewBingDorkSource("", reg, lim) + + out := make(chan recon.Finding, 10) + if err := s.Sweep(context.Background(), "", out); err != nil { + t.Fatalf("expected nil, got %v", err) + } + close(out) + if n := countFindings(out); n != 0 { + t.Fatalf("expected 0 findings, got %d", n) + } +} + +func TestBingDorkSource_SweepEmitsFindings(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("bing", 1000, 100) + + var calls int32 + srv := httptest.NewServer(bingStubHandler(t, &calls)) + defer srv.Close() + + s := NewBingDorkSource("testkey", reg, lim) + s.BaseURL = srv.URL + + out := make(chan recon.Finding, 32) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + done := make(chan error, 1) + go func() { done <- s.Sweep(ctx, "", out); close(out) }() + + var findings []recon.Finding + for f := range out { + findings = append(findings, f) + } + if err := <-done; err != nil { + t.Fatalf("Sweep error: %v", err) + } + + // 2 keywords * 3 items = 6 findings + if len(findings) != 6 { + t.Fatalf("expected 6 findings, got %d", len(findings)) + } + for _, f := range findings { + if f.SourceType != "recon:bing" { + t.Errorf("SourceType=%q want recon:bing", f.SourceType) + } + } + if got := atomic.LoadInt32(&calls); got != 2 { + t.Errorf("expected 2 calls, got %d", got) + } +} + +func TestBingDorkSource_CtxCancelled(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("bing", 1000, 100) + + s := NewBingDorkSource("key", reg, lim) + s.BaseURL = "http://127.0.0.1:1" + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + out := make(chan recon.Finding, 1) + err := s.Sweep(ctx, "", out) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } +} + +func TestBingDorkSource_Unauthorized(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("bing", 1000, 100) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte("invalid key")) + })) + defer srv.Close() + + s := NewBingDorkSource("key", reg, lim) + s.BaseURL = srv.URL + + out := make(chan recon.Finding, 1) + err := s.Sweep(context.Background(), "", out) + if !errors.Is(err, ErrUnauthorized) { + t.Fatalf("expected ErrUnauthorized, got %v", err) + } +} diff --git a/pkg/recon/sources/google.go b/pkg/recon/sources/google.go new file mode 100644 index 0000000..3db4e79 --- /dev/null +++ b/pkg/recon/sources/google.go @@ -0,0 +1,172 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// GoogleDorkSource implements recon.ReconSource against the Google Custom +// Search JSON API. It iterates provider keyword queries (via BuildQueries) +// and emits a recon.Finding for every search result item returned. +// +// Both APIKey and CX (custom search engine ID) must be set for the source to +// be enabled. Missing credentials disable the source without error. +type GoogleDorkSource struct { + APIKey string + CX string + BaseURL string + Registry *providers.Registry + Limiters *recon.LimiterRegistry + client *Client +} + +// Compile-time assertion. +var _ recon.ReconSource = (*GoogleDorkSource)(nil) + +// NewGoogleDorkSource constructs a GoogleDorkSource with the shared retry client. +func NewGoogleDorkSource(apiKey, cx string, reg *providers.Registry, lim *recon.LimiterRegistry) *GoogleDorkSource { + return &GoogleDorkSource{ + APIKey: apiKey, + CX: cx, + BaseURL: "https://www.googleapis.com", + Registry: reg, + Limiters: lim, + client: NewClient(), + } +} + +func (s *GoogleDorkSource) Name() string { return "google" } +func (s *GoogleDorkSource) RateLimit() rate.Limit { return rate.Every(1 * time.Second) } +func (s *GoogleDorkSource) Burst() int { return 1 } +func (s *GoogleDorkSource) RespectsRobots() bool { return false } + +// Enabled returns true only when both APIKey and CX are configured. +func (s *GoogleDorkSource) Enabled(_ recon.Config) bool { + return s.APIKey != "" && s.CX != "" +} + +// Sweep issues one Custom Search request per provider keyword and emits a +// Finding for every result item. +func (s *GoogleDorkSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error { + if s.APIKey == "" || s.CX == "" { + return nil + } + base := s.BaseURL + if base == "" { + base = "https://www.googleapis.com" + } + + queries := BuildQueries(s.Registry, "google") + kwIndex := googleKeywordIndex(s.Registry) + + for _, q := range queries { + if err := ctx.Err(); err != nil { + return err + } + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + endpoint := fmt.Sprintf("%s/customsearch/v1?key=%s&cx=%s&q=%s&num=10", + base, url.QueryEscape(s.APIKey), url.QueryEscape(s.CX), url.QueryEscape(q)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("google: build request: %w", err) + } + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "keyhunter-recon") + + resp, err := s.client.Do(ctx, req) + if err != nil { + if errors.Is(err, ErrUnauthorized) { + return err + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + continue + } + + var parsed googleSearchResponse + decErr := json.NewDecoder(resp.Body).Decode(&parsed) + _ = resp.Body.Close() + if decErr != nil { + continue + } + + provName := kwIndex[strings.ToLower(extractGoogleKeyword(q))] + for _, it := range parsed.Items { + f := recon.Finding{ + ProviderName: provName, + Confidence: "low", + Source: it.Link, + SourceType: "recon:google", + DetectedAt: time.Now(), + } + select { + case out <- f: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil +} + +type googleSearchResponse struct { + Items []googleSearchItem `json:"items"` +} + +type googleSearchItem struct { + Title string `json:"title"` + Link string `json:"link"` + Snippet string `json:"snippet"` +} + +// googleKeywordIndex maps lowercased keywords to provider names. +func googleKeywordIndex(reg *providers.Registry) map[string]string { + m := make(map[string]string) + if reg == nil { + return m + } + for _, p := range reg.List() { + for _, k := range p.Keywords { + kl := strings.ToLower(strings.TrimSpace(k)) + if kl == "" { + continue + } + if _, exists := m[kl]; !exists { + m[kl] = p.Name + } + } + } + return m +} + +// extractGoogleKeyword reverses the dork query format to recover the keyword. +func extractGoogleKeyword(q string) string { + // Format: site:pastebin.com OR site:github.com "keyword" + idx := strings.LastIndex(q, `"`) + if idx <= 0 { + return q + } + inner := q[:idx] + start := strings.LastIndex(inner, `"`) + if start < 0 { + return q + } + return inner[start+1:] +} diff --git a/pkg/recon/sources/google_test.go b/pkg/recon/sources/google_test.go new file mode 100644 index 0000000..a6d2072 --- /dev/null +++ b/pkg/recon/sources/google_test.go @@ -0,0 +1,158 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +func googleStubHandler(t *testing.T, calls *int32) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(calls, 1) + if !strings.HasPrefix(r.URL.Path, "/customsearch/v1") { + t.Errorf("unexpected path: %s", r.URL.Path) + } + if r.URL.Query().Get("key") != "testkey" { + t.Errorf("missing api key in query") + } + if r.URL.Query().Get("cx") != "testcx" { + t.Errorf("missing cx in query") + } + body := map[string]any{ + "items": []map[string]any{ + {"title": "result1", "link": "https://pastebin.com/abc123", "snippet": "found key"}, + {"title": "result2", "link": "https://github.com/org/repo/blob/main/env", "snippet": "another"}, + }, + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(body) + } +} + +func TestGoogleDorkSource_EnabledRequiresBothKeys(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + + tests := []struct { + apiKey, cx string + want bool + }{ + {"", "", false}, + {"key", "", false}, + {"", "cx", false}, + {"key", "cx", true}, + } + for _, tt := range tests { + s := NewGoogleDorkSource(tt.apiKey, tt.cx, reg, lim) + if got := s.Enabled(recon.Config{}); got != tt.want { + t.Errorf("Enabled(apiKey=%q, cx=%q) = %v, want %v", tt.apiKey, tt.cx, got, tt.want) + } + } +} + +func TestGoogleDorkSource_SweepEmptyCredsReturnsNil(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + s := NewGoogleDorkSource("", "", reg, lim) + + out := make(chan recon.Finding, 10) + if err := s.Sweep(context.Background(), "", out); err != nil { + t.Fatalf("expected nil err, got %v", err) + } + close(out) + if n := countFindings(out); n != 0 { + t.Fatalf("expected 0 findings, got %d", n) + } +} + +func TestGoogleDorkSource_SweepEmitsFindings(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("google", 1000, 100) + + var calls int32 + srv := httptest.NewServer(googleStubHandler(t, &calls)) + defer srv.Close() + + s := NewGoogleDorkSource("testkey", "testcx", reg, lim) + s.BaseURL = srv.URL + + out := make(chan recon.Finding, 32) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + done := make(chan error, 1) + go func() { done <- s.Sweep(ctx, "", out); close(out) }() + + var findings []recon.Finding + for f := range out { + findings = append(findings, f) + } + if err := <-done; err != nil { + t.Fatalf("Sweep error: %v", err) + } + + // 2 keywords * 2 items = 4 findings + if len(findings) != 4 { + t.Fatalf("expected 4 findings, got %d", len(findings)) + } + for _, f := range findings { + if f.SourceType != "recon:google" { + t.Errorf("SourceType=%q want recon:google", f.SourceType) + } + if f.Confidence != "low" { + t.Errorf("Confidence=%q want low", f.Confidence) + } + } + if got := atomic.LoadInt32(&calls); got != 2 { + t.Errorf("expected 2 API calls, got %d", got) + } +} + +func TestGoogleDorkSource_CtxCancelled(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("google", 1000, 100) + + s := NewGoogleDorkSource("key", "cx", reg, lim) + s.BaseURL = "http://127.0.0.1:1" + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + out := make(chan recon.Finding, 1) + err := s.Sweep(ctx, "", out) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } +} + +func TestGoogleDorkSource_Unauthorized(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("google", 1000, 100) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte("bad key")) + })) + defer srv.Close() + + s := NewGoogleDorkSource("key", "cx", reg, lim) + s.BaseURL = srv.URL + + out := make(chan recon.Finding, 1) + err := s.Sweep(context.Background(), "", out) + if !errors.Is(err, ErrUnauthorized) { + t.Fatalf("expected ErrUnauthorized, got %v", err) + } +} diff --git a/pkg/recon/sources/queries.go b/pkg/recon/sources/queries.go index 6734054..4592136 100644 --- a/pkg/recon/sources/queries.go +++ b/pkg/recon/sources/queries.go @@ -47,6 +47,8 @@ func formatQuery(source, keyword string) string { switch source { case "github", "gist": return fmt.Sprintf("%q in:file", keyword) + case "google", "bing", "duckduckgo", "yandex", "brave": + return fmt.Sprintf(`site:pastebin.com OR site:github.com "%s"`, keyword) default: // GitLab, Bitbucket, Codeberg, HuggingFace, Kaggle, Replit, // CodeSandbox, sandboxes, and unknown sources use bare keywords.