feat(11-01): add DuckDuckGoSource, YandexSource, and BraveSource
- DuckDuckGoSource scrapes HTML search (no API key, always enabled, RespectsRobots=true) - YandexSource uses Yandex XML Search API (user+key required, XML response parsing) - BraveSource uses Brave Search API (X-Subscription-Token header, JSON response) - All three follow established error handling: 401 aborts, transient continues, ctx cancellation returns
This commit is contained in:
171
pkg/recon/sources/yandex_test.go
Normal file
171
pkg/recon/sources/yandex_test.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
const yandexXMLFixture = `<?xml version="1.0" encoding="utf-8"?>
|
||||
<yandexsearch>
|
||||
<response>
|
||||
<results>
|
||||
<grouping>
|
||||
<group>
|
||||
<doc>
|
||||
<url>https://pastebin.com/yandex1</url>
|
||||
</doc>
|
||||
</group>
|
||||
<group>
|
||||
<doc>
|
||||
<url>https://github.com/user/repo/blob/main/secrets.env</url>
|
||||
</doc>
|
||||
<doc>
|
||||
<url>https://example.com/leaked</url>
|
||||
</doc>
|
||||
</group>
|
||||
</grouping>
|
||||
</results>
|
||||
</response>
|
||||
</yandexsearch>`
|
||||
|
||||
func yandexStubHandler(t *testing.T, calls *int32) http.HandlerFunc {
|
||||
t.Helper()
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(calls, 1)
|
||||
if !strings.HasPrefix(r.URL.Path, "/search/xml") {
|
||||
t.Errorf("unexpected path: %s", r.URL.Path)
|
||||
}
|
||||
if r.URL.Query().Get("user") != "testuser" {
|
||||
t.Errorf("missing user param")
|
||||
}
|
||||
if r.URL.Query().Get("key") != "testkey" {
|
||||
t.Errorf("missing key param")
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
_, _ = w.Write([]byte(yandexXMLFixture))
|
||||
}
|
||||
}
|
||||
|
||||
func TestYandexSource_EnabledRequiresBoth(t *testing.T) {
|
||||
reg := syntheticRegistry()
|
||||
lim := recon.NewLimiterRegistry()
|
||||
|
||||
tests := []struct {
|
||||
user, key string
|
||||
want bool
|
||||
}{
|
||||
{"", "", false},
|
||||
{"user", "", false},
|
||||
{"", "key", false},
|
||||
{"user", "key", true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
s := NewYandexSource(tt.user, tt.key, reg, lim)
|
||||
if got := s.Enabled(recon.Config{}); got != tt.want {
|
||||
t.Errorf("Enabled(user=%q, key=%q) = %v, want %v", tt.user, tt.key, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestYandexSource_SweepEmptyCredsReturnsNil(t *testing.T) {
|
||||
reg := syntheticRegistry()
|
||||
lim := recon.NewLimiterRegistry()
|
||||
s := NewYandexSource("", "", reg, lim)
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
if err := s.Sweep(context.Background(), "", out); err != nil {
|
||||
t.Fatalf("expected nil, got %v", err)
|
||||
}
|
||||
close(out)
|
||||
if n := countFindings(out); n != 0 {
|
||||
t.Fatalf("expected 0 findings, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestYandexSource_SweepEmitsFindings(t *testing.T) {
|
||||
reg := syntheticRegistry()
|
||||
lim := recon.NewLimiterRegistry()
|
||||
_ = lim.For("yandex", 1000, 100)
|
||||
|
||||
var calls int32
|
||||
srv := httptest.NewServer(yandexStubHandler(t, &calls))
|
||||
defer srv.Close()
|
||||
|
||||
s := NewYandexSource("testuser", "testkey", reg, lim)
|
||||
s.BaseURL = srv.URL
|
||||
|
||||
out := make(chan recon.Finding, 32)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- s.Sweep(ctx, "", out); close(out) }()
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
// 2 keywords * 3 URLs in XML = 6 findings
|
||||
if len(findings) != 6 {
|
||||
t.Fatalf("expected 6 findings, got %d", len(findings))
|
||||
}
|
||||
for _, f := range findings {
|
||||
if f.SourceType != "recon:yandex" {
|
||||
t.Errorf("SourceType=%q want recon:yandex", f.SourceType)
|
||||
}
|
||||
}
|
||||
if got := atomic.LoadInt32(&calls); got != 2 {
|
||||
t.Errorf("expected 2 calls, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestYandexSource_CtxCancelled(t *testing.T) {
|
||||
reg := syntheticRegistry()
|
||||
lim := recon.NewLimiterRegistry()
|
||||
_ = lim.For("yandex", 1000, 100)
|
||||
|
||||
s := NewYandexSource("user", "key", reg, lim)
|
||||
s.BaseURL = "http://127.0.0.1:1"
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
out := make(chan recon.Finding, 1)
|
||||
err := s.Sweep(ctx, "", out)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("expected context.Canceled, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestYandexSource_Unauthorized(t *testing.T) {
|
||||
reg := syntheticRegistry()
|
||||
lim := recon.NewLimiterRegistry()
|
||||
_ = lim.For("yandex", 1000, 100)
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_, _ = w.Write([]byte("bad creds"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
s := NewYandexSource("user", "key", reg, lim)
|
||||
s.BaseURL = srv.URL
|
||||
|
||||
out := make(chan recon.Finding, 1)
|
||||
err := s.Sweep(context.Background(), "", out)
|
||||
if !errors.Is(err, ErrUnauthorized) {
|
||||
t.Fatalf("expected ErrUnauthorized, got %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user