- DuckDuckGoSource scrapes HTML search (no API key, always enabled, RespectsRobots=true) - YandexSource uses Yandex XML Search API (user+key required, XML response parsing) - BraveSource uses Brave Search API (X-Subscription-Token header, JSON response) - All three follow established error handling: 401 aborts, transient continues, ctx cancellation returns
146 lines
3.8 KiB
Go
146 lines
3.8 KiB
Go
package sources
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/salvacybersec/keyhunter/pkg/recon"
|
|
)
|
|
|
|
func braveStubHandler(t *testing.T, calls *int32) http.HandlerFunc {
|
|
t.Helper()
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
atomic.AddInt32(calls, 1)
|
|
if !strings.HasPrefix(r.URL.Path, "/res/v1/web/search") {
|
|
t.Errorf("unexpected path: %s", r.URL.Path)
|
|
}
|
|
if got := r.Header.Get("X-Subscription-Token"); got != "testtoken" {
|
|
t.Errorf("missing subscription token: %q", got)
|
|
}
|
|
body := map[string]any{
|
|
"web": map[string]any{
|
|
"results": []map[string]any{
|
|
{"url": "https://pastebin.com/brave1", "title": "Brave Result 1", "description": "found key"},
|
|
{"url": "https://github.com/org/repo/blob/main/config.env", "title": "Brave Result 2", "description": "leaked"},
|
|
},
|
|
},
|
|
}
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(body)
|
|
}
|
|
}
|
|
|
|
func TestBraveSource_EnabledRequiresAPIKey(t *testing.T) {
|
|
reg := syntheticRegistry()
|
|
lim := recon.NewLimiterRegistry()
|
|
|
|
if s := NewBraveSource("", reg, lim); s.Enabled(recon.Config{}) {
|
|
t.Error("expected Enabled=false with empty key")
|
|
}
|
|
if s := NewBraveSource("key", reg, lim); !s.Enabled(recon.Config{}) {
|
|
t.Error("expected Enabled=true with key")
|
|
}
|
|
}
|
|
|
|
func TestBraveSource_SweepEmptyKeyReturnsNil(t *testing.T) {
|
|
reg := syntheticRegistry()
|
|
lim := recon.NewLimiterRegistry()
|
|
s := NewBraveSource("", reg, lim)
|
|
|
|
out := make(chan recon.Finding, 10)
|
|
if err := s.Sweep(context.Background(), "", out); err != nil {
|
|
t.Fatalf("expected nil, got %v", err)
|
|
}
|
|
close(out)
|
|
if n := countFindings(out); n != 0 {
|
|
t.Fatalf("expected 0 findings, got %d", n)
|
|
}
|
|
}
|
|
|
|
func TestBraveSource_SweepEmitsFindings(t *testing.T) {
|
|
reg := syntheticRegistry()
|
|
lim := recon.NewLimiterRegistry()
|
|
_ = lim.For("brave", 1000, 100)
|
|
|
|
var calls int32
|
|
srv := httptest.NewServer(braveStubHandler(t, &calls))
|
|
defer srv.Close()
|
|
|
|
s := NewBraveSource("testtoken", reg, lim)
|
|
s.BaseURL = srv.URL
|
|
|
|
out := make(chan recon.Finding, 32)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
|
|
done := make(chan error, 1)
|
|
go func() { done <- s.Sweep(ctx, "", out); close(out) }()
|
|
|
|
var findings []recon.Finding
|
|
for f := range out {
|
|
findings = append(findings, f)
|
|
}
|
|
if err := <-done; err != nil {
|
|
t.Fatalf("Sweep error: %v", err)
|
|
}
|
|
|
|
// 2 keywords * 2 items = 4 findings
|
|
if len(findings) != 4 {
|
|
t.Fatalf("expected 4 findings, got %d", len(findings))
|
|
}
|
|
for _, f := range findings {
|
|
if f.SourceType != "recon:brave" {
|
|
t.Errorf("SourceType=%q want recon:brave", f.SourceType)
|
|
}
|
|
}
|
|
if got := atomic.LoadInt32(&calls); got != 2 {
|
|
t.Errorf("expected 2 calls, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestBraveSource_CtxCancelled(t *testing.T) {
|
|
reg := syntheticRegistry()
|
|
lim := recon.NewLimiterRegistry()
|
|
_ = lim.For("brave", 1000, 100)
|
|
|
|
s := NewBraveSource("key", reg, lim)
|
|
s.BaseURL = "http://127.0.0.1:1"
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
|
|
out := make(chan recon.Finding, 1)
|
|
err := s.Sweep(ctx, "", out)
|
|
if !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("expected context.Canceled, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestBraveSource_Unauthorized(t *testing.T) {
|
|
reg := syntheticRegistry()
|
|
lim := recon.NewLimiterRegistry()
|
|
_ = lim.For("brave", 1000, 100)
|
|
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusUnauthorized)
|
|
_, _ = w.Write([]byte("bad token"))
|
|
}))
|
|
defer srv.Close()
|
|
|
|
s := NewBraveSource("key", reg, lim)
|
|
s.BaseURL = srv.URL
|
|
|
|
out := make(chan recon.Finding, 1)
|
|
err := s.Sweep(context.Background(), "", out)
|
|
if !errors.Is(err, ErrUnauthorized) {
|
|
t.Fatalf("expected ErrUnauthorized, got %v", err)
|
|
}
|
|
}
|