From 243b7405cdce22276dd8f5f26796a1035dad5a81 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Mon, 6 Apr 2026 01:15:23 +0300 Subject: [PATCH] feat(10-08): add KaggleSource with HTTP Basic auth - KaggleSource queries /api/v1/kernels/list with SetBasicAuth(user, key) - Disabled when either KaggleUser or KaggleKey is empty (no HTTP calls) - Emits Findings tagged recon:kaggle with Source = /code/ - 60/min rate limit via rate.Every(1s), burst 1 - httptest-driven tests cover enabled, auth header, missing creds, 401 unauthorized, and ctx cancellation - RECON-CODE-09 --- pkg/recon/sources/kaggle.go | 149 ++++++++++++++++++++++ pkg/recon/sources/kaggle_test.go | 204 +++++++++++++++++++++++++++++++ 2 files changed, 353 insertions(+) create mode 100644 pkg/recon/sources/kaggle.go create mode 100644 pkg/recon/sources/kaggle_test.go diff --git a/pkg/recon/sources/kaggle.go b/pkg/recon/sources/kaggle.go new file mode 100644 index 0000000..de1a8ba --- /dev/null +++ b/pkg/recon/sources/kaggle.go @@ -0,0 +1,149 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// KaggleSource implements recon.ReconSource against the Kaggle public REST API. +// +// RECON-CODE-09: queries GET /api/v1/kernels/list?search=&pageSize=50 with +// HTTP Basic authentication (username + API key from kaggle.json). Emits +// engine.Finding entries for every returned kernel ref, with Source pointing +// to https://www.kaggle.com/code/. +type KaggleSource struct { + User string + Key string + BaseURL string // API base, default https://www.kaggle.com + WebBaseURL string // Web UI base for Finding URLs, default https://www.kaggle.com + Registry *providers.Registry + Limiters *recon.LimiterRegistry + client *Client +} + +// NewKaggleSource constructs a KaggleSource with default URLs and a shared Client. +func NewKaggleSource(user, key string, reg *providers.Registry, lim *recon.LimiterRegistry) *KaggleSource { + return &KaggleSource{ + User: user, + Key: key, + BaseURL: "https://www.kaggle.com", + WebBaseURL: "https://www.kaggle.com", + Registry: reg, + Limiters: lim, + client: NewClient(), + } +} + +// Name returns the stable source identifier. +func (s *KaggleSource) Name() string { return "kaggle" } + +// RateLimit enforces Kaggle's documented 60 requests/minute ceiling. +func (s *KaggleSource) RateLimit() rate.Limit { return rate.Every(1 * time.Second) } + +// Burst returns the per-source burst capacity. +func (s *KaggleSource) Burst() int { return 1 } + +// RespectsRobots is false — Kaggle exposes a public REST API, not scraped HTML. +func (s *KaggleSource) RespectsRobots() bool { return false } + +// Enabled reports whether both User and Key credentials are present. +func (s *KaggleSource) Enabled(_ recon.Config) bool { + return s.User != "" && s.Key != "" +} + +// Sweep iterates provider keyword queries, calling the Kaggle kernels/list API +// with Basic auth for each. For every returned kernel ref, a Finding is emitted +// on out with SourceType "recon:kaggle" and Source pointing at the web UI URL. +// +// Missing credentials short-circuit to nil without issuing any HTTP calls. +func (s *KaggleSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error { + if s.User == "" || s.Key == "" { + return nil + } + + base := s.BaseURL + if base == "" { + base = "https://www.kaggle.com" + } + web := s.WebBaseURL + if web == "" { + web = "https://www.kaggle.com" + } + + queries := BuildQueries(s.Registry, "kaggle") + + for _, q := range queries { + if err := ctx.Err(); err != nil { + return err + } + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + endpoint := fmt.Sprintf("%s/api/v1/kernels/list?search=%s&pageSize=50", base, url.QueryEscape(q)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return err + } + req.SetBasicAuth(s.User, s.Key) + req.Header.Set("Accept", "application/json") + + resp, err := s.client.Do(ctx, req) + if err != nil { + if errors.Is(err, ErrUnauthorized) { + return err + } + // Sources downgrade on transient errors rather than aborting + // the whole sweep — skip to the next query. + continue + } + + var kernels []kaggleKernel + decodeErr := json.NewDecoder(resp.Body).Decode(&kernels) + resp.Body.Close() + if decodeErr != nil { + continue + } + + for _, k := range kernels { + if k.Ref == "" { + continue + } + f := recon.Finding{ + Confidence: "low", + Source: web + "/code/" + k.Ref, + SourceType: "recon:kaggle", + DetectedAt: time.Now(), + } + select { + case out <- f: + case <-ctx.Done(): + return ctx.Err() + } + } + } + + return nil +} + +// kaggleKernel mirrors the subset of fields returned by /api/v1/kernels/list +// that this source consumes. Additional fields (title, author, language) are +// ignored on purpose to keep the decoder tolerant of future API changes. +type kaggleKernel struct { + Ref string `json:"ref"` +} + +// Compile-time assertion that KaggleSource satisfies recon.ReconSource. +var _ recon.ReconSource = (*KaggleSource)(nil) diff --git a/pkg/recon/sources/kaggle_test.go b/pkg/recon/sources/kaggle_test.go new file mode 100644 index 0000000..90dac47 --- /dev/null +++ b/pkg/recon/sources/kaggle_test.go @@ -0,0 +1,204 @@ +package sources + +import ( + "context" + "encoding/base64" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +func kaggleTestRegistry() *providers.Registry { + return providers.NewRegistryFromProviders([]providers.Provider{ + {Name: "openai", Keywords: []string{"sk-proj-"}}, + }) +} + +func newKaggleSource(t *testing.T, user, key, baseURL string) *KaggleSource { + t.Helper() + s := NewKaggleSource(user, key, kaggleTestRegistry(), recon.NewLimiterRegistry()) + s.BaseURL = baseURL + s.WebBaseURL = "https://www.kaggle.com" + return s +} + +func TestKaggle_Enabled(t *testing.T) { + reg := kaggleTestRegistry() + lim := recon.NewLimiterRegistry() + + cases := []struct { + user, key string + want bool + }{ + {"", "", false}, + {"user", "", false}, + {"", "key", false}, + {"user", "key", true}, + } + for _, c := range cases { + s := NewKaggleSource(c.user, c.key, reg, lim) + if got := s.Enabled(recon.Config{}); got != c.want { + t.Errorf("Enabled(user=%q,key=%q) = %v, want %v", c.user, c.key, got, c.want) + } + } +} + +func TestKaggle_Sweep_BasicAuthAndFindings(t *testing.T) { + var gotAuth string + var gotQuery string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + gotQuery = r.URL.Query().Get("search") + if r.URL.Query().Get("pageSize") != "50" { + t.Errorf("expected pageSize=50, got %q", r.URL.Query().Get("pageSize")) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + _, _ = w.Write([]byte(`[{"ref":"alice/notebook-one","title":"one"},{"ref":"bob/notebook-two","title":"two"}]`)) + })) + defer srv.Close() + + s := newKaggleSource(t, "testuser", "testkey", srv.URL) + + out := make(chan recon.Finding, 8) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := s.Sweep(ctx, "", out); err != nil { + t.Fatalf("Sweep returned error: %v", err) + } + close(out) + + if !strings.HasPrefix(gotAuth, "Basic ") { + t.Fatalf("expected Basic auth header, got %q", gotAuth) + } + decoded, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(gotAuth, "Basic ")) + if err != nil { + t.Fatalf("failed to decode Basic auth: %v", err) + } + if string(decoded) != "testuser:testkey" { + t.Fatalf("expected credentials testuser:testkey, got %q", string(decoded)) + } + + if gotQuery != "sk-proj-" { + t.Errorf("expected search=sk-proj-, got %q", gotQuery) + } + + var findings []recon.Finding + for f := range out { + findings = append(findings, f) + } + if len(findings) != 2 { + t.Fatalf("expected 2 findings, got %d", len(findings)) + } + wantSources := map[string]bool{ + "https://www.kaggle.com/code/alice/notebook-one": false, + "https://www.kaggle.com/code/bob/notebook-two": false, + } + for _, f := range findings { + if f.SourceType != "recon:kaggle" { + t.Errorf("expected SourceType recon:kaggle, got %q", f.SourceType) + } + if _, ok := wantSources[f.Source]; !ok { + t.Errorf("unexpected Source: %q", f.Source) + } + wantSources[f.Source] = true + } + for src, seen := range wantSources { + if !seen { + t.Errorf("missing expected finding source: %s", src) + } + } +} + +func TestKaggle_Sweep_MissingCredentials_NoHTTP(t *testing.T) { + var calls int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&calls, 1) + w.WriteHeader(200) + _, _ = w.Write([]byte("[]")) + })) + defer srv.Close() + + s := newKaggleSource(t, "testuser", "", srv.URL) + out := make(chan recon.Finding, 1) + if err := s.Sweep(context.Background(), "", out); err != nil { + t.Fatalf("expected nil error for missing key, got %v", err) + } + close(out) + + s2 := newKaggleSource(t, "", "testkey", srv.URL) + out2 := make(chan recon.Finding, 1) + if err := s2.Sweep(context.Background(), "", out2); err != nil { + t.Fatalf("expected nil error for missing user, got %v", err) + } + close(out2) + + if n := atomic.LoadInt32(&calls); n != 0 { + t.Fatalf("expected 0 HTTP calls when credentials missing, got %d", n) + } +} + +func TestKaggle_Sweep_Unauthorized(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(401) + _, _ = w.Write([]byte("bad creds")) + })) + defer srv.Close() + + s := newKaggleSource(t, "testuser", "testkey", srv.URL) + out := make(chan recon.Finding, 1) + err := s.Sweep(context.Background(), "", out) + if err == nil { + t.Fatal("expected error on 401") + } + if !errors.Is(err, ErrUnauthorized) { + t.Fatalf("expected ErrUnauthorized, got %v", err) + } +} + +func TestKaggle_Sweep_CtxCancellation(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(2 * time.Second) + w.WriteHeader(200) + _, _ = w.Write([]byte("[]")) + })) + defer srv.Close() + + s := newKaggleSource(t, "testuser", "testkey", srv.URL) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + out := make(chan recon.Finding, 1) + err := s.Sweep(ctx, "", out) + if err == nil { + t.Fatal("expected error from cancelled context") + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } +} + +func TestKaggle_ReconSourceInterface(t *testing.T) { + var _ recon.ReconSource = (*KaggleSource)(nil) + s := NewKaggleSource("u", "k", nil, nil) + if s.Name() != "kaggle" { + t.Errorf("Name = %q, want kaggle", s.Name()) + } + if s.Burst() != 1 { + t.Errorf("Burst = %d, want 1", s.Burst()) + } + if s.RespectsRobots() { + t.Error("RespectsRobots should be false") + } + if s.RateLimit() <= 0 { + t.Error("RateLimit should be > 0") + } +}