Merge branch 'worktree-agent-a2637f83'

pkg/recon/sources/codeberg.go

@@ -0,0 +1,167 @@
+package sources
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+
+	"golang.org/x/time/rate"
+
+	"github.com/salvacybersec/keyhunter/pkg/providers"
+	"github.com/salvacybersec/keyhunter/pkg/recon"
+)
+
+// DefaultCodebergBaseURL is the public Codeberg instance. Any Gitea-compatible
+// server can be substituted by setting CodebergSource.BaseURL.
+const DefaultCodebergBaseURL = "https://codeberg.org"
+
+// CodebergSource implements recon.ReconSource against a Gitea-compatible REST
+// API (Codeberg runs Gitea). Public repository metadata searches do not
+// require authentication; when a Token is provided it is sent as
+// "Authorization: token <t>" which raises Gitea's per-user rate limit from
+// 60/hour to ~1000/hour.
+//
+// Sweep iterates every keyword from the provider registry, queries
+// /api/v1/repos/search?q=<kw>&limit=50, and emits one recon.Finding per
+// returned repository. The html_url is used as Source; the matching provider
+// name is attached so downstream verification can target the correct API.
+type CodebergSource struct {
+	Token    string
+	BaseURL  string
+	Registry *providers.Registry
+	Limiters *recon.LimiterRegistry
+
+	client *Client
+}
+
+// Compile-time interface assertion.
+var _ recon.ReconSource = (*CodebergSource)(nil)
+
+// Name returns the stable identifier used by the limiter registry and
+// Finding.SourceType.
+func (s *CodebergSource) Name() string { return "codeberg" }
+
+// RateLimit returns rate.Every(60s) unauthenticated (60/hour) or
+// rate.Every(3.6s) authenticated (~1000/hour).
+func (s *CodebergSource) RateLimit() rate.Limit {
+	if s.Token == "" {
+		return rate.Every(60 * time.Second)
+	}
+	return rate.Every(3600 * time.Millisecond)
+}
+
+// Burst returns 1 — Gitea's rate limits are per-hour smoothed, a burst of one
+// keeps us safely within headroom for both auth modes.
+func (s *CodebergSource) Burst() int { return 1 }
+
+// RespectsRobots is false — /api/v1/repos/search is a documented REST API.
+func (s *CodebergSource) RespectsRobots() bool { return false }
+
+// Enabled is always true because the /repos/search endpoint works anonymously.
+// A token, when present, only raises the rate limit.
+func (s *CodebergSource) Enabled(_ recon.Config) bool { return true }
+
+// Sweep queries Gitea /api/v1/repos/search for every keyword in the provider
+// registry, decodes the data array, and emits one Finding per result. The
+// query parameter is ignored — Codeberg is swept by provider keyword, not by
+// arbitrary Config.Query text, matching the sibling GitHub/GitLab sources.
+func (s *CodebergSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+
+	base := s.BaseURL
+	if base == "" {
+		base = DefaultCodebergBaseURL
+	}
+	if s.client == nil {
+		s.client = NewClient()
+	}
+
+	// Build a keyword → providerName map once so emitted findings are
+	// correctly attributed even though BuildQueries returns bare strings.
+	keywordIndex := make(map[string]string)
+	if s.Registry != nil {
+		for _, p := range s.Registry.List() {
+			for _, k := range p.Keywords {
+				if k == "" {
+					continue
+				}
+				if _, exists := keywordIndex[k]; !exists {
+					keywordIndex[k] = p.Name
+				}
+			}
+		}
+	}
+
+	queries := BuildQueries(s.Registry, s.Name())
+	if len(queries) == 0 {
+		return nil
+	}
+
+	for _, q := range queries {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		if s.Limiters != nil {
+			if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
+				return err
+			}
+		}
+
+		endpoint := fmt.Sprintf("%s/api/v1/repos/search?q=%s&limit=50",
+			base, url.QueryEscape(q))
+
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+		if err != nil {
+			return fmt.Errorf("codeberg: build request: %w", err)
+		}
+		req.Header.Set("Accept", "application/json")
+		if s.Token != "" {
+			req.Header.Set("Authorization", "token "+s.Token)
+		}
+
+		resp, err := s.client.Do(ctx, req)
+		if err != nil {
+			return fmt.Errorf("codeberg: search %q: %w", q, err)
+		}
+
+		var decoded struct {
+			OK   bool `json:"ok"`
+			Data []struct {
+				FullName string `json:"full_name"`
+				HTMLURL  string `json:"html_url"`
+			} `json:"data"`
+		}
+		if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
+			_, _ = io.Copy(io.Discard, resp.Body)
+			_ = resp.Body.Close()
+			return fmt.Errorf("codeberg: decode: %w", err)
+		}
+		_ = resp.Body.Close()
+
+		provider := keywordIndex[q]
+		for _, item := range decoded.Data {
+			if item.HTMLURL == "" {
+				continue
+			}
+			select {
+			case out <- recon.Finding{
+				ProviderName: provider,
+				Source:       item.HTMLURL,
+				SourceType:   "recon:codeberg",
+				Confidence:   "low",
+				DetectedAt:   time.Now().UTC(),
+			}:
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+		}
+	}
+
+	return nil
+}

pkg/recon/sources/codeberg_test.go

@@ -0,0 +1,182 @@
+package sources
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/time/rate"
+
+	"github.com/salvacybersec/keyhunter/pkg/providers"
+	"github.com/salvacybersec/keyhunter/pkg/recon"
+)
+
+func newCodebergTestRegistry() *providers.Registry {
+	return providers.NewRegistryFromProviders([]providers.Provider{
+		{
+			Name:          "openai",
+			DisplayName:   "OpenAI",
+			Tier:          1,
+			Keywords:      []string{"sk-proj-"},
+			FormatVersion: 1,
+		},
+	})
+}
+
+func TestCodebergSource_NameAndInterface(t *testing.T) {
+	var _ recon.ReconSource = (*CodebergSource)(nil)
+
+	s := &CodebergSource{}
+	if got := s.Name(); got != "codeberg" {
+		t.Errorf("Name() = %q, want %q", got, "codeberg")
+	}
+	if s.RespectsRobots() {
+		t.Errorf("RespectsRobots() = true, want false")
+	}
+	if !s.Enabled(recon.Config{}) {
+		t.Errorf("Enabled() = false, want true (public API)")
+	}
+}
+
+func TestCodebergSource_RateLimitUnauthenticated(t *testing.T) {
+	s := &CodebergSource{}
+	got := s.RateLimit()
+	want := rate.Every(60 * time.Second)
+	if got != want {
+		t.Errorf("RateLimit() no token = %v, want %v", got, want)
+	}
+	if s.Burst() != 1 {
+		t.Errorf("Burst() = %d, want 1", s.Burst())
+	}
+}
+
+func TestCodebergSource_RateLimitAuthenticated(t *testing.T) {
+	s := &CodebergSource{Token: "abc123"}
+	got := s.RateLimit()
+	want := rate.Every(3600 * time.Millisecond)
+	if got != want {
+		t.Errorf("RateLimit() with token = %v, want %v", got, want)
+	}
+}
+
+func TestCodebergSource_SweepEmitsFindings(t *testing.T) {
+	var gotAuth string
+	var gotPath string
+	var gotQuery string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotAuth = r.Header.Get("Authorization")
+		gotPath = r.URL.Path
+		gotQuery = r.URL.Query().Get("q")
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(map[string]any{
+			"ok": true,
+			"data": []map[string]any{
+				{
+					"full_name": "alice/leaked-keys",
+					"html_url":  "https://codeberg.org/alice/leaked-keys",
+				},
+			},
+		})
+	}))
+	defer srv.Close()
+
+	s := &CodebergSource{
+		BaseURL:  srv.URL,
+		Registry: newCodebergTestRegistry(),
+		Limiters: recon.NewLimiterRegistry(),
+	}
+
+	out := make(chan recon.Finding, 8)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	if err := s.Sweep(ctx, "", out); err != nil {
+		t.Fatalf("Sweep: %v", err)
+	}
+	close(out)
+
+	var findings []recon.Finding
+	for f := range out {
+		findings = append(findings, f)
+	}
+	if len(findings) == 0 {
+		t.Fatalf("expected at least one finding")
+	}
+
+	f := findings[0]
+	if f.Source != "https://codeberg.org/alice/leaked-keys" {
+		t.Errorf("Source = %q, want codeberg html_url", f.Source)
+	}
+	if f.SourceType != "recon:codeberg" {
+		t.Errorf("SourceType = %q, want recon:codeberg", f.SourceType)
+	}
+	if f.ProviderName != "openai" {
+		t.Errorf("ProviderName = %q, want openai", f.ProviderName)
+	}
+
+	if gotPath != "/api/v1/repos/search" {
+		t.Errorf("path = %q, want /api/v1/repos/search", gotPath)
+	}
+	if gotQuery == "" {
+		t.Errorf("query param empty")
+	}
+	if gotAuth != "" {
+		t.Errorf("Authorization header should be absent without token, got %q", gotAuth)
+	}
+}
+
+func TestCodebergSource_SweepWithTokenSetsAuthHeader(t *testing.T) {
+	var gotAuth string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotAuth = r.Header.Get("Authorization")
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"ok":true,"data":[]}`))
+	}))
+	defer srv.Close()
+
+	s := &CodebergSource{
+		Token:    "s3cret",
+		BaseURL:  srv.URL,
+		Registry: newCodebergTestRegistry(),
+		Limiters: recon.NewLimiterRegistry(),
+	}
+	out := make(chan recon.Finding, 1)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	if err := s.Sweep(ctx, "", out); err != nil {
+		t.Fatalf("Sweep: %v", err)
+	}
+
+	if !strings.HasPrefix(gotAuth, "token ") || !strings.Contains(gotAuth, "s3cret") {
+		t.Errorf("Authorization header = %q, want \"token s3cret\"", gotAuth)
+	}
+}
+
+func TestCodebergSource_SweepContextCancellation(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		select {
+		case <-r.Context().Done():
+		case <-time.After(3 * time.Second):
+		}
+	}))
+	defer srv.Close()
+
+	s := &CodebergSource{
+		BaseURL:  srv.URL,
+		Registry: newCodebergTestRegistry(),
+		Limiters: recon.NewLimiterRegistry(),
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	out := make(chan recon.Finding, 1)
+	err := s.Sweep(ctx, "", out)
+	if err == nil {
+		t.Fatalf("expected error on cancelled context")
+	}
+}