feat(10-05): implement CodebergSource for Gitea REST API
- Add CodebergSource targeting /api/v1/repos/search (Codeberg + any Gitea) - Public API by default; Authorization: token <t> when Token set - Unauth rate limit 60/hour, authenticated ~1000/hour - Emit Findings keyed to repo html_url with SourceType=recon:codeberg - Keyword index maps BuildQueries output back to ProviderName - httptest coverage: name/interface, rate limits (both modes), sweep decoding, header presence/absence, ctx cancellation
This commit is contained in:
167
pkg/recon/sources/codeberg.go
Normal file
167
pkg/recon/sources/codeberg.go
Normal file
@@ -0,0 +1,167 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
// DefaultCodebergBaseURL is the public Codeberg instance. Any Gitea-compatible
|
||||
// server can be substituted by setting CodebergSource.BaseURL.
|
||||
const DefaultCodebergBaseURL = "https://codeberg.org"
|
||||
|
||||
// CodebergSource implements recon.ReconSource against a Gitea-compatible REST
|
||||
// API (Codeberg runs Gitea). Public repository metadata searches do not
|
||||
// require authentication; when a Token is provided it is sent as
|
||||
// "Authorization: token <t>" which raises Gitea's per-user rate limit from
|
||||
// 60/hour to ~1000/hour.
|
||||
//
|
||||
// Sweep iterates every keyword from the provider registry, queries
|
||||
// /api/v1/repos/search?q=<kw>&limit=50, and emits one recon.Finding per
|
||||
// returned repository. The html_url is used as Source; the matching provider
|
||||
// name is attached so downstream verification can target the correct API.
|
||||
type CodebergSource struct {
|
||||
Token string
|
||||
BaseURL string
|
||||
Registry *providers.Registry
|
||||
Limiters *recon.LimiterRegistry
|
||||
|
||||
client *Client
|
||||
}
|
||||
|
||||
// Compile-time interface assertion.
|
||||
var _ recon.ReconSource = (*CodebergSource)(nil)
|
||||
|
||||
// Name returns the stable identifier used by the limiter registry and
|
||||
// Finding.SourceType.
|
||||
func (s *CodebergSource) Name() string { return "codeberg" }
|
||||
|
||||
// RateLimit returns rate.Every(60s) unauthenticated (60/hour) or
|
||||
// rate.Every(3.6s) authenticated (~1000/hour).
|
||||
func (s *CodebergSource) RateLimit() rate.Limit {
|
||||
if s.Token == "" {
|
||||
return rate.Every(60 * time.Second)
|
||||
}
|
||||
return rate.Every(3600 * time.Millisecond)
|
||||
}
|
||||
|
||||
// Burst returns 1 — Gitea's rate limits are per-hour smoothed, a burst of one
|
||||
// keeps us safely within headroom for both auth modes.
|
||||
func (s *CodebergSource) Burst() int { return 1 }
|
||||
|
||||
// RespectsRobots is false — /api/v1/repos/search is a documented REST API.
|
||||
func (s *CodebergSource) RespectsRobots() bool { return false }
|
||||
|
||||
// Enabled is always true because the /repos/search endpoint works anonymously.
|
||||
// A token, when present, only raises the rate limit.
|
||||
func (s *CodebergSource) Enabled(_ recon.Config) bool { return true }
|
||||
|
||||
// Sweep queries Gitea /api/v1/repos/search for every keyword in the provider
|
||||
// registry, decodes the data array, and emits one Finding per result. The
|
||||
// query parameter is ignored — Codeberg is swept by provider keyword, not by
|
||||
// arbitrary Config.Query text, matching the sibling GitHub/GitLab sources.
|
||||
func (s *CodebergSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
base := s.BaseURL
|
||||
if base == "" {
|
||||
base = DefaultCodebergBaseURL
|
||||
}
|
||||
if s.client == nil {
|
||||
s.client = NewClient()
|
||||
}
|
||||
|
||||
// Build a keyword → providerName map once so emitted findings are
|
||||
// correctly attributed even though BuildQueries returns bare strings.
|
||||
keywordIndex := make(map[string]string)
|
||||
if s.Registry != nil {
|
||||
for _, p := range s.Registry.List() {
|
||||
for _, k := range p.Keywords {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := keywordIndex[k]; !exists {
|
||||
keywordIndex[k] = p.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
queries := BuildQueries(s.Registry, s.Name())
|
||||
if len(queries) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, q := range queries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.Limiters != nil {
|
||||
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
endpoint := fmt.Sprintf("%s/api/v1/repos/search?q=%s&limit=50",
|
||||
base, url.QueryEscape(q))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("codeberg: build request: %w", err)
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if s.Token != "" {
|
||||
req.Header.Set("Authorization", "token "+s.Token)
|
||||
}
|
||||
|
||||
resp, err := s.client.Do(ctx, req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("codeberg: search %q: %w", q, err)
|
||||
}
|
||||
|
||||
var decoded struct {
|
||||
OK bool `json:"ok"`
|
||||
Data []struct {
|
||||
FullName string `json:"full_name"`
|
||||
HTMLURL string `json:"html_url"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
return fmt.Errorf("codeberg: decode: %w", err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
|
||||
provider := keywordIndex[q]
|
||||
for _, item := range decoded.Data {
|
||||
if item.HTMLURL == "" {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case out <- recon.Finding{
|
||||
ProviderName: provider,
|
||||
Source: item.HTMLURL,
|
||||
SourceType: "recon:codeberg",
|
||||
Confidence: "low",
|
||||
DetectedAt: time.Now().UTC(),
|
||||
}:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user