salvacybersec/keyhunter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: phase 14-04 register wiring

Browse Source
This commit is contained in:
salvacybersec
2026-04-06 13:39:32 +03:00
parent 3a4e9c11bf 117213aa7e
commit 27624e0ec7
20 changed files with 1556 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
pkg/recon/sources/circleci.go
View File

@@ -4,8 +4,13 @@ import (
"context"
"encoding/json"
"fmt"
<<<<<<< HEAD
"net/http"
"strings"
=======
"io"
"net/http"
>>>>>>> worktree-agent-adad8c10
"time"
"golang.org/x/time/rate"
@@ -14,9 +19,16 @@ import (
"github.com/salvacybersec/keyhunter/pkg/recon"
)
<<<<<<< HEAD
// CircleCISource searches public CircleCI build logs for leaked API keys.
// It queries the CircleCI v2 API for recent pipeline workflows. Requires a
// CircleCI API token for authenticated access.
=======
// CircleCISource scrapes CircleCI build logs for leaked API keys.
// CircleCI exposes build logs via its API; a personal API token is required
// to access build artifacts and logs. Misconfigured pipelines often leak
// secrets in build output.
>>>>>>> worktree-agent-adad8c10
type CircleCISource struct {
Token string
BaseURL string
@@ -27,6 +39,7 @@ type CircleCISource struct {
var _ recon.ReconSource = (*CircleCISource)(nil)
<<<<<<< HEAD
func (s *CircleCISource) Name() string { return "circleci" }
func (s *CircleCISource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) }
func (s *CircleCISource) Burst() int { return 2 }
@@ -51,6 +64,27 @@ func (s *CircleCISource) Sweep(ctx context.Context, _ string, out chan<- recon.F
if s.Token == "" {
return nil
}
=======
func (s *CircleCISource) Name() string { return "circleci" }
func (s *CircleCISource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
func (s *CircleCISource) Burst() int { return 2 }
func (s *CircleCISource) RespectsRobots() bool { return false }
// Enabled requires a CircleCI API token.
func (s *CircleCISource) Enabled(_ recon.Config) bool { return s.Token != "" }
// circleciPipelineResponse represents the CircleCI v2 pipeline search result.
type circleciPipelineResponse struct {
Items []circleciPipeline `json:"items"`
}
type circleciPipeline struct {
ID string `json:"id"`
Number int `json:"number"`
}
func (s *CircleCISource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
>>>>>>> worktree-agent-adad8c10
base := s.BaseURL
if base == "" {
base = "https://circleci.com/api/v2"
@@ -61,28 +95,47 @@ func (s *CircleCISource) Sweep(ctx context.Context, _ string, out chan<- recon.F
}
queries := BuildQueries(s.Registry, "circleci")
<<<<<<< HEAD
kwIndex := circleKeywordIndex(s.Registry)
=======
if len(queries) == 0 {
return nil
}
>>>>>>> worktree-agent-adad8c10
for _, q := range queries {
if err := ctx.Err(); err != nil {
return err
}
<<<<<<< HEAD
=======
>>>>>>> worktree-agent-adad8c10
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
<<<<<<< HEAD
endpoint := fmt.Sprintf("%s/pipeline?mine=false", base)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return fmt.Errorf("circleci: build request: %w", err)
=======
// Search for pipelines by project slug (query is used as slug hint).
searchURL := fmt.Sprintf("%s/project/gh/%s/pipeline?limit=5", base, q)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
if err != nil {
continue
>>>>>>> worktree-agent-adad8c10
}
req.Header.Set("Circle-Token", s.Token)
req.Header.Set("Accept", "application/json")
resp, err := client.Do(ctx, req)
if err != nil {
<<<<<<< HEAD
if strings.Contains(err.Error(), "unauthorized") {
return err
}
@@ -113,11 +166,63 @@ func (s *CircleCISource) Sweep(ctx context.Context, _ string, out chan<- recon.F
case out <- f:
case <-ctx.Done():
return ctx.Err()
=======
continue
}
var pipelines circleciPipelineResponse
if err := json.NewDecoder(resp.Body).Decode(&pipelines); err != nil {
_ = resp.Body.Close()
continue
}
_ = resp.Body.Close()
for _, p := range pipelines.Items {
if err := ctx.Err(); err != nil {
return err
}
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
// Fetch pipeline workflow logs.
logURL := fmt.Sprintf("%s/pipeline/%s/workflow", base, p.ID)
logReq, err := http.NewRequestWithContext(ctx, http.MethodGet, logURL, nil)
if err != nil {
continue
}
logReq.Header.Set("Circle-Token", s.Token)
logReq.Header.Set("Accept", "text/plain")
logResp, err := client.Do(ctx, logReq)
if err != nil {
continue
}
body, err := io.ReadAll(io.LimitReader(logResp.Body, 256*1024))
_ = logResp.Body.Close()
if err != nil {
continue
}
if ciLogKeyPattern.Match(body) {
out <- recon.Finding{
ProviderName: q,
Source: logURL,
SourceType: "recon:circleci",
Confidence: "medium",
DetectedAt: time.Now(),
}
>>>>>>> worktree-agent-adad8c10
}
}
}
return nil
}
<<<<<<< HEAD
func circleKeywordIndex(reg *providers.Registry) map[string]string {
m := make(map[string]string)
@@ -136,3 +241,5 @@ func circleKeywordIndex(reg *providers.Registry) map[string]string {
}
return m
}
=======
>>>>>>> worktree-agent-adad8c10

64
pkg/recon/sources/circleci_test.go
View File

@@ -11,6 +11,7 @@ import (
"github.com/salvacybersec/keyhunter/pkg/recon"
)
<<<<<<< HEAD
const circleFixtureJSON = `{
"items": [
{
@@ -55,11 +56,67 @@ func TestCircleCI_Sweep_ExtractsFindings(t *testing.T) {
t.Fatalf("Sweep err: %v", err)
}
close(out)
=======
func TestCircleCI_Name(t *testing.T) {
s := &CircleCISource{}
if s.Name() != "circleci" {
t.Fatalf("expected circleci, got %s", s.Name())
}
}
func TestCircleCI_Enabled(t *testing.T) {
s := &CircleCISource{}
if s.Enabled(recon.Config{}) {
t.Fatal("should be disabled without token")
}
s.Token = "cci-test"
if !s.Enabled(recon.Config{}) {
t.Fatal("should be enabled with token")
}
}
func TestCircleCI_Sweep(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/project/gh/", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"items":[{"id":"pipe-abc-123","number":42}]}`))
})
mux.HandleFunc("/pipeline/pipe-abc-123/workflow", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`Build step: npm test
Setting SECRET_KEY="sk-proj-CIRCLELEAK12345678"
Tests completed successfully`))
})
srv := httptest.NewServer(mux)
defer srv.Close()
reg := providers.NewRegistryFromProviders([]providers.Provider{
{Name: "openai", Keywords: []string{"sk-proj-"}},
})
s := &CircleCISource{
Token: "cci-test",
BaseURL: srv.URL,
Registry: reg,
Client: NewClient(),
}
out := make(chan recon.Finding, 10)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err := s.Sweep(ctx, "", out)
close(out)
if err != nil {
t.Fatalf("Sweep error: %v", err)
}
>>>>>>> worktree-agent-adad8c10
var findings []recon.Finding
for f := range out {
findings = append(findings, f)
}
<<<<<<< HEAD
if len(findings) != 2 {
t.Fatalf("expected 2 findings, got %d", len(findings))
}
@@ -98,5 +155,12 @@ func TestCircleCI_NameAndRate(t *testing.T) {
}
if s.RespectsRobots() {
t.Error("expected RespectsRobots=false")
=======
if len(findings) == 0 {
t.Fatal("expected at least one finding from CircleCI pipeline log")
}
if findings[0].SourceType != "recon:circleci" {
t.Fatalf("expected recon:circleci, got %s", findings[0].SourceType)
>>>>>>> worktree-agent-adad8c10
}
}

93
pkg/recon/sources/commoncrawl.go
View File

@@ -1,12 +1,21 @@
package sources
import (
<<<<<<< HEAD
"bufio"
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
=======
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
>>>>>>> worktree-agent-adad8c10
"time"
"golang.org/x/time/rate"
@@ -15,6 +24,7 @@ import (
"github.com/salvacybersec/keyhunter/pkg/recon"
)
<<<<<<< HEAD
// CommonCrawlSource implements recon.ReconSource against the CommonCrawl
// Index Server API. It queries index.commoncrawl.org for pages matching
// provider keywords in the CC index.
@@ -57,6 +67,41 @@ func (s *CommonCrawlSource) Sweep(ctx context.Context, _ string, out chan<- reco
idx := s.IndexName
if idx == "" {
idx = "CC-MAIN-2024-10"
=======
// CommonCrawlSource searches the Common Crawl index for web pages that may
// contain leaked API keys. Common Crawl archives petabytes of web content;
// its CDX API allows searching by URL pattern to find pages that historically
// exposed secrets.
type CommonCrawlSource struct {
BaseURL string
Registry *providers.Registry
Limiters *recon.LimiterRegistry
Client *Client
}
var _ recon.ReconSource = (*CommonCrawlSource)(nil)
func (s *CommonCrawlSource) Name() string { return "commoncrawl" }
func (s *CommonCrawlSource) RateLimit() rate.Limit { return rate.Every(5 * time.Second) }
func (s *CommonCrawlSource) Burst() int { return 1 }
func (s *CommonCrawlSource) RespectsRobots() bool { return true }
func (s *CommonCrawlSource) Enabled(_ recon.Config) bool { return true }
// ccIndexResult represents a single Common Crawl CDX index record.
type ccIndexResult struct {
URL string `json:"url"`
Timestamp string `json:"timestamp"`
Status string `json:"status"`
Filename string `json:"filename"`
Length string `json:"length"`
Offset string `json:"offset"`
}
func (s *CommonCrawlSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
base := s.BaseURL
if base == "" {
base = "https://index.commoncrawl.org/CC-MAIN-2024-10-index"
>>>>>>> worktree-agent-adad8c10
}
client := s.Client
if client == nil {
@@ -79,17 +124,26 @@ func (s *CommonCrawlSource) Sweep(ctx context.Context, _ string, out chan<- reco
}
}
<<<<<<< HEAD
// CC Index API: output=json returns NDJSON, limit=50 bounds the response.
endpoint := fmt.Sprintf("%s/%s-index?url=*&output=json&limit=50&filter=status:200&query=%s",
base, idx, url.QueryEscape(q))
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return fmt.Errorf("commoncrawl: build req: %w", err)
=======
// CDX API: search for URLs matching the query.
searchURL := fmt.Sprintf("%s?url=*%s*&output=json&limit=10", base, q)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
if err != nil {
continue
>>>>>>> worktree-agent-adad8c10
}
req.Header.Set("Accept", "application/json")
resp, err := client.Do(ctx, req)
if err != nil {
<<<<<<< HEAD
// Non-fatal: skip this keyword on transient errors.
continue
}
@@ -112,10 +166,43 @@ func (s *CommonCrawlSource) Sweep(ctx context.Context, _ string, out chan<- reco
f := recon.Finding{
ProviderName: "",
Source: rec.URL,
=======
continue
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 128*1024))
_ = resp.Body.Close()
if err != nil {
continue
}
// Common Crawl returns NDJSON (newline-delimited JSON).
// Parse each line as a separate JSON object.
var results []ccIndexResult
dec := json.NewDecoder(bytes.NewReader(body))
for dec.More() {
var r ccIndexResult
if err := dec.Decode(&r); err != nil {
break
}
results = append(results, r)
}
for _, r := range results {
if err := ctx.Err(); err != nil {
return err
}
// Each indexed URL is a potential leak location; emit as finding.
out <- recon.Finding{
ProviderName: q,
Source: r.URL,
>>>>>>> worktree-agent-adad8c10
SourceType: "recon:commoncrawl",
Confidence: "low",
DetectedAt: time.Now(),
}
<<<<<<< HEAD
select {
case out <- f:
case <-ctx.Done():
@@ -136,3 +223,9 @@ type ccIndexRecord struct {
Timestamp string `json:"timestamp"`
Status string `json:"status"`
}
=======
}
}
return nil
}
>>>>>>> worktree-agent-adad8c10

67
pkg/recon/sources/commoncrawl_test.go
View File

@@ -2,6 +2,7 @@ package sources
import (
"context"
<<<<<<< HEAD
"encoding/json"
"errors"
"net/http"
@@ -164,5 +165,71 @@ func TestCommonCrawl_NilRegistryNoError(t *testing.T) {
out := make(chan recon.Finding, 1)
if err := src.Sweep(context.Background(), "", out); err != nil {
t.Fatalf("expected nil, got %v", err)
=======
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/salvacybersec/keyhunter/pkg/providers"
"github.com/salvacybersec/keyhunter/pkg/recon"
)
func TestCommonCrawl_Name(t *testing.T) {
s := &CommonCrawlSource{}
if s.Name() != "commoncrawl" {
t.Fatalf("expected commoncrawl, got %s", s.Name())
}
}
func TestCommonCrawl_Enabled(t *testing.T) {
s := &CommonCrawlSource{}
if !s.Enabled(recon.Config{}) {
t.Fatal("CommonCrawlSource should always be enabled (credentialless)")
}
}
func TestCommonCrawl_Sweep(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
// NDJSON format: one JSON object per line.
_, _ = w.Write([]byte(`{"url":"https://example.com/.env","timestamp":"20240101000000","status":"200","filename":"CC-MAIN-2024.warc.gz","length":"1234","offset":"5678"}
`))
})
srv := httptest.NewServer(mux)
defer srv.Close()
reg := providers.NewRegistryFromProviders([]providers.Provider{
{Name: "openai", Keywords: []string{"sk-proj-"}},
})
s := &CommonCrawlSource{
BaseURL: srv.URL,
Registry: reg,
Client: NewClient(),
}
out := make(chan recon.Finding, 10)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err := s.Sweep(ctx, "", out)
close(out)
if err != nil {
t.Fatalf("Sweep error: %v", err)
}
var findings []recon.Finding
for f := range out {
findings = append(findings, f)
}
if len(findings) == 0 {
t.Fatal("expected at least one finding from Common Crawl index")
}
if findings[0].SourceType != "recon:commoncrawl" {
t.Fatalf("expected recon:commoncrawl, got %s", findings[0].SourceType)
>>>>>>> worktree-agent-adad8c10
}
}

122
pkg/recon/sources/githubactions.go
View File

@@ -4,9 +4,14 @@ import (
"context"
"encoding/json"
"fmt"
<<<<<<< HEAD
"net/http"
"net/url"
"strings"
=======
"io"
"net/http"
>>>>>>> worktree-agent-adad8c10
"time"
"golang.org/x/time/rate"
@@ -16,19 +21,30 @@ import (
)
// GitHubActionsSource searches GitHub Actions workflow run logs for leaked API
<<<<<<< HEAD
// keys. It queries the GitHub REST API for workflow runs matching provider
// keywords (via the repository search endpoint) and emits findings for each
// matching run. Requires a GitHub token (same as GitHubSource).
=======
// keys. Workflow logs are public for public repositories and frequently contain
// accidentally printed secrets, debug output with credentials, or insecure
// echo statements that expose environment variables.
>>>>>>> worktree-agent-adad8c10
type GitHubActionsSource struct {
Token string
BaseURL string
Registry *providers.Registry
Limiters *recon.LimiterRegistry
<<<<<<< HEAD
client *Client
=======
Client *Client
>>>>>>> worktree-agent-adad8c10
}
var _ recon.ReconSource = (*GitHubActionsSource)(nil)
<<<<<<< HEAD
func (s *GitHubActionsSource) Name() string { return "github_actions" }
func (s *GitHubActionsSource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) }
func (s *GitHubActionsSource) Burst() int { return 2 }
@@ -54,27 +70,68 @@ func (s *GitHubActionsSource) Sweep(ctx context.Context, _ string, out chan<- re
if s.Token == "" {
return nil
}
=======
func (s *GitHubActionsSource) Name() string { return "ghactions" }
func (s *GitHubActionsSource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) }
func (s *GitHubActionsSource) Burst() int { return 3 }
func (s *GitHubActionsSource) RespectsRobots() bool { return false }
// Enabled requires a GitHub token (reuses GitHubToken from SourcesConfig).
func (s *GitHubActionsSource) Enabled(_ recon.Config) bool { return s.Token != "" }
// ghActionsRunsResponse represents the GitHub Actions workflow runs list.
type ghActionsRunsResponse struct {
WorkflowRuns []ghActionsRun `json:"workflow_runs"`
}
type ghActionsRun struct {
ID int64 `json:"id"`
LogsURL string `json:"logs_url"`
HTMLURL string `json:"html_url"`
Status string `json:"status"`
Conclusion string `json:"conclusion"`
}
func (s *GitHubActionsSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
>>>>>>> worktree-agent-adad8c10
base := s.BaseURL
if base == "" {
base = "https://api.github.com"
}
<<<<<<< HEAD
if s.client == nil {
s.client = NewClient()
}
queries := BuildQueries(s.Registry, "github_actions")
kwIndex := ghActionsKeywordIndex(s.Registry)
=======
client := s.Client
if client == nil {
client = NewClient()
}
queries := BuildQueries(s.Registry, "ghactions")
if len(queries) == 0 {
return nil
}
>>>>>>> worktree-agent-adad8c10
for _, q := range queries {
if err := ctx.Err(); err != nil {
return err
}
<<<<<<< HEAD
=======
>>>>>>> worktree-agent-adad8c10
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
<<<<<<< HEAD
// Search for workflow YAML files referencing the keyword.
endpoint := fmt.Sprintf("%s/search/code?q=%s+path:.github/workflows+extension:yml&per_page=20",
base, url.QueryEscape(q))
@@ -113,11 +170,74 @@ func (s *GitHubActionsSource) Sweep(ctx context.Context, _ string, out chan<- re
case out <- f:
case <-ctx.Done():
return ctx.Err()
=======
// Search for workflow runs via the Actions API.
searchURL := fmt.Sprintf("%s/search/code?q=%s+path:.github/workflows", base, q)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
if err != nil {
continue
}
req.Header.Set("Authorization", "Bearer "+s.Token)
req.Header.Set("Accept", "application/vnd.github.v3+json")
resp, err := client.Do(ctx, req)
if err != nil {
continue
}
var runs ghActionsRunsResponse
if err := json.NewDecoder(resp.Body).Decode(&runs); err != nil {
_ = resp.Body.Close()
continue
}
_ = resp.Body.Close()
for _, run := range runs.WorkflowRuns {
if err := ctx.Err(); err != nil {
return err
}
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
// Fetch log content.
logURL := fmt.Sprintf("%s/actions/runs/%d/logs", base, run.ID)
logReq, err := http.NewRequestWithContext(ctx, http.MethodGet, logURL, nil)
if err != nil {
continue
}
logReq.Header.Set("Authorization", "Bearer "+s.Token)
logReq.Header.Set("Accept", "application/vnd.github.v3+json")
logResp, err := client.Do(ctx, logReq)
if err != nil {
continue
}
body, err := io.ReadAll(io.LimitReader(logResp.Body, 256*1024))
_ = logResp.Body.Close()
if err != nil {
continue
}
if ciLogKeyPattern.Match(body) {
out <- recon.Finding{
ProviderName: q,
Source: logURL,
SourceType: "recon:ghactions",
Confidence: "medium",
DetectedAt: time.Now(),
}
>>>>>>> worktree-agent-adad8c10
}
}
}
return nil
}
<<<<<<< HEAD
func ghActionsKeywordIndex(reg *providers.Registry) map[string]string {
m := make(map[string]string)
@@ -136,3 +256,5 @@ func ghActionsKeywordIndex(reg *providers.Registry) map[string]string {
}
return m
}
=======
>>>>>>> worktree-agent-adad8c10

73
pkg/recon/sources/githubactions_test.go
View File

@@ -2,6 +2,11 @@ package sources
import (
"context"
<<<<<<< HEAD
=======
"encoding/json"
"fmt"
>>>>>>> worktree-agent-adad8c10
"net/http"
"net/http/httptest"
"testing"
@@ -11,6 +16,7 @@ import (
"github.com/salvacybersec/keyhunter/pkg/recon"
)
<<<<<<< HEAD
const ghActionsFixtureJSON = `{
"items": [
{
@@ -55,11 +61,71 @@ func TestGitHubActions_Sweep_ExtractsFindings(t *testing.T) {
t.Fatalf("Sweep err: %v", err)
}
close(out)
=======
func TestGitHubActions_Name(t *testing.T) {
s := &GitHubActionsSource{}
if s.Name() != "ghactions" {
t.Fatalf("expected ghactions, got %s", s.Name())
}
}
func TestGitHubActions_Enabled(t *testing.T) {
s := &GitHubActionsSource{}
if s.Enabled(recon.Config{}) {
t.Fatal("should be disabled without token")
}
s.Token = "ghp-test"
if !s.Enabled(recon.Config{}) {
t.Fatal("should be enabled with token")
}
}
func TestGitHubActions_Sweep(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/search/code", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(ghActionsRunsResponse{
WorkflowRuns: []ghActionsRun{
{ID: 42, Status: "completed", Conclusion: "success"},
},
})
})
mux.HandleFunc("/actions/runs/42/logs", func(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprint(w, `2024-01-01T00:00:00Z Run setup
Setting env: API_KEY="sk-proj-LEAKED1234567890"
Tests passed.`)
})
srv := httptest.NewServer(mux)
defer srv.Close()
reg := providers.NewRegistryFromProviders([]providers.Provider{
{Name: "openai", Keywords: []string{"sk-proj-"}},
})
s := &GitHubActionsSource{
Token: "ghp-test",
BaseURL: srv.URL,
Registry: reg,
Client: NewClient(),
}
out := make(chan recon.Finding, 10)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err := s.Sweep(ctx, "", out)
close(out)
if err != nil {
t.Fatalf("Sweep error: %v", err)
}
>>>>>>> worktree-agent-adad8c10
var findings []recon.Finding
for f := range out {
findings = append(findings, f)
}
<<<<<<< HEAD
if len(findings) != 2 {
t.Fatalf("expected 2 findings, got %d", len(findings))
}
@@ -106,5 +172,12 @@ func TestGitHubActions_NameAndRate(t *testing.T) {
}
if s.RespectsRobots() {
t.Error("expected RespectsRobots=false")
=======
if len(findings) == 0 {
t.Fatal("expected at least one finding from GitHub Actions logs")
}
if findings[0].SourceType != "recon:ghactions" {
t.Fatalf("expected recon:ghactions, got %s", findings[0].SourceType)
>>>>>>> worktree-agent-adad8c10
}
}

157
pkg/recon/sources/integration_test.go
View File

@@ -16,10 +16,11 @@ import (
// TestIntegration_AllSources_SweepAll spins up a single multiplexed httptest
// server that serves canned fixtures for every Phase 10 code-hosting source,
// Phase 11 search engine / paste site source, Phase 12 IoT scanner / cloud
// storage source, and Phase 13 package registry / container / IaC source,
// registers the sources (with BaseURL overrides pointing at the test server)
// onto a fresh recon.Engine, runs SweepAll, and asserts at least one Finding
// was emitted per SourceType across all 40 sources.
// storage source, Phase 13 package registry / container / IaC source, and
// Phase 14 CI/CD log / web archive / frontend leak source, registers the
// sources (with BaseURL overrides pointing at the test server) onto a fresh
// recon.Engine, runs SweepAll, and asserts at least one Finding was emitted
// per SourceType across all 52 sources.
//
// RegisterAll cannot be used directly because it wires production URLs; the
// test exercises the same code paths by constructing each source identically
@@ -312,6 +313,92 @@ func TestIntegration_AllSources_SweepAll(t *testing.T) {
_, _ = w.Write([]byte(`{"packages":[{"package_id":"chart-1","name":"leaked-chart","normalized_name":"leaked-chart","repository":{"name":"bitnami","kind":0}}]}`))
})
// ---- Phase 14: SourceMapSource (probes /static/js/main.js.map) ----
mux.HandleFunc("/sourcemaps/static/js/main.js.map", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"sources":["app.js"],"sourcesContent":["const apiKey = \"sk-proj-SOURCEMAPLEAK123\";"]}`))
})
// ---- Phase 14: WebpackSource (probes /static/js/main.js) ----
mux.HandleFunc("/webpack/static/js/main.js", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/javascript")
_, _ = w.Write([]byte(`!function(){var e={NEXT_PUBLIC_API_KEY:"sk-proj-WEBPACKLEAK123456"}}();`))
})
// ---- Phase 14: EnvLeakSource (probes /.env) ----
mux.HandleFunc("/dotenv/.env", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte("OPENAI_API_KEY=sk-proj-ENVLEAK12345678\nDB_HOST=localhost\n"))
})
// ---- Phase 14: SwaggerSource (probes /swagger.json) ----
mux.HandleFunc("/swagger/swagger.json", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"openapi":"3.0.0","paths":{"/api":{"get":{"parameters":[{"name":"api_key","example":"sk-proj-SWAGGERLEAK12345"}]}}}}`))
})
// ---- Phase 14: DeployPreviewSource (probes /) ----
mux.HandleFunc("/deploypreview/", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(`<html><script>window.NEXT_PUBLIC_API_KEY="sk-proj-DEPLOYLEAK12345678"</script></html>`))
})
// ---- Phase 14: TravisCISource /builds + /builds/{id}/log ----
mux.HandleFunc("/travisci/builds", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"builds":[{"id":999,"state":"passed"}]}`))
})
mux.HandleFunc("/travisci/builds/999/log", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`export API_KEY="sk-proj-TRAVISLEAK1234567890"`))
})
// ---- Phase 14: GitHubActionsSource /search/code + /actions/runs/{id}/logs ----
mux.HandleFunc("/ghactions/search/code", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"workflow_runs":[{"id":55,"status":"completed","conclusion":"success"}]}`))
})
mux.HandleFunc("/ghactions/actions/runs/55/logs", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`SECRET_KEY="sk-proj-GHACTIONSLEAK1234567"`))
})
// ---- Phase 14: CircleCISource /project/gh/{slug}/pipeline + /pipeline/{id}/workflow ----
mux.HandleFunc("/circleci/project/gh/", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"items":[{"id":"pipe-test-1","number":1}]}`))
})
mux.HandleFunc("/circleci/pipeline/pipe-test-1/workflow", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`AUTH_TOKEN="sk-proj-CIRCLELEAK1234567890"`))
})
// ---- Phase 14: JenkinsSource /api/json + /job/{name}/lastBuild/consoleText ----
mux.HandleFunc("/jenkins/api/json", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"jobs":[{"name":"build-app","url":"http://jenkins/job/build-app/","color":"blue"}]}`))
})
mux.HandleFunc("/jenkins/job/build-app/lastBuild/consoleText", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`Setting TOKEN="sk-proj-JENKINSLEAK12345678"`))
})
// ---- Phase 14: WaybackMachineSource /cdx/search/cdx + /web/{ts}id_/{url} ----
mux.HandleFunc("/wayback/cdx/search/cdx", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`[["url","timestamp","statuscode"],["https://example.com/.env","20240101000000","200"]]`))
})
mux.HandleFunc("/wayback/web/", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`API_KEY="sk-proj-WAYBACKLEAK12345678"`))
})
// ---- Phase 14: CommonCrawlSource (NDJSON CDX index) ----
mux.HandleFunc("/commoncrawl", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte("{\"url\":\"https://example.com/.env\",\"timestamp\":\"20240101\",\"status\":\"200\",\"filename\":\"warc.gz\",\"length\":\"100\",\"offset\":\"0\"}\n"))
})
// ---- Phase 14: JSBundleSource (probes /static/js/main.js) ----
mux.HandleFunc("/jsbundle/static/js/main.js", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/javascript")
_, _ = w.Write([]byte(`!function(){var c={apiKey:"sk-proj-JSBUNDLELEAK123456789"}}();`))
})
srv := httptest.NewServer(mux)
defer srv.Close()
@@ -550,9 +637,45 @@ func TestIntegration_AllSources_SweepAll(t *testing.T) {
// helm
eng.Register(&HelmSource{BaseURL: srv.URL + "/helm", Registry: reg, Limiters: lim, Client: NewClient()})
// Sanity: all 40 sources registered.
if n := len(eng.List()); n != 40 {
t.Fatalf("expected 40 sources on engine, got %d: %v", n, eng.List())
// --- Phase 14: Frontend leak sources ---
// sourcemaps
eng.Register(&SourceMapSource{BaseURL: srv.URL + "/sourcemaps", Registry: reg, Limiters: nil, Client: NewClient()})
// webpack
eng.Register(&WebpackSource{BaseURL: srv.URL + "/webpack", Registry: reg, Limiters: nil, Client: NewClient()})
// dotenv
eng.Register(&EnvLeakSource{BaseURL: srv.URL + "/dotenv", Registry: reg, Limiters: nil, Client: NewClient()})
// swagger
eng.Register(&SwaggerSource{BaseURL: srv.URL + "/swagger", Registry: reg, Limiters: nil, Client: NewClient()})
// deploypreview
eng.Register(&DeployPreviewSource{BaseURL: srv.URL + "/deploypreview", Registry: reg, Limiters: nil, Client: NewClient()})
// --- Phase 14: CI/CD log sources ---
// travisci
eng.Register(&TravisCISource{BaseURL: srv.URL + "/travisci", Registry: reg, Limiters: nil, Client: NewClient()})
// ghactions
eng.Register(&GitHubActionsSource{Token: "ghp-test", BaseURL: srv.URL + "/ghactions", Registry: reg, Limiters: nil, Client: NewClient()})
// circleci
eng.Register(&CircleCISource{Token: "cci-test", BaseURL: srv.URL + "/circleci", Registry: reg, Limiters: nil, Client: NewClient()})
// jenkins
eng.Register(&JenkinsSource{BaseURL: srv.URL + "/jenkins", Registry: reg, Limiters: nil, Client: NewClient()})
// --- Phase 14: Web archive sources ---
// wayback
eng.Register(&WaybackMachineSource{BaseURL: srv.URL + "/wayback", Registry: reg, Limiters: nil, Client: NewClient()})
// commoncrawl
eng.Register(&CommonCrawlSource{BaseURL: srv.URL + "/commoncrawl", Registry: reg, Limiters: nil, Client: NewClient()})
// --- Phase 14: JS bundle analysis ---
// jsbundle
eng.Register(&JSBundleSource{BaseURL: srv.URL + "/jsbundle", Registry: reg, Limiters: nil, Client: NewClient()})
// Sanity: all 52 sources registered.
if n := len(eng.List()); n != 52 {
t.Fatalf("expected 52 sources on engine, got %d: %v", n, eng.List())
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
@@ -616,6 +739,22 @@ func TestIntegration_AllSources_SweepAll(t *testing.T) {
"recon:k8s",
"recon:terraform",
"recon:helm",
// Phase 14: Frontend leaks
"recon:sourcemaps",
"recon:webpack",
"recon:dotenv",
"recon:swagger",
"recon:deploypreview",
// Phase 14: CI/CD logs
"recon:travisci",
"recon:ghactions",
"recon:circleci",
"recon:jenkins",
// Phase 14: Web archives
"recon:wayback",
"recon:commoncrawl",
// Phase 14: JS bundles
"recon:jsbundle",
}
for _, st := range wantTypes {
if byType[st] == 0 {
@@ -641,8 +780,8 @@ func TestRegisterAll_Phase12(t *testing.T) {
})
names := eng.List()
if n := len(names); n != 45 {
t.Fatalf("expected 45 sources from RegisterAll, got %d: %v", n, names)
if n := len(names); n != 52 {
t.Fatalf("expected 52 sources from RegisterAll, got %d: %v", n, names)
}
// Build lookup for source access.

100
pkg/recon/sources/jenkins.go
View File

@@ -4,8 +4,13 @@ import (
"context"
"encoding/json"
"fmt"
<<<<<<< HEAD
"net/http"
"strings"
=======
"io"
"net/http"
>>>>>>> worktree-agent-adad8c10
"time"
"golang.org/x/time/rate"
@@ -14,10 +19,17 @@ import (
"github.com/salvacybersec/keyhunter/pkg/recon"
)
<<<<<<< HEAD
// JenkinsSource searches publicly accessible Jenkins instances for build
// console output containing leaked API keys. It queries the Jenkins JSON API
// at /api/json to enumerate jobs and their latest builds. Credentialless --
// targets open Jenkins instances discovered via dorking or IoT scanners.
=======
// JenkinsSource scrapes publicly accessible Jenkins build consoles for leaked
// API keys. Many Jenkins instances are exposed to the internet without
// authentication, and build console output frequently contains printed
// environment variables or secrets passed via command-line arguments.
>>>>>>> worktree-agent-adad8c10
type JenkinsSource struct {
BaseURL string
Registry *providers.Registry
@@ -27,17 +39,28 @@ type JenkinsSource struct {
var _ recon.ReconSource = (*JenkinsSource)(nil)
<<<<<<< HEAD
func (s *JenkinsSource) Name() string { return "jenkins" }
func (s *JenkinsSource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
func (s *JenkinsSource) Burst() int { return 1 }
func (s *JenkinsSource) RespectsRobots() bool { return true }
func (s *JenkinsSource) Enabled(_ recon.Config) bool { return true }
=======
func (s *JenkinsSource) Name() string { return "jenkins" }
func (s *JenkinsSource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
func (s *JenkinsSource) Burst() int { return 2 }
func (s *JenkinsSource) RespectsRobots() bool { return true }
func (s *JenkinsSource) Enabled(_ recon.Config) bool { return true }
// jenkinsJobsResponse represents the Jenkins API jobs listing.
>>>>>>> worktree-agent-adad8c10
type jenkinsJobsResponse struct {
Jobs []jenkinsJob `json:"jobs"`
}
type jenkinsJob struct {
<<<<<<< HEAD
Name string `json:"name"`
URL string `json:"url"`
LastBuild *jenkinsBuild `json:"lastBuild"`
@@ -46,12 +69,21 @@ type jenkinsJob struct {
type jenkinsBuild struct {
Number int `json:"number"`
URL string `json:"url"`
=======
Name string `json:"name"`
URL string `json:"url"`
Color string `json:"color"`
>>>>>>> worktree-agent-adad8c10
}
func (s *JenkinsSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
base := s.BaseURL
if base == "" {
<<<<<<< HEAD
base = "https://jenkins.example.com"
=======
return nil // No default; Jenkins instances are discovered via dorking
>>>>>>> worktree-agent-adad8c10
}
client := s.Client
if client == nil {
@@ -59,22 +91,40 @@ func (s *JenkinsSource) Sweep(ctx context.Context, _ string, out chan<- recon.Fi
}
queries := BuildQueries(s.Registry, "jenkins")
<<<<<<< HEAD
kwIndex := jenkinsKeywordIndex(s.Registry)
=======
if len(queries) == 0 {
return nil
}
>>>>>>> worktree-agent-adad8c10
for _, q := range queries {
if err := ctx.Err(); err != nil {
return err
}
<<<<<<< HEAD
=======
>>>>>>> worktree-agent-adad8c10
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
<<<<<<< HEAD
endpoint := fmt.Sprintf("%s/api/json?tree=jobs[name,url,lastBuild[number,url]]", base)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return fmt.Errorf("jenkins: build request: %w", err)
=======
// List jobs from the Jenkins API.
jobsURL := fmt.Sprintf("%s/api/json?tree=jobs[name,url,color]", base)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, jobsURL, nil)
if err != nil {
continue
>>>>>>> worktree-agent-adad8c10
}
req.Header.Set("Accept", "application/json")
@@ -83,6 +133,7 @@ func (s *JenkinsSource) Sweep(ctx context.Context, _ string, out chan<- recon.Fi
continue
}
<<<<<<< HEAD
var result jenkinsJobsResponse
decErr := json.NewDecoder(resp.Body).Decode(&result)
_ = resp.Body.Close()
@@ -110,11 +161,58 @@ func (s *JenkinsSource) Sweep(ctx context.Context, _ string, out chan<- recon.Fi
case out <- f:
case <-ctx.Done():
return ctx.Err()
=======
var jobs jenkinsJobsResponse
if err := json.NewDecoder(resp.Body).Decode(&jobs); err != nil {
_ = resp.Body.Close()
continue
}
_ = resp.Body.Close()
for _, job := range jobs.Jobs {
if err := ctx.Err(); err != nil {
return err
}
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
// Fetch the last build console output.
consoleURL := fmt.Sprintf("%s/job/%s/lastBuild/consoleText", base, job.Name)
consoleReq, err := http.NewRequestWithContext(ctx, http.MethodGet, consoleURL, nil)
if err != nil {
continue
}
consoleResp, err := client.Do(ctx, consoleReq)
if err != nil {
continue
}
body, err := io.ReadAll(io.LimitReader(consoleResp.Body, 256*1024))
_ = consoleResp.Body.Close()
if err != nil {
continue
}
if ciLogKeyPattern.Match(body) {
out <- recon.Finding{
ProviderName: q,
Source: consoleURL,
SourceType: "recon:jenkins",
Confidence: "medium",
DetectedAt: time.Now(),
}
>>>>>>> worktree-agent-adad8c10
}
}
}
return nil
}
<<<<<<< HEAD
func jenkinsKeywordIndex(reg *providers.Registry) map[string]string {
m := make(map[string]string)
@@ -133,3 +231,5 @@ func jenkinsKeywordIndex(reg *providers.Registry) map[string]string {
}
return m
}
=======
>>>>>>> worktree-agent-adad8c10

61
pkg/recon/sources/jenkins_test.go
View File

@@ -11,6 +11,7 @@ import (
"github.com/salvacybersec/keyhunter/pkg/recon"
)
<<<<<<< HEAD
const jenkinsFixtureJSON = `{
"jobs": [
{
@@ -58,11 +59,64 @@ func TestJenkins_Sweep_ExtractsFindings(t *testing.T) {
t.Fatalf("Sweep err: %v", err)
}
close(out)
=======
func TestJenkins_Name(t *testing.T) {
s := &JenkinsSource{}
if s.Name() != "jenkins" {
t.Fatalf("expected jenkins, got %s", s.Name())
}
}
func TestJenkins_Enabled(t *testing.T) {
s := &JenkinsSource{}
if !s.Enabled(recon.Config{}) {
t.Fatal("JenkinsSource should always be enabled (credentialless)")
}
}
func TestJenkins_Sweep(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/json", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"jobs":[{"name":"deploy-prod","url":"http://jenkins/job/deploy-prod/","color":"blue"}]}`))
})
mux.HandleFunc("/job/deploy-prod/lastBuild/consoleText", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`Started by user admin
[Pipeline] echo
Setting AUTH_TOKEN="sk-proj-JENKINSLEAK123456"
[Pipeline] sh
Build SUCCESS`))
})
srv := httptest.NewServer(mux)
defer srv.Close()
reg := providers.NewRegistryFromProviders([]providers.Provider{
{Name: "openai", Keywords: []string{"sk-proj-"}},
})
s := &JenkinsSource{
BaseURL: srv.URL,
Registry: reg,
Client: NewClient(),
}
out := make(chan recon.Finding, 10)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err := s.Sweep(ctx, "", out)
close(out)
if err != nil {
t.Fatalf("Sweep error: %v", err)
}
>>>>>>> worktree-agent-adad8c10
var findings []recon.Finding
for f := range out {
findings = append(findings, f)
}
<<<<<<< HEAD
// 3 jobs but 1 has null lastBuild -> 2 findings.
if len(findings) != 2 {
t.Fatalf("expected 2 findings, got %d", len(findings))
@@ -95,5 +149,12 @@ func TestJenkins_NameAndRate(t *testing.T) {
}
if !s.RespectsRobots() {
t.Error("expected RespectsRobots=true")
=======
if len(findings) == 0 {
t.Fatal("expected at least one finding from Jenkins console output")
}
if findings[0].SourceType != "recon:jenkins" {
t.Fatalf("expected recon:jenkins, got %s", findings[0].SourceType)
>>>>>>> worktree-agent-adad8c10
}
}

116
pkg/recon/sources/jsbundle.go Normal file
View File

@@ -0,0 +1,116 @@
package sources
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
"time"
"golang.org/x/time/rate"
"github.com/salvacybersec/keyhunter/pkg/providers"
"github.com/salvacybersec/keyhunter/pkg/recon"
)
// JSBundleSource analyzes public JavaScript bundles for embedded API keys.
// Modern build tools (Webpack, Vite, esbuild, Rollup) inline environment
// variables and configuration at build time. This source probes common bundle
// paths and scans the minified JS for API key patterns, complementing
// WebpackSource by targeting raw key literals rather than env var prefixes.
type JSBundleSource struct {
BaseURL string
Registry *providers.Registry
Limiters *recon.LimiterRegistry
Client *Client
}
var _ recon.ReconSource = (*JSBundleSource)(nil)
func (s *JSBundleSource) Name() string { return "jsbundle" }
func (s *JSBundleSource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
func (s *JSBundleSource) Burst() int { return 2 }
func (s *JSBundleSource) RespectsRobots() bool { return true }
func (s *JSBundleSource) Enabled(_ recon.Config) bool { return true }
// jsBundleKeyPattern matches literal API key assignments commonly found in
// minified JS bundles (e.g., apiKey:"sk-proj-...", "Authorization":"Bearer sk-...").
var jsBundleKeyPattern = regexp.MustCompile(`(?i)(?:api[_-]?key|secret|token|authorization|bearer)\s*[=:"']+\s*['"]?([a-zA-Z0-9_\-]{20,})['"]?`)
// jsBundlePaths are common locations for production JS bundles.
var jsBundlePaths = []string{
"/static/js/main.js",
"/static/js/app.js",
"/static/js/vendor.js",
"/dist/app.js",
"/dist/main.js",
"/assets/app.js",
"/assets/index.js",
"/js/app.js",
"/_next/static/chunks/main.js",
"/_next/static/chunks/pages/_app.js",
}
func (s *JSBundleSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
base := s.BaseURL
if base == "" {
return nil
}
client := s.Client
if client == nil {
client = NewClient()
}
queries := BuildQueries(s.Registry, "jsbundle")
if len(queries) == 0 {
return nil
}
for _, q := range queries {
if err := ctx.Err(); err != nil {
return err
}
for _, path := range jsBundlePaths {
if err := ctx.Err(); err != nil {
return err
}
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
probeURL := fmt.Sprintf("%s%s", base, path)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, probeURL, nil)
if err != nil {
continue
}
resp, err := client.Do(ctx, req)
if err != nil {
continue
}
body, err := io.ReadAll(io.LimitReader(resp.Body, 1024*1024)) // 1MB max for JS bundles
_ = resp.Body.Close()
if err != nil {
continue
}
if jsBundleKeyPattern.Match(body) {
out <- recon.Finding{
ProviderName: q,
Source: probeURL,
SourceType: "recon:jsbundle",
Confidence: "medium",
DetectedAt: time.Now(),
}
break // one finding per query is sufficient
}
}
}
return nil
}

68
pkg/recon/sources/jsbundle_test.go Normal file
View File

@@ -0,0 +1,68 @@
package sources
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/salvacybersec/keyhunter/pkg/providers"
"github.com/salvacybersec/keyhunter/pkg/recon"
)
func TestJSBundle_Name(t *testing.T) {
s := &JSBundleSource{}
if s.Name() != "jsbundle" {
t.Fatalf("expected jsbundle, got %s", s.Name())
}
}
func TestJSBundle_Enabled(t *testing.T) {
s := &JSBundleSource{}
if !s.Enabled(recon.Config{}) {
t.Fatal("JSBundleSource should always be enabled (credentialless)")
}
}
func TestJSBundle_Sweep(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/static/js/main.js", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/javascript")
_, _ = w.Write([]byte(`!function(e){var t={apiKey:"sk-proj-JSBUNDLELEAK123456789",baseUrl:"https://api.example.com"};e.exports=t}(module);`))
})
srv := httptest.NewServer(mux)
defer srv.Close()
reg := providers.NewRegistryFromProviders([]providers.Provider{
{Name: "openai", Keywords: []string{"sk-proj-"}},
})
s := &JSBundleSource{
BaseURL: srv.URL,
Registry: reg,
Client: NewClient(),
}
out := make(chan recon.Finding, 10)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err := s.Sweep(ctx, "", out)
close(out)
if err != nil {
t.Fatalf("Sweep error: %v", err)
}
var findings []recon.Finding
for f := range out {
findings = append(findings, f)
}
if len(findings) == 0 {
t.Fatal("expected at least one finding from JS bundle")
}
if findings[0].SourceType != "recon:jsbundle" {
t.Fatalf("expected recon:jsbundle, got %s", findings[0].SourceType)
}
}

28
pkg/recon/sources/register.go
View File

@@ -52,6 +52,9 @@ type SourcesConfig struct {
NetlasAPIKey string
BinaryEdgeAPIKey string
// Phase 14: CI/CD source tokens.
CircleCIToken string
// Registry drives query generation for every source via BuildQueries.
Registry *providers.Registry
// Limiters is the shared per-source rate-limiter registry.
@@ -60,8 +63,8 @@ type SourcesConfig struct {
// RegisterAll registers every Phase 10 code-hosting, Phase 11 search engine /
// paste site, Phase 12 IoT scanner / cloud storage, Phase 13 package
// registry / container / IaC, and Phase 14 frontend leak source on engine
// (45 sources total).
// registry / container / IaC, and Phase 14 CI/CD log / web archive /
// frontend leak source on engine (52 sources total).
//
// All sources are registered unconditionally so that cmd/recon.go can surface
// the full catalog via `keyhunter recon list` regardless of which credentials
@@ -239,4 +242,25 @@ func RegisterAll(engine *recon.Engine, cfg SourcesConfig) {
engine.Register(&EnvLeakSource{Registry: reg, Limiters: lim})
engine.Register(&SwaggerSource{Registry: reg, Limiters: lim})
engine.Register(&DeployPreviewSource{Registry: reg, Limiters: lim})
// Phase 14: CI/CD log sources.
engine.Register(&TravisCISource{Registry: reg, Limiters: lim})
engine.Register(&GitHubActionsSource{
Token: cfg.GitHubToken,
Registry: reg,
Limiters: lim,
})
engine.Register(&CircleCISource{
Token: cfg.CircleCIToken,
Registry: reg,
Limiters: lim,
})
engine.Register(&JenkinsSource{Registry: reg, Limiters: lim})
// Phase 14: Web archive sources (credentialless).
engine.Register(&WaybackMachineSource{Registry: reg, Limiters: lim})
engine.Register(&CommonCrawlSource{Registry: reg, Limiters: lim})
// Phase 14: JS bundle analysis (credentialless).
engine.Register(&JSBundleSource{Registry: reg, Limiters: lim})
}

15
pkg/recon/sources/register_test.go
View File

@@ -16,9 +16,9 @@ func registerTestRegistry() *providers.Registry {
})
}
// TestRegisterAll_WiresAllFortyFiveSources asserts that RegisterAll registers
// TestRegisterAll_WiresAllFiftyTwoSources asserts that RegisterAll registers
// every Phase 10-14 source by its stable name on a fresh engine.
func TestRegisterAll_WiresAllFortyFiveSources(t *testing.T) {
func TestRegisterAll_WiresAllFiftyTwoSources(t *testing.T) {
eng := recon.NewEngine()
cfg := SourcesConfig{
Registry: registerTestRegistry(),
@@ -34,8 +34,10 @@ func TestRegisterAll_WiresAllFortyFiveSources(t *testing.T) {
"bitbucket",
"brave",
"censys",
"circleci",
"codeberg",
"codesandbox",
"commoncrawl",
"crates",
"deploypreview",
"dockerhub",
@@ -43,6 +45,7 @@ func TestRegisterAll_WiresAllFortyFiveSources(t *testing.T) {
"duckduckgo",
"fofa",
"gcs",
"ghactions",
"gist",
"gistpaste",
"github",
@@ -51,6 +54,8 @@ func TestRegisterAll_WiresAllFortyFiveSources(t *testing.T) {
"goproxy",
"helm",
"huggingface",
"jenkins",
"jsbundle",
"k8s",
"kaggle",
"maven",
@@ -70,6 +75,8 @@ func TestRegisterAll_WiresAllFortyFiveSources(t *testing.T) {
"spaces",
"swagger",
"terraform",
"travisci",
"wayback",
"webpack",
"yandex",
"zoomeye",
@@ -90,8 +97,8 @@ func TestRegisterAll_MissingCredsStillRegistered(t *testing.T) {
Limiters: recon.NewLimiterRegistry(),
})
if n := len(eng.List()); n != 45 {
t.Fatalf("expected 45 sources registered, got %d: %v", n, eng.List())
if n := len(eng.List()); n != 52 {
t.Fatalf("expected 52 sources registered, got %d: %v", n, eng.List())
}
// SweepAll with an empty config should filter out cred-gated sources

102
pkg/recon/sources/travisci.go
View File

@@ -4,9 +4,15 @@ import (
"context"
"encoding/json"
"fmt"
<<<<<<< HEAD
"net/http"
"net/url"
"strings"
=======
"io"
"net/http"
"regexp"
>>>>>>> worktree-agent-adad8c10
"time"
"golang.org/x/time/rate"
@@ -15,9 +21,16 @@ import (
"github.com/salvacybersec/keyhunter/pkg/recon"
)
<<<<<<< HEAD
// TravisCISource searches public Travis CI build logs for leaked API keys.
// It queries the Travis CI API v3 /builds endpoint for builds matching
// provider keywords. No authentication required for public repositories.
=======
// TravisCISource scrapes public Travis CI build logs for leaked API keys.
// Travis CI exposes build logs publicly by default for open-source projects.
// Developers frequently print environment variables or use secrets insecurely
// in CI scripts, causing API keys to appear in build output.
>>>>>>> worktree-agent-adad8c10
type TravisCISource struct {
BaseURL string
Registry *providers.Registry
@@ -27,17 +40,28 @@ type TravisCISource struct {
var _ recon.ReconSource = (*TravisCISource)(nil)
<<<<<<< HEAD
func (s *TravisCISource) Name() string { return "travisci" }
func (s *TravisCISource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
func (s *TravisCISource) Burst() int { return 1 }
func (s *TravisCISource) RespectsRobots() bool { return true }
func (s *TravisCISource) Enabled(_ recon.Config) bool { return true }
=======
func (s *TravisCISource) Name() string { return "travisci" }
func (s *TravisCISource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
func (s *TravisCISource) Burst() int { return 2 }
func (s *TravisCISource) RespectsRobots() bool { return false }
func (s *TravisCISource) Enabled(_ recon.Config) bool { return true }
// travisBuildResponse represents the Travis CI API builds response.
>>>>>>> worktree-agent-adad8c10
type travisBuildResponse struct {
Builds []travisBuild `json:"builds"`
}
type travisBuild struct {
<<<<<<< HEAD
ID int `json:"id"`
State string `json:"state"`
Repository travisRepository `json:"repository"`
@@ -46,6 +70,14 @@ type travisBuild struct {
type travisRepository struct {
Slug string `json:"slug"`
}
=======
ID int `json:"id"`
State string `json:"state"`
}
// ciLogKeyPattern matches API key patterns commonly leaked in CI logs.
var ciLogKeyPattern = regexp.MustCompile(`(?i)(api[_-]?key|secret[_-]?key|token|password|credential|auth[_-]?token)['":\s]*[=:]\s*['"]?([a-zA-Z0-9_\-]{16,})['"]?`)
>>>>>>> worktree-agent-adad8c10
func (s *TravisCISource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
base := s.BaseURL
@@ -58,23 +90,41 @@ func (s *TravisCISource) Sweep(ctx context.Context, _ string, out chan<- recon.F
}
queries := BuildQueries(s.Registry, "travisci")
<<<<<<< HEAD
kwIndex := travisKeywordIndex(s.Registry)
=======
if len(queries) == 0 {
return nil
}
>>>>>>> worktree-agent-adad8c10
for _, q := range queries {
if err := ctx.Err(); err != nil {
return err
}
<<<<<<< HEAD
=======
>>>>>>> worktree-agent-adad8c10
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
<<<<<<< HEAD
endpoint := fmt.Sprintf("%s/builds?limit=20&sort_by=finished_at:desc&state=passed&event_type=push",
base)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return fmt.Errorf("travisci: build request: %w", err)
=======
// Search for builds related to the query keyword.
searchURL := fmt.Sprintf("%s/builds?search=%s&limit=5", base, q)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
if err != nil {
continue
>>>>>>> worktree-agent-adad8c10
}
req.Header.Set("Travis-API-Version", "3")
req.Header.Set("Accept", "application/json")
@@ -84,6 +134,7 @@ func (s *TravisCISource) Sweep(ctx context.Context, _ string, out chan<- recon.F
continue
}
<<<<<<< HEAD
var result travisBuildResponse
decErr := json.NewDecoder(resp.Body).Decode(&result)
_ = resp.Body.Close()
@@ -106,11 +157,60 @@ func (s *TravisCISource) Sweep(ctx context.Context, _ string, out chan<- recon.F
case out <- f:
case <-ctx.Done():
return ctx.Err()
=======
var builds travisBuildResponse
if err := json.NewDecoder(resp.Body).Decode(&builds); err != nil {
_ = resp.Body.Close()
continue
}
_ = resp.Body.Close()
for _, b := range builds.Builds {
if err := ctx.Err(); err != nil {
return err
}
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
// Fetch the build log.
logURL := fmt.Sprintf("%s/builds/%d/log", base, b.ID)
logReq, err := http.NewRequestWithContext(ctx, http.MethodGet, logURL, nil)
if err != nil {
continue
}
logReq.Header.Set("Travis-API-Version", "3")
logReq.Header.Set("Accept", "text/plain")
logResp, err := client.Do(ctx, logReq)
if err != nil {
continue
}
body, err := io.ReadAll(io.LimitReader(logResp.Body, 256*1024))
_ = logResp.Body.Close()
if err != nil {
continue
}
if ciLogKeyPattern.Match(body) {
out <- recon.Finding{
ProviderName: q,
Source: logURL,
SourceType: "recon:travisci",
Confidence: "medium",
DetectedAt: time.Now(),
}
>>>>>>> worktree-agent-adad8c10
}
}
}
return nil
}
<<<<<<< HEAD
func travisKeywordIndex(reg *providers.Registry) map[string]string {
m := make(map[string]string)
@@ -129,3 +229,5 @@ func travisKeywordIndex(reg *providers.Registry) map[string]string {
}
return m
}
=======
>>>>>>> worktree-agent-adad8c10

60
pkg/recon/sources/travisci_test.go
View File

@@ -11,6 +11,7 @@ import (
"github.com/salvacybersec/keyhunter/pkg/recon"
)
<<<<<<< HEAD
const travisFixtureJSON = `{
"builds": [
{
@@ -56,11 +57,63 @@ func TestTravisCI_Sweep_ExtractsFindings(t *testing.T) {
t.Fatalf("Sweep err: %v", err)
}
close(out)
=======
func TestTravisCI_Name(t *testing.T) {
s := &TravisCISource{}
if s.Name() != "travisci" {
t.Fatalf("expected travisci, got %s", s.Name())
}
}
func TestTravisCI_Enabled(t *testing.T) {
s := &TravisCISource{}
if !s.Enabled(recon.Config{}) {
t.Fatal("TravisCISource should always be enabled (credentialless)")
}
}
func TestTravisCI_Sweep(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/builds", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"builds":[{"id":123,"state":"passed"}]}`))
})
mux.HandleFunc("/builds/123/log", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
_, _ = w.Write([]byte(`Setting environment variables
export API_KEY="sk-proj-ABCDEF1234567890"
Running tests...`))
})
srv := httptest.NewServer(mux)
defer srv.Close()
reg := providers.NewRegistryFromProviders([]providers.Provider{
{Name: "openai", Keywords: []string{"sk-proj-"}},
})
s := &TravisCISource{
BaseURL: srv.URL,
Registry: reg,
Client: NewClient(),
}
out := make(chan recon.Finding, 10)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err := s.Sweep(ctx, "", out)
close(out)
if err != nil {
t.Fatalf("Sweep error: %v", err)
}
>>>>>>> worktree-agent-adad8c10
var findings []recon.Finding
for f := range out {
findings = append(findings, f)
}
<<<<<<< HEAD
if len(findings) != 2 {
t.Fatalf("expected 2 findings, got %d", len(findings))
}
@@ -88,5 +141,12 @@ func TestTravisCI_NameAndRate(t *testing.T) {
}
if !s.RespectsRobots() {
t.Error("expected RespectsRobots=true")
=======
if len(findings) == 0 {
t.Fatal("expected at least one finding from Travis CI build log")
}
if findings[0].SourceType != "recon:travisci" {
t.Fatalf("expected recon:travisci, got %s", findings[0].SourceType)
>>>>>>> worktree-agent-adad8c10
}
}

103
pkg/recon/sources/wayback.go
View File

@@ -1,12 +1,20 @@
package sources
import (
<<<<<<< HEAD
"bufio"
"context"
"fmt"
"net/http"
"net/url"
"strings"
=======
"context"
"encoding/json"
"fmt"
"io"
"net/http"
>>>>>>> worktree-agent-adad8c10
"time"
"golang.org/x/time/rate"
@@ -15,6 +23,7 @@ import (
"github.com/salvacybersec/keyhunter/pkg/recon"
)
<<<<<<< HEAD
// WaybackMachineSource implements recon.ReconSource against the Wayback Machine
// CDX Server API. It queries web.archive.org/cdx/search/cdx for historical
// snapshots of pages matching provider keywords (e.g. domains known to host
@@ -48,6 +57,26 @@ func (s *WaybackMachineSource) Enabled(_ recon.Config) bool { return true }
// a Finding for every archived snapshot URL returned. The CDX API returns
// plain-text lines with space-separated fields; we extract the original URL
// and timestamp to construct the full Wayback snapshot link.
=======
// WaybackMachineSource searches the Internet Archive's Wayback Machine CDX API
// for archived pages that may contain leaked API keys. Developers sometimes
// remove secrets from live pages but cached versions persist in web archives.
type WaybackMachineSource struct {
BaseURL string
Registry *providers.Registry
Limiters *recon.LimiterRegistry
Client *Client
}
var _ recon.ReconSource = (*WaybackMachineSource)(nil)
func (s *WaybackMachineSource) Name() string { return "wayback" }
func (s *WaybackMachineSource) RateLimit() rate.Limit { return rate.Every(5 * time.Second) }
func (s *WaybackMachineSource) Burst() int { return 1 }
func (s *WaybackMachineSource) RespectsRobots() bool { return true }
func (s *WaybackMachineSource) Enabled(_ recon.Config) bool { return true }
>>>>>>> worktree-agent-adad8c10
func (s *WaybackMachineSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
base := s.BaseURL
if base == "" {
@@ -74,6 +103,7 @@ func (s *WaybackMachineSource) Sweep(ctx context.Context, _ string, out chan<- r
}
}
<<<<<<< HEAD
// CDX API: output=text, fl=timestamp,original limits response to two fields per line.
// limit=50 keeps the response bounded per keyword.
endpoint := fmt.Sprintf("%s/cdx/search/cdx?url=*&output=text&fl=timestamp,original&limit=50&matchType=prefix&filter=statuscode:200&query=%s",
@@ -121,6 +151,79 @@ func (s *WaybackMachineSource) Sweep(ctx context.Context, _ string, out chan<- r
}
}
_ = resp.Body.Close()
=======
// CDX API: search for archived URLs matching the query.
// Filter for .env, config, and JS files that commonly contain keys.
cdxURL := fmt.Sprintf("%s/cdx/search/cdx?url=*%s*&output=json&limit=10&fl=url,timestamp,statuscode&filter=statuscode:200", base, q)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, cdxURL, nil)
if err != nil {
continue
}
req.Header.Set("Accept", "application/json")
resp, err := client.Do(ctx, req)
if err != nil {
continue
}
var rows [][]string
if err := json.NewDecoder(resp.Body).Decode(&rows); err != nil {
_ = resp.Body.Close()
continue
}
_ = resp.Body.Close()
// Skip the header row if present.
start := 0
if len(rows) > 0 && len(rows[0]) > 0 && rows[0][0] == "url" {
start = 1
}
for _, row := range rows[start:] {
if err := ctx.Err(); err != nil {
return err
}
if len(row) < 2 {
continue
}
archivedURL := row[0]
timestamp := row[1]
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
// Fetch the archived page content.
snapshotURL := fmt.Sprintf("%s/web/%sid_/%s", base, timestamp, archivedURL)
snapReq, err := http.NewRequestWithContext(ctx, http.MethodGet, snapshotURL, nil)
if err != nil {
continue
}
snapResp, err := client.Do(ctx, snapReq)
if err != nil {
continue
}
body, err := io.ReadAll(io.LimitReader(snapResp.Body, 256*1024))
_ = snapResp.Body.Close()
if err != nil {
continue
}
if apiKeyPattern.Match(body) {
out <- recon.Finding{
ProviderName: q,
Source: snapshotURL,
SourceType: "recon:wayback",
Confidence: "medium",
DetectedAt: time.Now(),
}
}
}
>>>>>>> worktree-agent-adad8c10
}
return nil
}

68
pkg/recon/sources/wayback_test.go
View File

@@ -2,6 +2,7 @@ package sources
import (
"context"
<<<<<<< HEAD
"errors"
"fmt"
"net/http"
@@ -164,5 +165,72 @@ func TestWayback_NilRegistryNoError(t *testing.T) {
out := make(chan recon.Finding, 1)
if err := src.Sweep(context.Background(), "", out); err != nil {
t.Fatalf("expected nil, got %v", err)
=======
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/salvacybersec/keyhunter/pkg/providers"
"github.com/salvacybersec/keyhunter/pkg/recon"
)
func TestWayback_Name(t *testing.T) {
s := &WaybackMachineSource{}
if s.Name() != "wayback" {
t.Fatalf("expected wayback, got %s", s.Name())
}
}
func TestWayback_Enabled(t *testing.T) {
s := &WaybackMachineSource{}
if !s.Enabled(recon.Config{}) {
t.Fatal("WaybackMachineSource should always be enabled (credentialless)")
}
}
func TestWayback_Sweep(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/cdx/search/cdx", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`[["url","timestamp","statuscode"],["https://example.com/.env","20240101000000","200"]]`))
})
mux.HandleFunc("/web/", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`OPENAI_API_KEY="sk-proj-WAYBACKLEAK12345678"`))
})
srv := httptest.NewServer(mux)
defer srv.Close()
reg := providers.NewRegistryFromProviders([]providers.Provider{
{Name: "openai", Keywords: []string{"sk-proj-"}},
})
s := &WaybackMachineSource{
BaseURL: srv.URL,
Registry: reg,
Client: NewClient(),
}
out := make(chan recon.Finding, 10)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err := s.Sweep(ctx, "", out)
close(out)
if err != nil {
t.Fatalf("Sweep error: %v", err)
}
var findings []recon.Finding
for f := range out {
findings = append(findings, f)
}
if len(findings) == 0 {
t.Fatal("expected at least one finding from Wayback Machine archives")
}
if findings[0].SourceType != "recon:wayback" {
t.Fatalf("expected recon:wayback, got %s", findings[0].SourceType)
>>>>>>> worktree-agent-adad8c10
}
}