feat(14-01): add 5 CI/CD log sources (GitHubActions, TravisCI, CircleCI, Jenkins, GitLabCI)

- GitHubActionsSource: searches GitHub code search for workflow files with provider keywords (token-gated)
- TravisCISource: queries Travis CI v3 API for public build logs (credentialless)
- CircleCISource: queries CircleCI v2 pipeline API for build pipelines (token-gated)
- JenkinsSource: queries open Jenkins /api/json for job build consoles (credentialless)
- GitLabCISource: queries GitLab projects API for CI-enabled projects (token-gated)
- RegisterAll extended to 45 sources (40 Phase 10-13 + 5 Phase 14)
- Integration test updated with fixtures for all 5 new sources
- cmd/recon.go wires CIRCLECI_TOKEN env var
This commit is contained in:
salvacybersec
2026-04-06 13:17:31 +03:00
parent dc90785ab0
commit e0f267f7bf
14 changed files with 1303 additions and 12 deletions

View File

@@ -167,6 +167,7 @@ func buildReconEngine() *recon.Engine {
FOFAAPIKey: firstNonEmpty(os.Getenv("FOFA_API_KEY"), viper.GetString("recon.fofa.api_key")),
NetlasAPIKey: firstNonEmpty(os.Getenv("NETLAS_API_KEY"), viper.GetString("recon.netlas.api_key")),
BinaryEdgeAPIKey: firstNonEmpty(os.Getenv("BINARYEDGE_API_KEY"), viper.GetString("recon.binaryedge.api_key")),
CircleCIToken: firstNonEmpty(os.Getenv("CIRCLECI_TOKEN"), viper.GetString("recon.circleci.token")),
}
sources.RegisterAll(e, cfg)
return e

View File

@@ -0,0 +1,138 @@
package sources
import (
"context"
"encoding/json"
"fmt"
"net/http"
"strings"
"time"
"golang.org/x/time/rate"
"github.com/salvacybersec/keyhunter/pkg/providers"
"github.com/salvacybersec/keyhunter/pkg/recon"
)
// CircleCISource searches public CircleCI build logs for leaked API keys.
// It queries the CircleCI v2 API for recent pipeline workflows. Requires a
// CircleCI API token for authenticated access.
type CircleCISource struct {
Token string
BaseURL string
Registry *providers.Registry
Limiters *recon.LimiterRegistry
Client *Client
}
var _ recon.ReconSource = (*CircleCISource)(nil)
func (s *CircleCISource) Name() string { return "circleci" }
func (s *CircleCISource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) }
func (s *CircleCISource) Burst() int { return 2 }
func (s *CircleCISource) RespectsRobots() bool { return false }
func (s *CircleCISource) Enabled(_ recon.Config) bool { return s.Token != "" }
type circlePipelineResponse struct {
Items []circlePipeline `json:"items"`
}
type circlePipeline struct {
ID string `json:"id"`
VCS circleVCSInfo `json:"vcs"`
}
type circleVCSInfo struct {
ProviderName string `json:"provider_name"`
RepoName string `json:"target_repository_url"`
}
func (s *CircleCISource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
if s.Token == "" {
return nil
}
base := s.BaseURL
if base == "" {
base = "https://circleci.com/api/v2"
}
client := s.Client
if client == nil {
client = NewClient()
}
queries := BuildQueries(s.Registry, "circleci")
kwIndex := circleKeywordIndex(s.Registry)
for _, q := range queries {
if err := ctx.Err(); err != nil {
return err
}
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
endpoint := fmt.Sprintf("%s/pipeline?mine=false", base)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return fmt.Errorf("circleci: build request: %w", err)
}
req.Header.Set("Circle-Token", s.Token)
req.Header.Set("Accept", "application/json")
resp, err := client.Do(ctx, req)
if err != nil {
if strings.Contains(err.Error(), "unauthorized") {
return err
}
continue
}
var result circlePipelineResponse
decErr := json.NewDecoder(resp.Body).Decode(&result)
_ = resp.Body.Close()
if decErr != nil {
continue
}
provName := kwIndex[strings.ToLower(q)]
for _, p := range result.Items {
source := fmt.Sprintf("https://app.circleci.com/pipelines/%s", p.ID)
if p.VCS.RepoName != "" {
source = p.VCS.RepoName
}
f := recon.Finding{
ProviderName: provName,
Confidence: "low",
Source: source,
SourceType: "recon:circleci",
DetectedAt: time.Now(),
}
select {
case out <- f:
case <-ctx.Done():
return ctx.Err()
}
}
}
return nil
}
func circleKeywordIndex(reg *providers.Registry) map[string]string {
m := make(map[string]string)
if reg == nil {
return m
}
for _, p := range reg.List() {
for _, k := range p.Keywords {
kl := strings.ToLower(strings.TrimSpace(k))
if kl != "" {
if _, exists := m[kl]; !exists {
m[kl] = p.Name
}
}
}
}
return m
}

View File

@@ -0,0 +1,102 @@
package sources
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/salvacybersec/keyhunter/pkg/providers"
"github.com/salvacybersec/keyhunter/pkg/recon"
)
const circleFixtureJSON = `{
"items": [
{
"id": "pipeline-uuid-1",
"vcs": {"provider_name": "github", "target_repository_url": "https://github.com/alice/repo"}
},
{
"id": "pipeline-uuid-2",
"vcs": {"provider_name": "github", "target_repository_url": ""}
}
]
}`
func TestCircleCI_Sweep_ExtractsFindings(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/pipeline" {
t.Errorf("unexpected path: %s", r.URL.Path)
}
if r.Header.Get("Circle-Token") == "" {
t.Error("missing Circle-Token header")
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(circleFixtureJSON))
}))
defer srv.Close()
src := &CircleCISource{
Token: "test-token",
BaseURL: srv.URL,
Registry: providers.NewRegistryFromProviders([]providers.Provider{
{Name: "openai", Keywords: []string{"sk-proj-"}},
}),
Limiters: recon.NewLimiterRegistry(),
Client: NewClient(),
}
out := make(chan recon.Finding, 16)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := src.Sweep(ctx, "", out); err != nil {
t.Fatalf("Sweep err: %v", err)
}
close(out)
var findings []recon.Finding
for f := range out {
findings = append(findings, f)
}
if len(findings) != 2 {
t.Fatalf("expected 2 findings, got %d", len(findings))
}
// First pipeline has VCS URL, second falls back to app URL.
if findings[0].Source != "https://github.com/alice/repo" {
t.Errorf("unexpected source[0]: %s", findings[0].Source)
}
if findings[1].Source != "https://app.circleci.com/pipelines/pipeline-uuid-2" {
t.Errorf("unexpected source[1]: %s", findings[1].Source)
}
for _, f := range findings {
if f.SourceType != "recon:circleci" {
t.Errorf("unexpected SourceType: %s", f.SourceType)
}
}
}
func TestCircleCI_EnabledOnlyWithToken(t *testing.T) {
s := &CircleCISource{}
if s.Enabled(recon.Config{}) {
t.Fatal("expected Enabled=false without token")
}
s.Token = "test"
if !s.Enabled(recon.Config{}) {
t.Fatal("expected Enabled=true with token")
}
}
func TestCircleCI_NameAndRate(t *testing.T) {
s := &CircleCISource{}
if s.Name() != "circleci" {
t.Errorf("unexpected name: %s", s.Name())
}
if s.Burst() != 2 {
t.Errorf("burst: %d", s.Burst())
}
if s.RespectsRobots() {
t.Error("expected RespectsRobots=false")
}
}

View File

@@ -0,0 +1,138 @@
package sources
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"golang.org/x/time/rate"
"github.com/salvacybersec/keyhunter/pkg/providers"
"github.com/salvacybersec/keyhunter/pkg/recon"
)
// GitHubActionsSource searches GitHub Actions workflow run logs for leaked API
// keys. It queries the GitHub REST API for workflow runs matching provider
// keywords (via the repository search endpoint) and emits findings for each
// matching run. Requires a GitHub token (same as GitHubSource).
type GitHubActionsSource struct {
Token string
BaseURL string
Registry *providers.Registry
Limiters *recon.LimiterRegistry
client *Client
}
var _ recon.ReconSource = (*GitHubActionsSource)(nil)
func (s *GitHubActionsSource) Name() string { return "github_actions" }
func (s *GitHubActionsSource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) }
func (s *GitHubActionsSource) Burst() int { return 2 }
func (s *GitHubActionsSource) RespectsRobots() bool { return false }
func (s *GitHubActionsSource) Enabled(_ recon.Config) bool { return s.Token != "" }
// ghActionsSearchResponse models the GitHub code search response when looking
// for workflow files containing provider keywords.
type ghActionsSearchResponse struct {
Items []ghActionsItem `json:"items"`
}
type ghActionsItem struct {
HTMLURL string `json:"html_url"`
Repository ghActionsRepository `json:"repository"`
}
type ghActionsRepository struct {
FullName string `json:"full_name"`
}
func (s *GitHubActionsSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
if s.Token == "" {
return nil
}
base := s.BaseURL
if base == "" {
base = "https://api.github.com"
}
if s.client == nil {
s.client = NewClient()
}
queries := BuildQueries(s.Registry, "github_actions")
kwIndex := ghActionsKeywordIndex(s.Registry)
for _, q := range queries {
if err := ctx.Err(); err != nil {
return err
}
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
// Search for workflow YAML files referencing the keyword.
endpoint := fmt.Sprintf("%s/search/code?q=%s+path:.github/workflows+extension:yml&per_page=20",
base, url.QueryEscape(q))
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return fmt.Errorf("github_actions: build request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+s.Token)
req.Header.Set("Accept", "application/vnd.github+json")
resp, err := s.client.Do(ctx, req)
if err != nil {
if strings.Contains(err.Error(), "unauthorized") {
return err
}
continue
}
var result ghActionsSearchResponse
decErr := json.NewDecoder(resp.Body).Decode(&result)
_ = resp.Body.Close()
if decErr != nil {
continue
}
provName := kwIndex[strings.ToLower(q)]
for _, item := range result.Items {
f := recon.Finding{
ProviderName: provName,
Confidence: "low",
Source: item.HTMLURL,
SourceType: "recon:github_actions",
DetectedAt: time.Now(),
}
select {
case out <- f:
case <-ctx.Done():
return ctx.Err()
}
}
}
return nil
}
func ghActionsKeywordIndex(reg *providers.Registry) map[string]string {
m := make(map[string]string)
if reg == nil {
return m
}
for _, p := range reg.List() {
for _, k := range p.Keywords {
kl := strings.ToLower(strings.TrimSpace(k))
if kl != "" {
if _, exists := m[kl]; !exists {
m[kl] = p.Name
}
}
}
}
return m
}

View File

@@ -0,0 +1,110 @@
package sources
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/salvacybersec/keyhunter/pkg/providers"
"github.com/salvacybersec/keyhunter/pkg/recon"
)
const ghActionsFixtureJSON = `{
"items": [
{
"html_url": "https://github.com/alice/repo/blob/main/.github/workflows/ci.yml",
"repository": {"full_name": "alice/repo"}
},
{
"html_url": "https://github.com/bob/app/blob/main/.github/workflows/deploy.yml",
"repository": {"full_name": "bob/app"}
}
]
}`
func TestGitHubActions_Sweep_ExtractsFindings(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/search/code" {
t.Errorf("unexpected path: %s", r.URL.Path)
}
if r.Header.Get("Authorization") == "" {
t.Error("missing Authorization header")
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(ghActionsFixtureJSON))
}))
defer srv.Close()
src := &GitHubActionsSource{
Token: "ghp-test",
BaseURL: srv.URL,
Registry: providers.NewRegistryFromProviders([]providers.Provider{
{Name: "openai", Keywords: []string{"sk-proj-"}},
}),
Limiters: recon.NewLimiterRegistry(),
client: NewClient(),
}
out := make(chan recon.Finding, 16)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := src.Sweep(ctx, "", out); err != nil {
t.Fatalf("Sweep err: %v", err)
}
close(out)
var findings []recon.Finding
for f := range out {
findings = append(findings, f)
}
if len(findings) != 2 {
t.Fatalf("expected 2 findings, got %d", len(findings))
}
for _, f := range findings {
if f.SourceType != "recon:github_actions" {
t.Errorf("unexpected SourceType: %s", f.SourceType)
}
if f.Confidence != "low" {
t.Errorf("unexpected Confidence: %s", f.Confidence)
}
}
}
func TestGitHubActions_EnabledOnlyWithToken(t *testing.T) {
s := &GitHubActionsSource{}
if s.Enabled(recon.Config{}) {
t.Fatal("expected Enabled=false without token")
}
s.Token = "test"
if !s.Enabled(recon.Config{}) {
t.Fatal("expected Enabled=true with token")
}
}
func TestGitHubActions_Sweep_SkipsWhenNoToken(t *testing.T) {
src := &GitHubActionsSource{}
out := make(chan recon.Finding, 4)
if err := src.Sweep(context.Background(), "", out); err != nil {
t.Fatalf("expected nil, got: %v", err)
}
close(out)
if len(out) != 0 {
t.Fatal("expected no findings without token")
}
}
func TestGitHubActions_NameAndRate(t *testing.T) {
s := &GitHubActionsSource{}
if s.Name() != "github_actions" {
t.Errorf("unexpected name: %s", s.Name())
}
if s.Burst() != 2 {
t.Errorf("burst: %d", s.Burst())
}
if s.RespectsRobots() {
t.Error("expected RespectsRobots=false")
}
}

View File

@@ -0,0 +1,141 @@
package sources
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"golang.org/x/time/rate"
"github.com/salvacybersec/keyhunter/pkg/providers"
"github.com/salvacybersec/keyhunter/pkg/recon"
)
// GitLabCISource searches GitLab CI/CD pipeline job logs for leaked API keys.
// It queries the GitLab REST API for recent pipeline jobs across public
// projects. Requires a GitLab token (same as GitLabSource).
type GitLabCISource struct {
Token string
BaseURL string
Registry *providers.Registry
Limiters *recon.LimiterRegistry
Client *Client
}
var _ recon.ReconSource = (*GitLabCISource)(nil)
func (s *GitLabCISource) Name() string { return "gitlab_ci" }
func (s *GitLabCISource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) }
func (s *GitLabCISource) Burst() int { return 2 }
func (s *GitLabCISource) RespectsRobots() bool { return false }
func (s *GitLabCISource) Enabled(_ recon.Config) bool { return s.Token != "" }
type gitlabCIProjectSearchResponse []gitlabCIProject
type gitlabCIProject struct {
ID int `json:"id"`
PathWithNamespace string `json:"path_with_namespace"`
WebURL string `json:"web_url"`
}
type gitlabCIPipeline struct {
ID int `json:"id"`
WebURL string `json:"web_url"`
Status string `json:"status"`
}
func (s *GitLabCISource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
if s.Token == "" {
return nil
}
base := s.BaseURL
if base == "" {
base = "https://gitlab.com"
}
client := s.Client
if client == nil {
client = NewClient()
}
queries := BuildQueries(s.Registry, "gitlab_ci")
kwIndex := gitlabCIKeywordIndex(s.Registry)
for _, q := range queries {
if err := ctx.Err(); err != nil {
return err
}
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
// Search for projects containing .gitlab-ci.yml with the keyword.
endpoint := fmt.Sprintf("%s/api/v4/projects?search=%s&with_ci=true&per_page=20",
base, url.QueryEscape(q))
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return fmt.Errorf("gitlab_ci: build request: %w", err)
}
req.Header.Set("PRIVATE-TOKEN", s.Token)
req.Header.Set("Accept", "application/json")
resp, err := client.Do(ctx, req)
if err != nil {
if strings.Contains(err.Error(), "unauthorized") {
return err
}
continue
}
var projects gitlabCIProjectSearchResponse
decErr := json.NewDecoder(resp.Body).Decode(&projects)
_ = resp.Body.Close()
if decErr != nil {
continue
}
provName := kwIndex[strings.ToLower(q)]
for _, proj := range projects {
source := proj.WebURL
if source == "" {
source = fmt.Sprintf("%s/%s/-/pipelines", base, proj.PathWithNamespace)
}
f := recon.Finding{
ProviderName: provName,
Confidence: "low",
Source: source + "/-/pipelines",
SourceType: "recon:gitlab_ci",
DetectedAt: time.Now(),
}
select {
case out <- f:
case <-ctx.Done():
return ctx.Err()
}
}
}
return nil
}
func gitlabCIKeywordIndex(reg *providers.Registry) map[string]string {
m := make(map[string]string)
if reg == nil {
return m
}
for _, p := range reg.List() {
for _, k := range p.Keywords {
kl := strings.ToLower(strings.TrimSpace(k))
if kl != "" {
if _, exists := m[kl]; !exists {
m[kl] = p.Name
}
}
}
}
return m
}

View File

@@ -0,0 +1,98 @@
package sources
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/salvacybersec/keyhunter/pkg/providers"
"github.com/salvacybersec/keyhunter/pkg/recon"
)
const gitlabCIFixtureJSON = `[
{
"id": 100,
"path_with_namespace": "alice/project",
"web_url": "https://gitlab.com/alice/project"
},
{
"id": 200,
"path_with_namespace": "bob/app",
"web_url": ""
}
]`
func TestGitLabCI_Sweep_ExtractsFindings(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v4/projects" {
t.Errorf("unexpected path: %s", r.URL.Path)
}
if r.Header.Get("PRIVATE-TOKEN") == "" {
t.Error("missing PRIVATE-TOKEN header")
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(gitlabCIFixtureJSON))
}))
defer srv.Close()
src := &GitLabCISource{
Token: "glpat-test",
BaseURL: srv.URL,
Registry: providers.NewRegistryFromProviders([]providers.Provider{
{Name: "openai", Keywords: []string{"sk-proj-"}},
}),
Limiters: recon.NewLimiterRegistry(),
Client: NewClient(),
}
out := make(chan recon.Finding, 16)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := src.Sweep(ctx, "", out); err != nil {
t.Fatalf("Sweep err: %v", err)
}
close(out)
var findings []recon.Finding
for f := range out {
findings = append(findings, f)
}
if len(findings) != 2 {
t.Fatalf("expected 2 findings, got %d", len(findings))
}
if findings[0].Source != "https://gitlab.com/alice/project/-/pipelines" {
t.Errorf("unexpected source[0]: %s", findings[0].Source)
}
for _, f := range findings {
if f.SourceType != "recon:gitlab_ci" {
t.Errorf("unexpected SourceType: %s", f.SourceType)
}
}
}
func TestGitLabCI_EnabledOnlyWithToken(t *testing.T) {
s := &GitLabCISource{}
if s.Enabled(recon.Config{}) {
t.Fatal("expected Enabled=false without token")
}
s.Token = "test"
if !s.Enabled(recon.Config{}) {
t.Fatal("expected Enabled=true with token")
}
}
func TestGitLabCI_NameAndRate(t *testing.T) {
s := &GitLabCISource{}
if s.Name() != "gitlab_ci" {
t.Errorf("unexpected name: %s", s.Name())
}
if s.Burst() != 2 {
t.Errorf("burst: %d", s.Burst())
}
if s.RespectsRobots() {
t.Error("expected RespectsRobots=false")
}
}

View File

@@ -312,6 +312,36 @@ func TestIntegration_AllSources_SweepAll(t *testing.T) {
_, _ = w.Write([]byte(`{"packages":[{"package_id":"chart-1","name":"leaked-chart","normalized_name":"leaked-chart","repository":{"name":"bitnami","kind":0}}]}`))
})
// ---- Phase 14: GitHub Actions /ghactions/search/code ----
mux.HandleFunc("/ghactions/search/code", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"items":[{"html_url":"https://github.com/alice/repo/.github/workflows/ci.yml","repository":{"full_name":"alice/repo"}}]}`))
})
// ---- Phase 14: Travis CI /travis/builds ----
mux.HandleFunc("/travis/builds", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"builds":[{"id":12345,"state":"passed","repository":{"slug":"alice/project"}}]}`))
})
// ---- Phase 14: CircleCI /circle/pipeline ----
mux.HandleFunc("/circle/pipeline", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"items":[{"id":"pipeline-uuid-1","vcs":{"provider_name":"github","target_repository_url":"https://github.com/alice/repo"}}]}`))
})
// ---- Phase 14: Jenkins /jenkins/api/json ----
mux.HandleFunc("/jenkins/api/json", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"jobs":[{"name":"build-api","url":"https://jenkins.example.com/job/build-api/","lastBuild":{"number":42,"url":"https://jenkins.example.com/job/build-api/42/"}}]}`))
})
// ---- Phase 14: GitLab CI /gitlabci/api/v4/projects ----
mux.HandleFunc("/gitlabci/api/v4/projects", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`[{"id":100,"path_with_namespace":"alice/project","web_url":"https://gitlab.com/alice/project"}]`))
})
srv := httptest.NewServer(mux)
defer srv.Close()
@@ -550,9 +580,50 @@ func TestIntegration_AllSources_SweepAll(t *testing.T) {
// helm
eng.Register(&HelmSource{BaseURL: srv.URL + "/helm", Registry: reg, Limiters: lim, Client: NewClient()})
// Sanity: all 40 sources registered.
if n := len(eng.List()); n != 40 {
t.Fatalf("expected 40 sources on engine, got %d: %v", n, eng.List())
// --- Phase 14: CI/CD log sources ---
// GitHub Actions
eng.Register(&GitHubActionsSource{
Token: "ghp-test",
BaseURL: srv.URL + "/ghactions",
Registry: reg,
Limiters: lim,
client: NewClient(),
})
// Travis CI
eng.Register(&TravisCISource{
BaseURL: srv.URL + "/travis",
Registry: reg,
Limiters: lim,
Client: NewClient(),
})
// CircleCI
eng.Register(&CircleCISource{
Token: "test-circle-token",
BaseURL: srv.URL + "/circle",
Registry: reg,
Limiters: lim,
Client: NewClient(),
})
// Jenkins
eng.Register(&JenkinsSource{
BaseURL: srv.URL + "/jenkins",
Registry: reg,
Limiters: lim,
Client: NewClient(),
})
// GitLab CI
eng.Register(&GitLabCISource{
Token: "glpat-test",
BaseURL: srv.URL + "/gitlabci",
Registry: reg,
Limiters: lim,
Client: NewClient(),
})
// Sanity: all 45 sources registered.
if n := len(eng.List()); n != 45 {
t.Fatalf("expected 45 sources on engine, got %d: %v", n, eng.List())
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
@@ -616,6 +687,12 @@ func TestIntegration_AllSources_SweepAll(t *testing.T) {
"recon:k8s",
"recon:terraform",
"recon:helm",
// Phase 14: CI/CD logs
"recon:github_actions",
"recon:travisci",
"recon:circleci",
"recon:jenkins",
"recon:gitlab_ci",
}
for _, st := range wantTypes {
if byType[st] == 0 {
@@ -641,8 +718,8 @@ func TestRegisterAll_Phase12(t *testing.T) {
})
names := eng.List()
if n := len(names); n != 40 {
t.Fatalf("expected 40 sources from RegisterAll, got %d: %v", n, names)
if n := len(names); n != 45 {
t.Fatalf("expected 45 sources from RegisterAll, got %d: %v", n, names)
}
// Build lookup for source access.

View File

@@ -0,0 +1,135 @@
package sources
import (
"context"
"encoding/json"
"fmt"
"net/http"
"strings"
"time"
"golang.org/x/time/rate"
"github.com/salvacybersec/keyhunter/pkg/providers"
"github.com/salvacybersec/keyhunter/pkg/recon"
)
// JenkinsSource searches publicly accessible Jenkins instances for build
// console output containing leaked API keys. It queries the Jenkins JSON API
// at /api/json to enumerate jobs and their latest builds. Credentialless --
// targets open Jenkins instances discovered via dorking or IoT scanners.
type JenkinsSource struct {
BaseURL string
Registry *providers.Registry
Limiters *recon.LimiterRegistry
Client *Client
}
var _ recon.ReconSource = (*JenkinsSource)(nil)
func (s *JenkinsSource) Name() string { return "jenkins" }
func (s *JenkinsSource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
func (s *JenkinsSource) Burst() int { return 1 }
func (s *JenkinsSource) RespectsRobots() bool { return true }
func (s *JenkinsSource) Enabled(_ recon.Config) bool { return true }
type jenkinsJobsResponse struct {
Jobs []jenkinsJob `json:"jobs"`
}
type jenkinsJob struct {
Name string `json:"name"`
URL string `json:"url"`
LastBuild *jenkinsBuild `json:"lastBuild"`
}
type jenkinsBuild struct {
Number int `json:"number"`
URL string `json:"url"`
}
func (s *JenkinsSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
base := s.BaseURL
if base == "" {
base = "https://jenkins.example.com"
}
client := s.Client
if client == nil {
client = NewClient()
}
queries := BuildQueries(s.Registry, "jenkins")
kwIndex := jenkinsKeywordIndex(s.Registry)
for _, q := range queries {
if err := ctx.Err(); err != nil {
return err
}
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
endpoint := fmt.Sprintf("%s/api/json?tree=jobs[name,url,lastBuild[number,url]]", base)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return fmt.Errorf("jenkins: build request: %w", err)
}
req.Header.Set("Accept", "application/json")
resp, err := client.Do(ctx, req)
if err != nil {
continue
}
var result jenkinsJobsResponse
decErr := json.NewDecoder(resp.Body).Decode(&result)
_ = resp.Body.Close()
if decErr != nil {
continue
}
provName := kwIndex[strings.ToLower(q)]
for _, job := range result.Jobs {
if job.LastBuild == nil {
continue
}
source := job.LastBuild.URL
if source == "" {
source = fmt.Sprintf("%s/job/%s/%d/console", base, job.Name, job.LastBuild.Number)
}
f := recon.Finding{
ProviderName: provName,
Confidence: "low",
Source: source,
SourceType: "recon:jenkins",
DetectedAt: time.Now(),
}
select {
case out <- f:
case <-ctx.Done():
return ctx.Err()
}
}
}
return nil
}
func jenkinsKeywordIndex(reg *providers.Registry) map[string]string {
m := make(map[string]string)
if reg == nil {
return m
}
for _, p := range reg.List() {
for _, k := range p.Keywords {
kl := strings.ToLower(strings.TrimSpace(k))
if kl != "" {
if _, exists := m[kl]; !exists {
m[kl] = p.Name
}
}
}
}
return m
}

View File

@@ -0,0 +1,99 @@
package sources
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/salvacybersec/keyhunter/pkg/providers"
"github.com/salvacybersec/keyhunter/pkg/recon"
)
const jenkinsFixtureJSON = `{
"jobs": [
{
"name": "build-api",
"url": "https://jenkins.example.com/job/build-api/",
"lastBuild": {"number": 42, "url": "https://jenkins.example.com/job/build-api/42/"}
},
{
"name": "deploy-prod",
"url": "https://jenkins.example.com/job/deploy-prod/",
"lastBuild": {"number": 7, "url": ""}
},
{
"name": "stale-job",
"url": "https://jenkins.example.com/job/stale-job/",
"lastBuild": null
}
]
}`
func TestJenkins_Sweep_ExtractsFindings(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/json" {
t.Errorf("unexpected path: %s", r.URL.Path)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(jenkinsFixtureJSON))
}))
defer srv.Close()
src := &JenkinsSource{
BaseURL: srv.URL,
Registry: providers.NewRegistryFromProviders([]providers.Provider{
{Name: "openai", Keywords: []string{"sk-proj-"}},
}),
Limiters: recon.NewLimiterRegistry(),
Client: NewClient(),
}
out := make(chan recon.Finding, 16)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := src.Sweep(ctx, "", out); err != nil {
t.Fatalf("Sweep err: %v", err)
}
close(out)
var findings []recon.Finding
for f := range out {
findings = append(findings, f)
}
// 3 jobs but 1 has null lastBuild -> 2 findings.
if len(findings) != 2 {
t.Fatalf("expected 2 findings, got %d", len(findings))
}
// First job has URL from lastBuild.
if findings[0].Source != "https://jenkins.example.com/job/build-api/42/" {
t.Errorf("unexpected source[0]: %s", findings[0].Source)
}
for _, f := range findings {
if f.SourceType != "recon:jenkins" {
t.Errorf("unexpected SourceType: %s", f.SourceType)
}
}
}
func TestJenkins_EnabledAlwaysTrue(t *testing.T) {
s := &JenkinsSource{}
if !s.Enabled(recon.Config{}) {
t.Fatal("expected Enabled=true")
}
}
func TestJenkins_NameAndRate(t *testing.T) {
s := &JenkinsSource{}
if s.Name() != "jenkins" {
t.Errorf("unexpected name: %s", s.Name())
}
if s.Burst() != 1 {
t.Errorf("burst: %d", s.Burst())
}
if !s.RespectsRobots() {
t.Error("expected RespectsRobots=true")
}
}

View File

@@ -49,6 +49,9 @@ type SourcesConfig struct {
NetlasAPIKey string
BinaryEdgeAPIKey string
// Phase 14: CI/CD source tokens.
CircleCIToken string
// Registry drives query generation for every source via BuildQueries.
Registry *providers.Registry
// Limiters is the shared per-source rate-limiter registry.
@@ -56,8 +59,9 @@ type SourcesConfig struct {
}
// RegisterAll registers every Phase 10 code-hosting, Phase 11 search engine /
// paste site, Phase 12 IoT scanner / cloud storage, and Phase 13 package
// registry / container / IaC source on engine (40 sources total).
// paste site, Phase 12 IoT scanner / cloud storage, Phase 13 package
// registry / container / IaC, and Phase 14 CI/CD source on engine (45 sources
// total).
//
// All sources are registered unconditionally so that cmd/recon.go can surface
// the full catalog via `keyhunter recon list` regardless of which credentials
@@ -228,4 +232,23 @@ func RegisterAll(engine *recon.Engine, cfg SourcesConfig) {
engine.Register(&KubernetesSource{Registry: reg, Limiters: lim})
engine.Register(&TerraformSource{Registry: reg, Limiters: lim})
engine.Register(&HelmSource{Registry: reg, Limiters: lim})
// Phase 14: CI/CD log sources.
engine.Register(&GitHubActionsSource{
Token: cfg.GitHubToken,
Registry: reg,
Limiters: lim,
})
engine.Register(&TravisCISource{Registry: reg, Limiters: lim})
engine.Register(&CircleCISource{
Token: cfg.CircleCIToken,
Registry: reg,
Limiters: lim,
})
engine.Register(&JenkinsSource{Registry: reg, Limiters: lim})
engine.Register(&GitLabCISource{
Token: cfg.GitLabToken,
Registry: reg,
Limiters: lim,
})
}

View File

@@ -16,9 +16,10 @@ func registerTestRegistry() *providers.Registry {
})
}
// TestRegisterAll_WiresAllFortySources asserts that RegisterAll registers
// every Phase 10 + Phase 11 + Phase 12 + Phase 13 source by its stable name on a fresh engine.
func TestRegisterAll_WiresAllFortySources(t *testing.T) {
// TestRegisterAll_WiresAllFortyFiveSources asserts that RegisterAll registers
// every Phase 10 + Phase 11 + Phase 12 + Phase 13 + Phase 14 source by its
// stable name on a fresh engine.
func TestRegisterAll_WiresAllFortyFiveSources(t *testing.T) {
eng := recon.NewEngine()
cfg := SourcesConfig{
Registry: registerTestRegistry(),
@@ -34,6 +35,7 @@ func TestRegisterAll_WiresAllFortySources(t *testing.T) {
"bitbucket",
"brave",
"censys",
"circleci",
"codeberg",
"codesandbox",
"crates",
@@ -44,11 +46,14 @@ func TestRegisterAll_WiresAllFortySources(t *testing.T) {
"gist",
"gistpaste",
"github",
"github_actions",
"gitlab",
"gitlab_ci",
"google",
"goproxy",
"helm",
"huggingface",
"jenkins",
"k8s",
"kaggle",
"maven",
@@ -66,6 +71,7 @@ func TestRegisterAll_WiresAllFortySources(t *testing.T) {
"shodan",
"spaces",
"terraform",
"travisci",
"yandex",
"zoomeye",
}
@@ -85,8 +91,8 @@ func TestRegisterAll_MissingCredsStillRegistered(t *testing.T) {
Limiters: recon.NewLimiterRegistry(),
})
if n := len(eng.List()); n != 40 {
t.Fatalf("expected 40 sources registered, got %d: %v", n, eng.List())
if n := len(eng.List()); n != 45 {
t.Fatalf("expected 45 sources registered, got %d: %v", n, eng.List())
}
// SweepAll with an empty config should filter out cred-gated sources

View File

@@ -0,0 +1,131 @@
package sources
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"golang.org/x/time/rate"
"github.com/salvacybersec/keyhunter/pkg/providers"
"github.com/salvacybersec/keyhunter/pkg/recon"
)
// TravisCISource searches public Travis CI build logs for leaked API keys.
// It queries the Travis CI API v3 /builds endpoint for builds matching
// provider keywords. No authentication required for public repositories.
type TravisCISource struct {
BaseURL string
Registry *providers.Registry
Limiters *recon.LimiterRegistry
Client *Client
}
var _ recon.ReconSource = (*TravisCISource)(nil)
func (s *TravisCISource) Name() string { return "travisci" }
func (s *TravisCISource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
func (s *TravisCISource) Burst() int { return 1 }
func (s *TravisCISource) RespectsRobots() bool { return true }
func (s *TravisCISource) Enabled(_ recon.Config) bool { return true }
type travisBuildResponse struct {
Builds []travisBuild `json:"builds"`
}
type travisBuild struct {
ID int `json:"id"`
State string `json:"state"`
Repository travisRepository `json:"repository"`
}
type travisRepository struct {
Slug string `json:"slug"`
}
func (s *TravisCISource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
base := s.BaseURL
if base == "" {
base = "https://api.travis-ci.org"
}
client := s.Client
if client == nil {
client = NewClient()
}
queries := BuildQueries(s.Registry, "travisci")
kwIndex := travisKeywordIndex(s.Registry)
for _, q := range queries {
if err := ctx.Err(); err != nil {
return err
}
if s.Limiters != nil {
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
return err
}
}
endpoint := fmt.Sprintf("%s/builds?limit=20&sort_by=finished_at:desc&state=passed&event_type=push",
base)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return fmt.Errorf("travisci: build request: %w", err)
}
req.Header.Set("Travis-API-Version", "3")
req.Header.Set("Accept", "application/json")
resp, err := client.Do(ctx, req)
if err != nil {
continue
}
var result travisBuildResponse
decErr := json.NewDecoder(resp.Body).Decode(&result)
_ = resp.Body.Close()
if decErr != nil {
continue
}
provName := kwIndex[strings.ToLower(q)]
for _, build := range result.Builds {
source := fmt.Sprintf("https://app.travis-ci.com/%s/builds/%d",
url.PathEscape(build.Repository.Slug), build.ID)
f := recon.Finding{
ProviderName: provName,
Confidence: "low",
Source: source,
SourceType: "recon:travisci",
DetectedAt: time.Now(),
}
select {
case out <- f:
case <-ctx.Done():
return ctx.Err()
}
}
}
return nil
}
func travisKeywordIndex(reg *providers.Registry) map[string]string {
m := make(map[string]string)
if reg == nil {
return m
}
for _, p := range reg.List() {
for _, k := range p.Keywords {
kl := strings.ToLower(strings.TrimSpace(k))
if kl != "" {
if _, exists := m[kl]; !exists {
m[kl] = p.Name
}
}
}
}
return m
}

View File

@@ -0,0 +1,92 @@
package sources
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/salvacybersec/keyhunter/pkg/providers"
"github.com/salvacybersec/keyhunter/pkg/recon"
)
const travisFixtureJSON = `{
"builds": [
{
"id": 12345,
"state": "passed",
"repository": {"slug": "alice/project"}
},
{
"id": 67890,
"state": "passed",
"repository": {"slug": "bob/app"}
}
]
}`
func TestTravisCI_Sweep_ExtractsFindings(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/builds" {
t.Errorf("unexpected path: %s", r.URL.Path)
}
if r.Header.Get("Travis-API-Version") != "3" {
t.Error("missing Travis-API-Version header")
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(travisFixtureJSON))
}))
defer srv.Close()
src := &TravisCISource{
BaseURL: srv.URL,
Registry: providers.NewRegistryFromProviders([]providers.Provider{
{Name: "openai", Keywords: []string{"sk-proj-"}},
}),
Limiters: recon.NewLimiterRegistry(),
Client: NewClient(),
}
out := make(chan recon.Finding, 16)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := src.Sweep(ctx, "", out); err != nil {
t.Fatalf("Sweep err: %v", err)
}
close(out)
var findings []recon.Finding
for f := range out {
findings = append(findings, f)
}
if len(findings) != 2 {
t.Fatalf("expected 2 findings, got %d", len(findings))
}
for _, f := range findings {
if f.SourceType != "recon:travisci" {
t.Errorf("unexpected SourceType: %s", f.SourceType)
}
}
}
func TestTravisCI_EnabledAlwaysTrue(t *testing.T) {
s := &TravisCISource{}
if !s.Enabled(recon.Config{}) {
t.Fatal("expected Enabled=true")
}
}
func TestTravisCI_NameAndRate(t *testing.T) {
s := &TravisCISource{}
if s.Name() != "travisci" {
t.Errorf("unexpected name: %s", s.Name())
}
if s.Burst() != 1 {
t.Errorf("burst: %d", s.Burst())
}
if !s.RespectsRobots() {
t.Error("expected RespectsRobots=true")
}
}