feat(14-01): add 5 CI/CD log sources (GitHubActions, TravisCI, CircleCI, Jenkins, GitLabCI)
- GitHubActionsSource: searches GitHub code search for workflow files with provider keywords (token-gated) - TravisCISource: queries Travis CI v3 API for public build logs (credentialless) - CircleCISource: queries CircleCI v2 pipeline API for build pipelines (token-gated) - JenkinsSource: queries open Jenkins /api/json for job build consoles (credentialless) - GitLabCISource: queries GitLab projects API for CI-enabled projects (token-gated) - RegisterAll extended to 45 sources (40 Phase 10-13 + 5 Phase 14) - Integration test updated with fixtures for all 5 new sources - cmd/recon.go wires CIRCLECI_TOKEN env var
This commit is contained in:
@@ -167,6 +167,7 @@ func buildReconEngine() *recon.Engine {
|
|||||||
FOFAAPIKey: firstNonEmpty(os.Getenv("FOFA_API_KEY"), viper.GetString("recon.fofa.api_key")),
|
FOFAAPIKey: firstNonEmpty(os.Getenv("FOFA_API_KEY"), viper.GetString("recon.fofa.api_key")),
|
||||||
NetlasAPIKey: firstNonEmpty(os.Getenv("NETLAS_API_KEY"), viper.GetString("recon.netlas.api_key")),
|
NetlasAPIKey: firstNonEmpty(os.Getenv("NETLAS_API_KEY"), viper.GetString("recon.netlas.api_key")),
|
||||||
BinaryEdgeAPIKey: firstNonEmpty(os.Getenv("BINARYEDGE_API_KEY"), viper.GetString("recon.binaryedge.api_key")),
|
BinaryEdgeAPIKey: firstNonEmpty(os.Getenv("BINARYEDGE_API_KEY"), viper.GetString("recon.binaryedge.api_key")),
|
||||||
|
CircleCIToken: firstNonEmpty(os.Getenv("CIRCLECI_TOKEN"), viper.GetString("recon.circleci.token")),
|
||||||
}
|
}
|
||||||
sources.RegisterAll(e, cfg)
|
sources.RegisterAll(e, cfg)
|
||||||
return e
|
return e
|
||||||
|
|||||||
138
pkg/recon/sources/circleci.go
Normal file
138
pkg/recon/sources/circleci.go
Normal file
@@ -0,0 +1,138 @@
|
|||||||
|
package sources
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/time/rate"
|
||||||
|
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CircleCISource searches public CircleCI build logs for leaked API keys.
|
||||||
|
// It queries the CircleCI v2 API for recent pipeline workflows. Requires a
|
||||||
|
// CircleCI API token for authenticated access.
|
||||||
|
type CircleCISource struct {
|
||||||
|
Token string
|
||||||
|
BaseURL string
|
||||||
|
Registry *providers.Registry
|
||||||
|
Limiters *recon.LimiterRegistry
|
||||||
|
Client *Client
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ recon.ReconSource = (*CircleCISource)(nil)
|
||||||
|
|
||||||
|
func (s *CircleCISource) Name() string { return "circleci" }
|
||||||
|
func (s *CircleCISource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) }
|
||||||
|
func (s *CircleCISource) Burst() int { return 2 }
|
||||||
|
func (s *CircleCISource) RespectsRobots() bool { return false }
|
||||||
|
func (s *CircleCISource) Enabled(_ recon.Config) bool { return s.Token != "" }
|
||||||
|
|
||||||
|
type circlePipelineResponse struct {
|
||||||
|
Items []circlePipeline `json:"items"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type circlePipeline struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
VCS circleVCSInfo `json:"vcs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type circleVCSInfo struct {
|
||||||
|
ProviderName string `json:"provider_name"`
|
||||||
|
RepoName string `json:"target_repository_url"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CircleCISource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
|
||||||
|
if s.Token == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
base := s.BaseURL
|
||||||
|
if base == "" {
|
||||||
|
base = "https://circleci.com/api/v2"
|
||||||
|
}
|
||||||
|
client := s.Client
|
||||||
|
if client == nil {
|
||||||
|
client = NewClient()
|
||||||
|
}
|
||||||
|
|
||||||
|
queries := BuildQueries(s.Registry, "circleci")
|
||||||
|
kwIndex := circleKeywordIndex(s.Registry)
|
||||||
|
|
||||||
|
for _, q := range queries {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if s.Limiters != nil {
|
||||||
|
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
endpoint := fmt.Sprintf("%s/pipeline?mine=false", base)
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("circleci: build request: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Circle-Token", s.Token)
|
||||||
|
req.Header.Set("Accept", "application/json")
|
||||||
|
|
||||||
|
resp, err := client.Do(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
if strings.Contains(err.Error(), "unauthorized") {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
var result circlePipelineResponse
|
||||||
|
decErr := json.NewDecoder(resp.Body).Decode(&result)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
if decErr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
provName := kwIndex[strings.ToLower(q)]
|
||||||
|
for _, p := range result.Items {
|
||||||
|
source := fmt.Sprintf("https://app.circleci.com/pipelines/%s", p.ID)
|
||||||
|
if p.VCS.RepoName != "" {
|
||||||
|
source = p.VCS.RepoName
|
||||||
|
}
|
||||||
|
f := recon.Finding{
|
||||||
|
ProviderName: provName,
|
||||||
|
Confidence: "low",
|
||||||
|
Source: source,
|
||||||
|
SourceType: "recon:circleci",
|
||||||
|
DetectedAt: time.Now(),
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case out <- f:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func circleKeywordIndex(reg *providers.Registry) map[string]string {
|
||||||
|
m := make(map[string]string)
|
||||||
|
if reg == nil {
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
for _, p := range reg.List() {
|
||||||
|
for _, k := range p.Keywords {
|
||||||
|
kl := strings.ToLower(strings.TrimSpace(k))
|
||||||
|
if kl != "" {
|
||||||
|
if _, exists := m[kl]; !exists {
|
||||||
|
m[kl] = p.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
102
pkg/recon/sources/circleci_test.go
Normal file
102
pkg/recon/sources/circleci_test.go
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
package sources
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||||
|
)
|
||||||
|
|
||||||
|
const circleFixtureJSON = `{
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"id": "pipeline-uuid-1",
|
||||||
|
"vcs": {"provider_name": "github", "target_repository_url": "https://github.com/alice/repo"}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "pipeline-uuid-2",
|
||||||
|
"vcs": {"provider_name": "github", "target_repository_url": ""}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
func TestCircleCI_Sweep_ExtractsFindings(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/pipeline" {
|
||||||
|
t.Errorf("unexpected path: %s", r.URL.Path)
|
||||||
|
}
|
||||||
|
if r.Header.Get("Circle-Token") == "" {
|
||||||
|
t.Error("missing Circle-Token header")
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(circleFixtureJSON))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
src := &CircleCISource{
|
||||||
|
Token: "test-token",
|
||||||
|
BaseURL: srv.URL,
|
||||||
|
Registry: providers.NewRegistryFromProviders([]providers.Provider{
|
||||||
|
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||||
|
}),
|
||||||
|
Limiters: recon.NewLimiterRegistry(),
|
||||||
|
Client: NewClient(),
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make(chan recon.Finding, 16)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
if err := src.Sweep(ctx, "", out); err != nil {
|
||||||
|
t.Fatalf("Sweep err: %v", err)
|
||||||
|
}
|
||||||
|
close(out)
|
||||||
|
|
||||||
|
var findings []recon.Finding
|
||||||
|
for f := range out {
|
||||||
|
findings = append(findings, f)
|
||||||
|
}
|
||||||
|
if len(findings) != 2 {
|
||||||
|
t.Fatalf("expected 2 findings, got %d", len(findings))
|
||||||
|
}
|
||||||
|
// First pipeline has VCS URL, second falls back to app URL.
|
||||||
|
if findings[0].Source != "https://github.com/alice/repo" {
|
||||||
|
t.Errorf("unexpected source[0]: %s", findings[0].Source)
|
||||||
|
}
|
||||||
|
if findings[1].Source != "https://app.circleci.com/pipelines/pipeline-uuid-2" {
|
||||||
|
t.Errorf("unexpected source[1]: %s", findings[1].Source)
|
||||||
|
}
|
||||||
|
for _, f := range findings {
|
||||||
|
if f.SourceType != "recon:circleci" {
|
||||||
|
t.Errorf("unexpected SourceType: %s", f.SourceType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCircleCI_EnabledOnlyWithToken(t *testing.T) {
|
||||||
|
s := &CircleCISource{}
|
||||||
|
if s.Enabled(recon.Config{}) {
|
||||||
|
t.Fatal("expected Enabled=false without token")
|
||||||
|
}
|
||||||
|
s.Token = "test"
|
||||||
|
if !s.Enabled(recon.Config{}) {
|
||||||
|
t.Fatal("expected Enabled=true with token")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCircleCI_NameAndRate(t *testing.T) {
|
||||||
|
s := &CircleCISource{}
|
||||||
|
if s.Name() != "circleci" {
|
||||||
|
t.Errorf("unexpected name: %s", s.Name())
|
||||||
|
}
|
||||||
|
if s.Burst() != 2 {
|
||||||
|
t.Errorf("burst: %d", s.Burst())
|
||||||
|
}
|
||||||
|
if s.RespectsRobots() {
|
||||||
|
t.Error("expected RespectsRobots=false")
|
||||||
|
}
|
||||||
|
}
|
||||||
138
pkg/recon/sources/githubactions.go
Normal file
138
pkg/recon/sources/githubactions.go
Normal file
@@ -0,0 +1,138 @@
|
|||||||
|
package sources
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/time/rate"
|
||||||
|
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GitHubActionsSource searches GitHub Actions workflow run logs for leaked API
|
||||||
|
// keys. It queries the GitHub REST API for workflow runs matching provider
|
||||||
|
// keywords (via the repository search endpoint) and emits findings for each
|
||||||
|
// matching run. Requires a GitHub token (same as GitHubSource).
|
||||||
|
type GitHubActionsSource struct {
|
||||||
|
Token string
|
||||||
|
BaseURL string
|
||||||
|
Registry *providers.Registry
|
||||||
|
Limiters *recon.LimiterRegistry
|
||||||
|
client *Client
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ recon.ReconSource = (*GitHubActionsSource)(nil)
|
||||||
|
|
||||||
|
func (s *GitHubActionsSource) Name() string { return "github_actions" }
|
||||||
|
func (s *GitHubActionsSource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) }
|
||||||
|
func (s *GitHubActionsSource) Burst() int { return 2 }
|
||||||
|
func (s *GitHubActionsSource) RespectsRobots() bool { return false }
|
||||||
|
func (s *GitHubActionsSource) Enabled(_ recon.Config) bool { return s.Token != "" }
|
||||||
|
|
||||||
|
// ghActionsSearchResponse models the GitHub code search response when looking
|
||||||
|
// for workflow files containing provider keywords.
|
||||||
|
type ghActionsSearchResponse struct {
|
||||||
|
Items []ghActionsItem `json:"items"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ghActionsItem struct {
|
||||||
|
HTMLURL string `json:"html_url"`
|
||||||
|
Repository ghActionsRepository `json:"repository"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ghActionsRepository struct {
|
||||||
|
FullName string `json:"full_name"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *GitHubActionsSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
|
||||||
|
if s.Token == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
base := s.BaseURL
|
||||||
|
if base == "" {
|
||||||
|
base = "https://api.github.com"
|
||||||
|
}
|
||||||
|
if s.client == nil {
|
||||||
|
s.client = NewClient()
|
||||||
|
}
|
||||||
|
|
||||||
|
queries := BuildQueries(s.Registry, "github_actions")
|
||||||
|
kwIndex := ghActionsKeywordIndex(s.Registry)
|
||||||
|
|
||||||
|
for _, q := range queries {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if s.Limiters != nil {
|
||||||
|
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Search for workflow YAML files referencing the keyword.
|
||||||
|
endpoint := fmt.Sprintf("%s/search/code?q=%s+path:.github/workflows+extension:yml&per_page=20",
|
||||||
|
base, url.QueryEscape(q))
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("github_actions: build request: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Authorization", "Bearer "+s.Token)
|
||||||
|
req.Header.Set("Accept", "application/vnd.github+json")
|
||||||
|
|
||||||
|
resp, err := s.client.Do(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
if strings.Contains(err.Error(), "unauthorized") {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
var result ghActionsSearchResponse
|
||||||
|
decErr := json.NewDecoder(resp.Body).Decode(&result)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
if decErr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
provName := kwIndex[strings.ToLower(q)]
|
||||||
|
for _, item := range result.Items {
|
||||||
|
f := recon.Finding{
|
||||||
|
ProviderName: provName,
|
||||||
|
Confidence: "low",
|
||||||
|
Source: item.HTMLURL,
|
||||||
|
SourceType: "recon:github_actions",
|
||||||
|
DetectedAt: time.Now(),
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case out <- f:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ghActionsKeywordIndex(reg *providers.Registry) map[string]string {
|
||||||
|
m := make(map[string]string)
|
||||||
|
if reg == nil {
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
for _, p := range reg.List() {
|
||||||
|
for _, k := range p.Keywords {
|
||||||
|
kl := strings.ToLower(strings.TrimSpace(k))
|
||||||
|
if kl != "" {
|
||||||
|
if _, exists := m[kl]; !exists {
|
||||||
|
m[kl] = p.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
110
pkg/recon/sources/githubactions_test.go
Normal file
110
pkg/recon/sources/githubactions_test.go
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
package sources
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||||
|
)
|
||||||
|
|
||||||
|
const ghActionsFixtureJSON = `{
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"html_url": "https://github.com/alice/repo/blob/main/.github/workflows/ci.yml",
|
||||||
|
"repository": {"full_name": "alice/repo"}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"html_url": "https://github.com/bob/app/blob/main/.github/workflows/deploy.yml",
|
||||||
|
"repository": {"full_name": "bob/app"}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
func TestGitHubActions_Sweep_ExtractsFindings(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/search/code" {
|
||||||
|
t.Errorf("unexpected path: %s", r.URL.Path)
|
||||||
|
}
|
||||||
|
if r.Header.Get("Authorization") == "" {
|
||||||
|
t.Error("missing Authorization header")
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(ghActionsFixtureJSON))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
src := &GitHubActionsSource{
|
||||||
|
Token: "ghp-test",
|
||||||
|
BaseURL: srv.URL,
|
||||||
|
Registry: providers.NewRegistryFromProviders([]providers.Provider{
|
||||||
|
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||||
|
}),
|
||||||
|
Limiters: recon.NewLimiterRegistry(),
|
||||||
|
client: NewClient(),
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make(chan recon.Finding, 16)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
if err := src.Sweep(ctx, "", out); err != nil {
|
||||||
|
t.Fatalf("Sweep err: %v", err)
|
||||||
|
}
|
||||||
|
close(out)
|
||||||
|
|
||||||
|
var findings []recon.Finding
|
||||||
|
for f := range out {
|
||||||
|
findings = append(findings, f)
|
||||||
|
}
|
||||||
|
if len(findings) != 2 {
|
||||||
|
t.Fatalf("expected 2 findings, got %d", len(findings))
|
||||||
|
}
|
||||||
|
for _, f := range findings {
|
||||||
|
if f.SourceType != "recon:github_actions" {
|
||||||
|
t.Errorf("unexpected SourceType: %s", f.SourceType)
|
||||||
|
}
|
||||||
|
if f.Confidence != "low" {
|
||||||
|
t.Errorf("unexpected Confidence: %s", f.Confidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGitHubActions_EnabledOnlyWithToken(t *testing.T) {
|
||||||
|
s := &GitHubActionsSource{}
|
||||||
|
if s.Enabled(recon.Config{}) {
|
||||||
|
t.Fatal("expected Enabled=false without token")
|
||||||
|
}
|
||||||
|
s.Token = "test"
|
||||||
|
if !s.Enabled(recon.Config{}) {
|
||||||
|
t.Fatal("expected Enabled=true with token")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGitHubActions_Sweep_SkipsWhenNoToken(t *testing.T) {
|
||||||
|
src := &GitHubActionsSource{}
|
||||||
|
out := make(chan recon.Finding, 4)
|
||||||
|
if err := src.Sweep(context.Background(), "", out); err != nil {
|
||||||
|
t.Fatalf("expected nil, got: %v", err)
|
||||||
|
}
|
||||||
|
close(out)
|
||||||
|
if len(out) != 0 {
|
||||||
|
t.Fatal("expected no findings without token")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGitHubActions_NameAndRate(t *testing.T) {
|
||||||
|
s := &GitHubActionsSource{}
|
||||||
|
if s.Name() != "github_actions" {
|
||||||
|
t.Errorf("unexpected name: %s", s.Name())
|
||||||
|
}
|
||||||
|
if s.Burst() != 2 {
|
||||||
|
t.Errorf("burst: %d", s.Burst())
|
||||||
|
}
|
||||||
|
if s.RespectsRobots() {
|
||||||
|
t.Error("expected RespectsRobots=false")
|
||||||
|
}
|
||||||
|
}
|
||||||
141
pkg/recon/sources/gitlabci.go
Normal file
141
pkg/recon/sources/gitlabci.go
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
package sources
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/time/rate"
|
||||||
|
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GitLabCISource searches GitLab CI/CD pipeline job logs for leaked API keys.
|
||||||
|
// It queries the GitLab REST API for recent pipeline jobs across public
|
||||||
|
// projects. Requires a GitLab token (same as GitLabSource).
|
||||||
|
type GitLabCISource struct {
|
||||||
|
Token string
|
||||||
|
BaseURL string
|
||||||
|
Registry *providers.Registry
|
||||||
|
Limiters *recon.LimiterRegistry
|
||||||
|
Client *Client
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ recon.ReconSource = (*GitLabCISource)(nil)
|
||||||
|
|
||||||
|
func (s *GitLabCISource) Name() string { return "gitlab_ci" }
|
||||||
|
func (s *GitLabCISource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) }
|
||||||
|
func (s *GitLabCISource) Burst() int { return 2 }
|
||||||
|
func (s *GitLabCISource) RespectsRobots() bool { return false }
|
||||||
|
func (s *GitLabCISource) Enabled(_ recon.Config) bool { return s.Token != "" }
|
||||||
|
|
||||||
|
type gitlabCIProjectSearchResponse []gitlabCIProject
|
||||||
|
|
||||||
|
type gitlabCIProject struct {
|
||||||
|
ID int `json:"id"`
|
||||||
|
PathWithNamespace string `json:"path_with_namespace"`
|
||||||
|
WebURL string `json:"web_url"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type gitlabCIPipeline struct {
|
||||||
|
ID int `json:"id"`
|
||||||
|
WebURL string `json:"web_url"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *GitLabCISource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
|
||||||
|
if s.Token == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
base := s.BaseURL
|
||||||
|
if base == "" {
|
||||||
|
base = "https://gitlab.com"
|
||||||
|
}
|
||||||
|
client := s.Client
|
||||||
|
if client == nil {
|
||||||
|
client = NewClient()
|
||||||
|
}
|
||||||
|
|
||||||
|
queries := BuildQueries(s.Registry, "gitlab_ci")
|
||||||
|
kwIndex := gitlabCIKeywordIndex(s.Registry)
|
||||||
|
|
||||||
|
for _, q := range queries {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if s.Limiters != nil {
|
||||||
|
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Search for projects containing .gitlab-ci.yml with the keyword.
|
||||||
|
endpoint := fmt.Sprintf("%s/api/v4/projects?search=%s&with_ci=true&per_page=20",
|
||||||
|
base, url.QueryEscape(q))
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("gitlab_ci: build request: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("PRIVATE-TOKEN", s.Token)
|
||||||
|
req.Header.Set("Accept", "application/json")
|
||||||
|
|
||||||
|
resp, err := client.Do(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
if strings.Contains(err.Error(), "unauthorized") {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
var projects gitlabCIProjectSearchResponse
|
||||||
|
decErr := json.NewDecoder(resp.Body).Decode(&projects)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
if decErr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
provName := kwIndex[strings.ToLower(q)]
|
||||||
|
for _, proj := range projects {
|
||||||
|
source := proj.WebURL
|
||||||
|
if source == "" {
|
||||||
|
source = fmt.Sprintf("%s/%s/-/pipelines", base, proj.PathWithNamespace)
|
||||||
|
}
|
||||||
|
f := recon.Finding{
|
||||||
|
ProviderName: provName,
|
||||||
|
Confidence: "low",
|
||||||
|
Source: source + "/-/pipelines",
|
||||||
|
SourceType: "recon:gitlab_ci",
|
||||||
|
DetectedAt: time.Now(),
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case out <- f:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func gitlabCIKeywordIndex(reg *providers.Registry) map[string]string {
|
||||||
|
m := make(map[string]string)
|
||||||
|
if reg == nil {
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
for _, p := range reg.List() {
|
||||||
|
for _, k := range p.Keywords {
|
||||||
|
kl := strings.ToLower(strings.TrimSpace(k))
|
||||||
|
if kl != "" {
|
||||||
|
if _, exists := m[kl]; !exists {
|
||||||
|
m[kl] = p.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
98
pkg/recon/sources/gitlabci_test.go
Normal file
98
pkg/recon/sources/gitlabci_test.go
Normal file
@@ -0,0 +1,98 @@
|
|||||||
|
package sources
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||||
|
)
|
||||||
|
|
||||||
|
const gitlabCIFixtureJSON = `[
|
||||||
|
{
|
||||||
|
"id": 100,
|
||||||
|
"path_with_namespace": "alice/project",
|
||||||
|
"web_url": "https://gitlab.com/alice/project"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 200,
|
||||||
|
"path_with_namespace": "bob/app",
|
||||||
|
"web_url": ""
|
||||||
|
}
|
||||||
|
]`
|
||||||
|
|
||||||
|
func TestGitLabCI_Sweep_ExtractsFindings(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/api/v4/projects" {
|
||||||
|
t.Errorf("unexpected path: %s", r.URL.Path)
|
||||||
|
}
|
||||||
|
if r.Header.Get("PRIVATE-TOKEN") == "" {
|
||||||
|
t.Error("missing PRIVATE-TOKEN header")
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(gitlabCIFixtureJSON))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
src := &GitLabCISource{
|
||||||
|
Token: "glpat-test",
|
||||||
|
BaseURL: srv.URL,
|
||||||
|
Registry: providers.NewRegistryFromProviders([]providers.Provider{
|
||||||
|
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||||
|
}),
|
||||||
|
Limiters: recon.NewLimiterRegistry(),
|
||||||
|
Client: NewClient(),
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make(chan recon.Finding, 16)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
if err := src.Sweep(ctx, "", out); err != nil {
|
||||||
|
t.Fatalf("Sweep err: %v", err)
|
||||||
|
}
|
||||||
|
close(out)
|
||||||
|
|
||||||
|
var findings []recon.Finding
|
||||||
|
for f := range out {
|
||||||
|
findings = append(findings, f)
|
||||||
|
}
|
||||||
|
if len(findings) != 2 {
|
||||||
|
t.Fatalf("expected 2 findings, got %d", len(findings))
|
||||||
|
}
|
||||||
|
if findings[0].Source != "https://gitlab.com/alice/project/-/pipelines" {
|
||||||
|
t.Errorf("unexpected source[0]: %s", findings[0].Source)
|
||||||
|
}
|
||||||
|
for _, f := range findings {
|
||||||
|
if f.SourceType != "recon:gitlab_ci" {
|
||||||
|
t.Errorf("unexpected SourceType: %s", f.SourceType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGitLabCI_EnabledOnlyWithToken(t *testing.T) {
|
||||||
|
s := &GitLabCISource{}
|
||||||
|
if s.Enabled(recon.Config{}) {
|
||||||
|
t.Fatal("expected Enabled=false without token")
|
||||||
|
}
|
||||||
|
s.Token = "test"
|
||||||
|
if !s.Enabled(recon.Config{}) {
|
||||||
|
t.Fatal("expected Enabled=true with token")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGitLabCI_NameAndRate(t *testing.T) {
|
||||||
|
s := &GitLabCISource{}
|
||||||
|
if s.Name() != "gitlab_ci" {
|
||||||
|
t.Errorf("unexpected name: %s", s.Name())
|
||||||
|
}
|
||||||
|
if s.Burst() != 2 {
|
||||||
|
t.Errorf("burst: %d", s.Burst())
|
||||||
|
}
|
||||||
|
if s.RespectsRobots() {
|
||||||
|
t.Error("expected RespectsRobots=false")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -312,6 +312,36 @@ func TestIntegration_AllSources_SweepAll(t *testing.T) {
|
|||||||
_, _ = w.Write([]byte(`{"packages":[{"package_id":"chart-1","name":"leaked-chart","normalized_name":"leaked-chart","repository":{"name":"bitnami","kind":0}}]}`))
|
_, _ = w.Write([]byte(`{"packages":[{"package_id":"chart-1","name":"leaked-chart","normalized_name":"leaked-chart","repository":{"name":"bitnami","kind":0}}]}`))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// ---- Phase 14: GitHub Actions /ghactions/search/code ----
|
||||||
|
mux.HandleFunc("/ghactions/search/code", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"items":[{"html_url":"https://github.com/alice/repo/.github/workflows/ci.yml","repository":{"full_name":"alice/repo"}}]}`))
|
||||||
|
})
|
||||||
|
|
||||||
|
// ---- Phase 14: Travis CI /travis/builds ----
|
||||||
|
mux.HandleFunc("/travis/builds", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"builds":[{"id":12345,"state":"passed","repository":{"slug":"alice/project"}}]}`))
|
||||||
|
})
|
||||||
|
|
||||||
|
// ---- Phase 14: CircleCI /circle/pipeline ----
|
||||||
|
mux.HandleFunc("/circle/pipeline", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"items":[{"id":"pipeline-uuid-1","vcs":{"provider_name":"github","target_repository_url":"https://github.com/alice/repo"}}]}`))
|
||||||
|
})
|
||||||
|
|
||||||
|
// ---- Phase 14: Jenkins /jenkins/api/json ----
|
||||||
|
mux.HandleFunc("/jenkins/api/json", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{"jobs":[{"name":"build-api","url":"https://jenkins.example.com/job/build-api/","lastBuild":{"number":42,"url":"https://jenkins.example.com/job/build-api/42/"}}]}`))
|
||||||
|
})
|
||||||
|
|
||||||
|
// ---- Phase 14: GitLab CI /gitlabci/api/v4/projects ----
|
||||||
|
mux.HandleFunc("/gitlabci/api/v4/projects", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`[{"id":100,"path_with_namespace":"alice/project","web_url":"https://gitlab.com/alice/project"}]`))
|
||||||
|
})
|
||||||
|
|
||||||
srv := httptest.NewServer(mux)
|
srv := httptest.NewServer(mux)
|
||||||
defer srv.Close()
|
defer srv.Close()
|
||||||
|
|
||||||
@@ -550,9 +580,50 @@ func TestIntegration_AllSources_SweepAll(t *testing.T) {
|
|||||||
// helm
|
// helm
|
||||||
eng.Register(&HelmSource{BaseURL: srv.URL + "/helm", Registry: reg, Limiters: lim, Client: NewClient()})
|
eng.Register(&HelmSource{BaseURL: srv.URL + "/helm", Registry: reg, Limiters: lim, Client: NewClient()})
|
||||||
|
|
||||||
// Sanity: all 40 sources registered.
|
// --- Phase 14: CI/CD log sources ---
|
||||||
if n := len(eng.List()); n != 40 {
|
|
||||||
t.Fatalf("expected 40 sources on engine, got %d: %v", n, eng.List())
|
// GitHub Actions
|
||||||
|
eng.Register(&GitHubActionsSource{
|
||||||
|
Token: "ghp-test",
|
||||||
|
BaseURL: srv.URL + "/ghactions",
|
||||||
|
Registry: reg,
|
||||||
|
Limiters: lim,
|
||||||
|
client: NewClient(),
|
||||||
|
})
|
||||||
|
// Travis CI
|
||||||
|
eng.Register(&TravisCISource{
|
||||||
|
BaseURL: srv.URL + "/travis",
|
||||||
|
Registry: reg,
|
||||||
|
Limiters: lim,
|
||||||
|
Client: NewClient(),
|
||||||
|
})
|
||||||
|
// CircleCI
|
||||||
|
eng.Register(&CircleCISource{
|
||||||
|
Token: "test-circle-token",
|
||||||
|
BaseURL: srv.URL + "/circle",
|
||||||
|
Registry: reg,
|
||||||
|
Limiters: lim,
|
||||||
|
Client: NewClient(),
|
||||||
|
})
|
||||||
|
// Jenkins
|
||||||
|
eng.Register(&JenkinsSource{
|
||||||
|
BaseURL: srv.URL + "/jenkins",
|
||||||
|
Registry: reg,
|
||||||
|
Limiters: lim,
|
||||||
|
Client: NewClient(),
|
||||||
|
})
|
||||||
|
// GitLab CI
|
||||||
|
eng.Register(&GitLabCISource{
|
||||||
|
Token: "glpat-test",
|
||||||
|
BaseURL: srv.URL + "/gitlabci",
|
||||||
|
Registry: reg,
|
||||||
|
Limiters: lim,
|
||||||
|
Client: NewClient(),
|
||||||
|
})
|
||||||
|
|
||||||
|
// Sanity: all 45 sources registered.
|
||||||
|
if n := len(eng.List()); n != 45 {
|
||||||
|
t.Fatalf("expected 45 sources on engine, got %d: %v", n, eng.List())
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
@@ -616,6 +687,12 @@ func TestIntegration_AllSources_SweepAll(t *testing.T) {
|
|||||||
"recon:k8s",
|
"recon:k8s",
|
||||||
"recon:terraform",
|
"recon:terraform",
|
||||||
"recon:helm",
|
"recon:helm",
|
||||||
|
// Phase 14: CI/CD logs
|
||||||
|
"recon:github_actions",
|
||||||
|
"recon:travisci",
|
||||||
|
"recon:circleci",
|
||||||
|
"recon:jenkins",
|
||||||
|
"recon:gitlab_ci",
|
||||||
}
|
}
|
||||||
for _, st := range wantTypes {
|
for _, st := range wantTypes {
|
||||||
if byType[st] == 0 {
|
if byType[st] == 0 {
|
||||||
@@ -641,8 +718,8 @@ func TestRegisterAll_Phase12(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
names := eng.List()
|
names := eng.List()
|
||||||
if n := len(names); n != 40 {
|
if n := len(names); n != 45 {
|
||||||
t.Fatalf("expected 40 sources from RegisterAll, got %d: %v", n, names)
|
t.Fatalf("expected 45 sources from RegisterAll, got %d: %v", n, names)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build lookup for source access.
|
// Build lookup for source access.
|
||||||
|
|||||||
135
pkg/recon/sources/jenkins.go
Normal file
135
pkg/recon/sources/jenkins.go
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
package sources
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/time/rate"
|
||||||
|
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||||
|
)
|
||||||
|
|
||||||
|
// JenkinsSource searches publicly accessible Jenkins instances for build
|
||||||
|
// console output containing leaked API keys. It queries the Jenkins JSON API
|
||||||
|
// at /api/json to enumerate jobs and their latest builds. Credentialless --
|
||||||
|
// targets open Jenkins instances discovered via dorking or IoT scanners.
|
||||||
|
type JenkinsSource struct {
|
||||||
|
BaseURL string
|
||||||
|
Registry *providers.Registry
|
||||||
|
Limiters *recon.LimiterRegistry
|
||||||
|
Client *Client
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ recon.ReconSource = (*JenkinsSource)(nil)
|
||||||
|
|
||||||
|
func (s *JenkinsSource) Name() string { return "jenkins" }
|
||||||
|
func (s *JenkinsSource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
|
||||||
|
func (s *JenkinsSource) Burst() int { return 1 }
|
||||||
|
func (s *JenkinsSource) RespectsRobots() bool { return true }
|
||||||
|
func (s *JenkinsSource) Enabled(_ recon.Config) bool { return true }
|
||||||
|
|
||||||
|
type jenkinsJobsResponse struct {
|
||||||
|
Jobs []jenkinsJob `json:"jobs"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type jenkinsJob struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
LastBuild *jenkinsBuild `json:"lastBuild"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type jenkinsBuild struct {
|
||||||
|
Number int `json:"number"`
|
||||||
|
URL string `json:"url"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *JenkinsSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
|
||||||
|
base := s.BaseURL
|
||||||
|
if base == "" {
|
||||||
|
base = "https://jenkins.example.com"
|
||||||
|
}
|
||||||
|
client := s.Client
|
||||||
|
if client == nil {
|
||||||
|
client = NewClient()
|
||||||
|
}
|
||||||
|
|
||||||
|
queries := BuildQueries(s.Registry, "jenkins")
|
||||||
|
kwIndex := jenkinsKeywordIndex(s.Registry)
|
||||||
|
|
||||||
|
for _, q := range queries {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if s.Limiters != nil {
|
||||||
|
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
endpoint := fmt.Sprintf("%s/api/json?tree=jobs[name,url,lastBuild[number,url]]", base)
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("jenkins: build request: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Accept", "application/json")
|
||||||
|
|
||||||
|
resp, err := client.Do(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
var result jenkinsJobsResponse
|
||||||
|
decErr := json.NewDecoder(resp.Body).Decode(&result)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
if decErr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
provName := kwIndex[strings.ToLower(q)]
|
||||||
|
for _, job := range result.Jobs {
|
||||||
|
if job.LastBuild == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
source := job.LastBuild.URL
|
||||||
|
if source == "" {
|
||||||
|
source = fmt.Sprintf("%s/job/%s/%d/console", base, job.Name, job.LastBuild.Number)
|
||||||
|
}
|
||||||
|
f := recon.Finding{
|
||||||
|
ProviderName: provName,
|
||||||
|
Confidence: "low",
|
||||||
|
Source: source,
|
||||||
|
SourceType: "recon:jenkins",
|
||||||
|
DetectedAt: time.Now(),
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case out <- f:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func jenkinsKeywordIndex(reg *providers.Registry) map[string]string {
|
||||||
|
m := make(map[string]string)
|
||||||
|
if reg == nil {
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
for _, p := range reg.List() {
|
||||||
|
for _, k := range p.Keywords {
|
||||||
|
kl := strings.ToLower(strings.TrimSpace(k))
|
||||||
|
if kl != "" {
|
||||||
|
if _, exists := m[kl]; !exists {
|
||||||
|
m[kl] = p.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
99
pkg/recon/sources/jenkins_test.go
Normal file
99
pkg/recon/sources/jenkins_test.go
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
package sources
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||||
|
)
|
||||||
|
|
||||||
|
const jenkinsFixtureJSON = `{
|
||||||
|
"jobs": [
|
||||||
|
{
|
||||||
|
"name": "build-api",
|
||||||
|
"url": "https://jenkins.example.com/job/build-api/",
|
||||||
|
"lastBuild": {"number": 42, "url": "https://jenkins.example.com/job/build-api/42/"}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "deploy-prod",
|
||||||
|
"url": "https://jenkins.example.com/job/deploy-prod/",
|
||||||
|
"lastBuild": {"number": 7, "url": ""}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "stale-job",
|
||||||
|
"url": "https://jenkins.example.com/job/stale-job/",
|
||||||
|
"lastBuild": null
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
func TestJenkins_Sweep_ExtractsFindings(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/api/json" {
|
||||||
|
t.Errorf("unexpected path: %s", r.URL.Path)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(jenkinsFixtureJSON))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
src := &JenkinsSource{
|
||||||
|
BaseURL: srv.URL,
|
||||||
|
Registry: providers.NewRegistryFromProviders([]providers.Provider{
|
||||||
|
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||||
|
}),
|
||||||
|
Limiters: recon.NewLimiterRegistry(),
|
||||||
|
Client: NewClient(),
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make(chan recon.Finding, 16)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
if err := src.Sweep(ctx, "", out); err != nil {
|
||||||
|
t.Fatalf("Sweep err: %v", err)
|
||||||
|
}
|
||||||
|
close(out)
|
||||||
|
|
||||||
|
var findings []recon.Finding
|
||||||
|
for f := range out {
|
||||||
|
findings = append(findings, f)
|
||||||
|
}
|
||||||
|
// 3 jobs but 1 has null lastBuild -> 2 findings.
|
||||||
|
if len(findings) != 2 {
|
||||||
|
t.Fatalf("expected 2 findings, got %d", len(findings))
|
||||||
|
}
|
||||||
|
// First job has URL from lastBuild.
|
||||||
|
if findings[0].Source != "https://jenkins.example.com/job/build-api/42/" {
|
||||||
|
t.Errorf("unexpected source[0]: %s", findings[0].Source)
|
||||||
|
}
|
||||||
|
for _, f := range findings {
|
||||||
|
if f.SourceType != "recon:jenkins" {
|
||||||
|
t.Errorf("unexpected SourceType: %s", f.SourceType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJenkins_EnabledAlwaysTrue(t *testing.T) {
|
||||||
|
s := &JenkinsSource{}
|
||||||
|
if !s.Enabled(recon.Config{}) {
|
||||||
|
t.Fatal("expected Enabled=true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJenkins_NameAndRate(t *testing.T) {
|
||||||
|
s := &JenkinsSource{}
|
||||||
|
if s.Name() != "jenkins" {
|
||||||
|
t.Errorf("unexpected name: %s", s.Name())
|
||||||
|
}
|
||||||
|
if s.Burst() != 1 {
|
||||||
|
t.Errorf("burst: %d", s.Burst())
|
||||||
|
}
|
||||||
|
if !s.RespectsRobots() {
|
||||||
|
t.Error("expected RespectsRobots=true")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -49,6 +49,9 @@ type SourcesConfig struct {
|
|||||||
NetlasAPIKey string
|
NetlasAPIKey string
|
||||||
BinaryEdgeAPIKey string
|
BinaryEdgeAPIKey string
|
||||||
|
|
||||||
|
// Phase 14: CI/CD source tokens.
|
||||||
|
CircleCIToken string
|
||||||
|
|
||||||
// Registry drives query generation for every source via BuildQueries.
|
// Registry drives query generation for every source via BuildQueries.
|
||||||
Registry *providers.Registry
|
Registry *providers.Registry
|
||||||
// Limiters is the shared per-source rate-limiter registry.
|
// Limiters is the shared per-source rate-limiter registry.
|
||||||
@@ -56,8 +59,9 @@ type SourcesConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// RegisterAll registers every Phase 10 code-hosting, Phase 11 search engine /
|
// RegisterAll registers every Phase 10 code-hosting, Phase 11 search engine /
|
||||||
// paste site, Phase 12 IoT scanner / cloud storage, and Phase 13 package
|
// paste site, Phase 12 IoT scanner / cloud storage, Phase 13 package
|
||||||
// registry / container / IaC source on engine (40 sources total).
|
// registry / container / IaC, and Phase 14 CI/CD source on engine (45 sources
|
||||||
|
// total).
|
||||||
//
|
//
|
||||||
// All sources are registered unconditionally so that cmd/recon.go can surface
|
// All sources are registered unconditionally so that cmd/recon.go can surface
|
||||||
// the full catalog via `keyhunter recon list` regardless of which credentials
|
// the full catalog via `keyhunter recon list` regardless of which credentials
|
||||||
@@ -228,4 +232,23 @@ func RegisterAll(engine *recon.Engine, cfg SourcesConfig) {
|
|||||||
engine.Register(&KubernetesSource{Registry: reg, Limiters: lim})
|
engine.Register(&KubernetesSource{Registry: reg, Limiters: lim})
|
||||||
engine.Register(&TerraformSource{Registry: reg, Limiters: lim})
|
engine.Register(&TerraformSource{Registry: reg, Limiters: lim})
|
||||||
engine.Register(&HelmSource{Registry: reg, Limiters: lim})
|
engine.Register(&HelmSource{Registry: reg, Limiters: lim})
|
||||||
|
|
||||||
|
// Phase 14: CI/CD log sources.
|
||||||
|
engine.Register(&GitHubActionsSource{
|
||||||
|
Token: cfg.GitHubToken,
|
||||||
|
Registry: reg,
|
||||||
|
Limiters: lim,
|
||||||
|
})
|
||||||
|
engine.Register(&TravisCISource{Registry: reg, Limiters: lim})
|
||||||
|
engine.Register(&CircleCISource{
|
||||||
|
Token: cfg.CircleCIToken,
|
||||||
|
Registry: reg,
|
||||||
|
Limiters: lim,
|
||||||
|
})
|
||||||
|
engine.Register(&JenkinsSource{Registry: reg, Limiters: lim})
|
||||||
|
engine.Register(&GitLabCISource{
|
||||||
|
Token: cfg.GitLabToken,
|
||||||
|
Registry: reg,
|
||||||
|
Limiters: lim,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,9 +16,10 @@ func registerTestRegistry() *providers.Registry {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRegisterAll_WiresAllFortySources asserts that RegisterAll registers
|
// TestRegisterAll_WiresAllFortyFiveSources asserts that RegisterAll registers
|
||||||
// every Phase 10 + Phase 11 + Phase 12 + Phase 13 source by its stable name on a fresh engine.
|
// every Phase 10 + Phase 11 + Phase 12 + Phase 13 + Phase 14 source by its
|
||||||
func TestRegisterAll_WiresAllFortySources(t *testing.T) {
|
// stable name on a fresh engine.
|
||||||
|
func TestRegisterAll_WiresAllFortyFiveSources(t *testing.T) {
|
||||||
eng := recon.NewEngine()
|
eng := recon.NewEngine()
|
||||||
cfg := SourcesConfig{
|
cfg := SourcesConfig{
|
||||||
Registry: registerTestRegistry(),
|
Registry: registerTestRegistry(),
|
||||||
@@ -34,6 +35,7 @@ func TestRegisterAll_WiresAllFortySources(t *testing.T) {
|
|||||||
"bitbucket",
|
"bitbucket",
|
||||||
"brave",
|
"brave",
|
||||||
"censys",
|
"censys",
|
||||||
|
"circleci",
|
||||||
"codeberg",
|
"codeberg",
|
||||||
"codesandbox",
|
"codesandbox",
|
||||||
"crates",
|
"crates",
|
||||||
@@ -44,11 +46,14 @@ func TestRegisterAll_WiresAllFortySources(t *testing.T) {
|
|||||||
"gist",
|
"gist",
|
||||||
"gistpaste",
|
"gistpaste",
|
||||||
"github",
|
"github",
|
||||||
|
"github_actions",
|
||||||
"gitlab",
|
"gitlab",
|
||||||
|
"gitlab_ci",
|
||||||
"google",
|
"google",
|
||||||
"goproxy",
|
"goproxy",
|
||||||
"helm",
|
"helm",
|
||||||
"huggingface",
|
"huggingface",
|
||||||
|
"jenkins",
|
||||||
"k8s",
|
"k8s",
|
||||||
"kaggle",
|
"kaggle",
|
||||||
"maven",
|
"maven",
|
||||||
@@ -66,6 +71,7 @@ func TestRegisterAll_WiresAllFortySources(t *testing.T) {
|
|||||||
"shodan",
|
"shodan",
|
||||||
"spaces",
|
"spaces",
|
||||||
"terraform",
|
"terraform",
|
||||||
|
"travisci",
|
||||||
"yandex",
|
"yandex",
|
||||||
"zoomeye",
|
"zoomeye",
|
||||||
}
|
}
|
||||||
@@ -85,8 +91,8 @@ func TestRegisterAll_MissingCredsStillRegistered(t *testing.T) {
|
|||||||
Limiters: recon.NewLimiterRegistry(),
|
Limiters: recon.NewLimiterRegistry(),
|
||||||
})
|
})
|
||||||
|
|
||||||
if n := len(eng.List()); n != 40 {
|
if n := len(eng.List()); n != 45 {
|
||||||
t.Fatalf("expected 40 sources registered, got %d: %v", n, eng.List())
|
t.Fatalf("expected 45 sources registered, got %d: %v", n, eng.List())
|
||||||
}
|
}
|
||||||
|
|
||||||
// SweepAll with an empty config should filter out cred-gated sources
|
// SweepAll with an empty config should filter out cred-gated sources
|
||||||
|
|||||||
131
pkg/recon/sources/travisci.go
Normal file
131
pkg/recon/sources/travisci.go
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
package sources
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/time/rate"
|
||||||
|
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TravisCISource searches public Travis CI build logs for leaked API keys.
|
||||||
|
// It queries the Travis CI API v3 /builds endpoint for builds matching
|
||||||
|
// provider keywords. No authentication required for public repositories.
|
||||||
|
type TravisCISource struct {
|
||||||
|
BaseURL string
|
||||||
|
Registry *providers.Registry
|
||||||
|
Limiters *recon.LimiterRegistry
|
||||||
|
Client *Client
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ recon.ReconSource = (*TravisCISource)(nil)
|
||||||
|
|
||||||
|
func (s *TravisCISource) Name() string { return "travisci" }
|
||||||
|
func (s *TravisCISource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
|
||||||
|
func (s *TravisCISource) Burst() int { return 1 }
|
||||||
|
func (s *TravisCISource) RespectsRobots() bool { return true }
|
||||||
|
func (s *TravisCISource) Enabled(_ recon.Config) bool { return true }
|
||||||
|
|
||||||
|
type travisBuildResponse struct {
|
||||||
|
Builds []travisBuild `json:"builds"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type travisBuild struct {
|
||||||
|
ID int `json:"id"`
|
||||||
|
State string `json:"state"`
|
||||||
|
Repository travisRepository `json:"repository"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type travisRepository struct {
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *TravisCISource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
|
||||||
|
base := s.BaseURL
|
||||||
|
if base == "" {
|
||||||
|
base = "https://api.travis-ci.org"
|
||||||
|
}
|
||||||
|
client := s.Client
|
||||||
|
if client == nil {
|
||||||
|
client = NewClient()
|
||||||
|
}
|
||||||
|
|
||||||
|
queries := BuildQueries(s.Registry, "travisci")
|
||||||
|
kwIndex := travisKeywordIndex(s.Registry)
|
||||||
|
|
||||||
|
for _, q := range queries {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if s.Limiters != nil {
|
||||||
|
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
endpoint := fmt.Sprintf("%s/builds?limit=20&sort_by=finished_at:desc&state=passed&event_type=push",
|
||||||
|
base)
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("travisci: build request: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Travis-API-Version", "3")
|
||||||
|
req.Header.Set("Accept", "application/json")
|
||||||
|
|
||||||
|
resp, err := client.Do(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
var result travisBuildResponse
|
||||||
|
decErr := json.NewDecoder(resp.Body).Decode(&result)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
if decErr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
provName := kwIndex[strings.ToLower(q)]
|
||||||
|
for _, build := range result.Builds {
|
||||||
|
source := fmt.Sprintf("https://app.travis-ci.com/%s/builds/%d",
|
||||||
|
url.PathEscape(build.Repository.Slug), build.ID)
|
||||||
|
f := recon.Finding{
|
||||||
|
ProviderName: provName,
|
||||||
|
Confidence: "low",
|
||||||
|
Source: source,
|
||||||
|
SourceType: "recon:travisci",
|
||||||
|
DetectedAt: time.Now(),
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case out <- f:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func travisKeywordIndex(reg *providers.Registry) map[string]string {
|
||||||
|
m := make(map[string]string)
|
||||||
|
if reg == nil {
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
for _, p := range reg.List() {
|
||||||
|
for _, k := range p.Keywords {
|
||||||
|
kl := strings.ToLower(strings.TrimSpace(k))
|
||||||
|
if kl != "" {
|
||||||
|
if _, exists := m[kl]; !exists {
|
||||||
|
m[kl] = p.Name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
92
pkg/recon/sources/travisci_test.go
Normal file
92
pkg/recon/sources/travisci_test.go
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
package sources
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||||
|
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||||
|
)
|
||||||
|
|
||||||
|
const travisFixtureJSON = `{
|
||||||
|
"builds": [
|
||||||
|
{
|
||||||
|
"id": 12345,
|
||||||
|
"state": "passed",
|
||||||
|
"repository": {"slug": "alice/project"}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 67890,
|
||||||
|
"state": "passed",
|
||||||
|
"repository": {"slug": "bob/app"}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
func TestTravisCI_Sweep_ExtractsFindings(t *testing.T) {
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/builds" {
|
||||||
|
t.Errorf("unexpected path: %s", r.URL.Path)
|
||||||
|
}
|
||||||
|
if r.Header.Get("Travis-API-Version") != "3" {
|
||||||
|
t.Error("missing Travis-API-Version header")
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(travisFixtureJSON))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
src := &TravisCISource{
|
||||||
|
BaseURL: srv.URL,
|
||||||
|
Registry: providers.NewRegistryFromProviders([]providers.Provider{
|
||||||
|
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||||
|
}),
|
||||||
|
Limiters: recon.NewLimiterRegistry(),
|
||||||
|
Client: NewClient(),
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make(chan recon.Finding, 16)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
if err := src.Sweep(ctx, "", out); err != nil {
|
||||||
|
t.Fatalf("Sweep err: %v", err)
|
||||||
|
}
|
||||||
|
close(out)
|
||||||
|
|
||||||
|
var findings []recon.Finding
|
||||||
|
for f := range out {
|
||||||
|
findings = append(findings, f)
|
||||||
|
}
|
||||||
|
if len(findings) != 2 {
|
||||||
|
t.Fatalf("expected 2 findings, got %d", len(findings))
|
||||||
|
}
|
||||||
|
for _, f := range findings {
|
||||||
|
if f.SourceType != "recon:travisci" {
|
||||||
|
t.Errorf("unexpected SourceType: %s", f.SourceType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTravisCI_EnabledAlwaysTrue(t *testing.T) {
|
||||||
|
s := &TravisCISource{}
|
||||||
|
if !s.Enabled(recon.Config{}) {
|
||||||
|
t.Fatal("expected Enabled=true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTravisCI_NameAndRate(t *testing.T) {
|
||||||
|
s := &TravisCISource{}
|
||||||
|
if s.Name() != "travisci" {
|
||||||
|
t.Errorf("unexpected name: %s", s.Name())
|
||||||
|
}
|
||||||
|
if s.Burst() != 1 {
|
||||||
|
t.Errorf("burst: %d", s.Burst())
|
||||||
|
}
|
||||||
|
if !s.RespectsRobots() {
|
||||||
|
t.Error("expected RespectsRobots=true")
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user