From 0727b51d793e98fb5c871788de639327c8630999 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Mon, 6 Apr 2026 12:53:58 +0300 Subject: [PATCH] feat(13-03): implement TerraformSource and HelmSource - Terraform searches registry.terraform.io v1 modules API with namespace/name/provider URLs - Helm searches artifacthub.io for charts (kind=0) with repo/chart URL construction - Both sources: context cancellation, nil registry, httptest-based tests --- pkg/recon/sources/helm.go | 137 ++++++++++++++++++++ pkg/recon/sources/helm_test.go | 192 ++++++++++++++++++++++++++++ pkg/recon/sources/terraform.go | 131 +++++++++++++++++++ pkg/recon/sources/terraform_test.go | 190 +++++++++++++++++++++++++++ 4 files changed, 650 insertions(+) create mode 100644 pkg/recon/sources/helm.go create mode 100644 pkg/recon/sources/helm_test.go create mode 100644 pkg/recon/sources/terraform.go create mode 100644 pkg/recon/sources/terraform_test.go diff --git a/pkg/recon/sources/helm.go b/pkg/recon/sources/helm.go new file mode 100644 index 0000000..6dec0d5 --- /dev/null +++ b/pkg/recon/sources/helm.go @@ -0,0 +1,137 @@ +package sources + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// HelmSource searches Artifact Hub for Helm charts (kind=0) matching provider +// keywords. Helm charts that reference LLM/AI services may contain API keys +// in their default values.yaml files. +// +// Emits one Finding per chart result, tagged SourceType=recon:helm. +type HelmSource struct { + // BaseURL defaults to https://artifacthub.io. Tests override with httptest URL. + BaseURL string + // Registry drives the keyword query list via BuildQueries. + Registry *providers.Registry + // Limiters is the shared recon.LimiterRegistry. + Limiters *recon.LimiterRegistry + // Client is the shared retry HTTP wrapper. If nil, a default is used. + Client *Client +} + +// Compile-time assertion that HelmSource satisfies recon.ReconSource. +var _ recon.ReconSource = (*HelmSource)(nil) + +func (s *HelmSource) Name() string { return "helm" } +func (s *HelmSource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) } +func (s *HelmSource) Burst() int { return 2 } +func (s *HelmSource) RespectsRobots() bool { return false } + +// Enabled always returns true: Artifact Hub search is unauthenticated. +func (s *HelmSource) Enabled(_ recon.Config) bool { return true } + +// Sweep iterates provider keywords, searches Artifact Hub for Helm charts +// (kind=0), and emits a Finding for each result. +func (s *HelmSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error { + base := s.BaseURL + if base == "" { + base = "https://artifacthub.io" + } + client := s.Client + if client == nil { + client = NewClient() + } + + queries := BuildQueries(s.Registry, "helm") + if len(queries) == 0 { + return nil + } + + for _, q := range queries { + if err := ctx.Err(); err != nil { + return err + } + + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + // kind=0 filters to Helm charts only. + endpoint := fmt.Sprintf("%s/api/v1/packages/search?ts_query_web=%s&kind=0&limit=20", + base, url.QueryEscape(q)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("helm: build req: %w", err) + } + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(ctx, req) + if err != nil { + continue + } + + var parsed artifactHubSearchResponse + decErr := json.NewDecoder(resp.Body).Decode(&parsed) + _ = resp.Body.Close() + if decErr != nil { + continue + } + + for _, pkg := range parsed.Packages { + if err := ctx.Err(); err != nil { + return err + } + + repoName := pkg.Repository.Name + sourceURL := fmt.Sprintf("https://artifacthub.io/packages/helm/%s/%s", + repoName, pkg.NormalizedName) + if base != "https://artifacthub.io" { + sourceURL = fmt.Sprintf("%s/packages/helm/%s/%s", + base, repoName, pkg.NormalizedName) + } + + f := recon.Finding{ + ProviderName: "", + Source: sourceURL, + SourceType: "recon:helm", + Confidence: "low", + DetectedAt: time.Now(), + } + select { + case out <- f: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil +} + +type artifactHubSearchResponse struct { + Packages []artifactHubPackage `json:"packages"` +} + +type artifactHubPackage struct { + PackageID string `json:"package_id"` + Name string `json:"name"` + NormalizedName string `json:"normalized_name"` + Repository artifactHubRepo `json:"repository"` +} + +type artifactHubRepo struct { + Name string `json:"name"` + Kind int `json:"kind"` +} diff --git a/pkg/recon/sources/helm_test.go b/pkg/recon/sources/helm_test.go new file mode 100644 index 0000000..cd7ad62 --- /dev/null +++ b/pkg/recon/sources/helm_test.go @@ -0,0 +1,192 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +func helmStubHandler(t *testing.T, calls *int32) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(calls, 1) + if r.URL.Path != "/api/v1/packages/search" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + if r.URL.Query().Get("ts_query_web") == "" { + t.Errorf("missing ts_query_web param") + } + if got := r.URL.Query().Get("kind"); got != "0" { + t.Errorf("expected kind=0, got %q", got) + } + body := artifactHubSearchResponse{ + Packages: []artifactHubPackage{ + { + PackageID: "chart-1", + Name: "openai-proxy", + NormalizedName: "openai-proxy", + Repository: artifactHubRepo{Name: "bitnami", Kind: 0}, + }, + { + PackageID: "chart-2", + Name: "llm-stack", + NormalizedName: "llm-stack", + Repository: artifactHubRepo{Name: "community", Kind: 0}, + }, + }, + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(body) + } +} + +func TestHelm_SweepEmitsFindings(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("helm", 1000, 100) + + var calls int32 + srv := httptest.NewServer(helmStubHandler(t, &calls)) + defer srv.Close() + + src := &HelmSource{ + BaseURL: srv.URL, + Registry: reg, + Limiters: lim, + Client: NewClient(), + } + + out := make(chan recon.Finding, 32) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + done := make(chan error, 1) + go func() { done <- src.Sweep(ctx, "", out); close(out) }() + + var findings []recon.Finding + for f := range out { + findings = append(findings, f) + } + if err := <-done; err != nil { + t.Fatalf("Sweep error: %v", err) + } + + // 2 keywords * 2 charts = 4 findings + if len(findings) != 4 { + t.Fatalf("expected 4 findings, got %d", len(findings)) + } + for _, f := range findings { + if f.SourceType != "recon:helm" { + t.Errorf("SourceType=%q want recon:helm", f.SourceType) + } + } + if got := atomic.LoadInt32(&calls); got != 2 { + t.Errorf("expected 2 server calls, got %d", got) + } +} + +func TestHelm_ChartURLConstruction(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("helm", 1000, 100) + + var calls int32 + srv := httptest.NewServer(helmStubHandler(t, &calls)) + defer srv.Close() + + src := &HelmSource{ + BaseURL: srv.URL, + Registry: reg, + Limiters: lim, + Client: NewClient(), + } + + out := make(chan recon.Finding, 32) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + done := make(chan error, 1) + go func() { done <- src.Sweep(ctx, "", out); close(out) }() + + var findings []recon.Finding + for f := range out { + findings = append(findings, f) + } + if err := <-done; err != nil { + t.Fatalf("Sweep error: %v", err) + } + + hasBitnami := false + hasCommunity := false + for _, f := range findings { + if contains(f.Source, "/packages/helm/bitnami/openai-proxy") { + hasBitnami = true + } + if contains(f.Source, "/packages/helm/community/llm-stack") { + hasCommunity = true + } + } + if !hasBitnami { + t.Error("expected finding with bitnami/openai-proxy chart URL") + } + if !hasCommunity { + t.Error("expected finding with community/llm-stack chart URL") + } +} + +func TestHelm_EnabledAlwaysTrue(t *testing.T) { + s := &HelmSource{} + if !s.Enabled(recon.Config{}) { + t.Fatal("expected Enabled=true") + } +} + +func TestHelm_NameAndRate(t *testing.T) { + s := &HelmSource{} + if s.Name() != "helm" { + t.Errorf("unexpected name: %s", s.Name()) + } + if s.Burst() != 2 { + t.Errorf("burst: %d", s.Burst()) + } + if s.RespectsRobots() { + t.Error("expected RespectsRobots=false") + } +} + +func TestHelm_CtxCancelled(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("helm", 1000, 100) + + src := &HelmSource{ + BaseURL: "http://127.0.0.1:1", + Registry: reg, + Limiters: lim, + Client: NewClient(), + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + out := make(chan recon.Finding, 1) + err := src.Sweep(ctx, "", out) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } +} + +func TestHelm_NilRegistryNoError(t *testing.T) { + src := &HelmSource{Client: NewClient()} + out := make(chan recon.Finding, 1) + if err := src.Sweep(context.Background(), "", out); err != nil { + t.Fatalf("expected nil, got %v", err) + } +} diff --git a/pkg/recon/sources/terraform.go b/pkg/recon/sources/terraform.go new file mode 100644 index 0000000..fca46bb --- /dev/null +++ b/pkg/recon/sources/terraform.go @@ -0,0 +1,131 @@ +package sources + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// TerraformSource searches the Terraform Registry for modules matching +// provider keywords. Modules that reference LLM/AI provider credentials may +// contain hardcoded API keys in their variable defaults or examples. +// +// Emits one Finding per module result, tagged SourceType=recon:terraform. +type TerraformSource struct { + // BaseURL defaults to https://registry.terraform.io. Tests override. + BaseURL string + // Registry drives the keyword query list via BuildQueries. + Registry *providers.Registry + // Limiters is the shared recon.LimiterRegistry. + Limiters *recon.LimiterRegistry + // Client is the shared retry HTTP wrapper. If nil, a default is used. + Client *Client +} + +// Compile-time assertion that TerraformSource satisfies recon.ReconSource. +var _ recon.ReconSource = (*TerraformSource)(nil) + +func (s *TerraformSource) Name() string { return "terraform" } +func (s *TerraformSource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) } +func (s *TerraformSource) Burst() int { return 2 } +func (s *TerraformSource) RespectsRobots() bool { return false } + +// Enabled always returns true: Terraform Registry search is unauthenticated. +func (s *TerraformSource) Enabled(_ recon.Config) bool { return true } + +// Sweep iterates provider keywords, searches Terraform Registry for matching +// modules, and emits a Finding for each result. +func (s *TerraformSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error { + base := s.BaseURL + if base == "" { + base = "https://registry.terraform.io" + } + client := s.Client + if client == nil { + client = NewClient() + } + + queries := BuildQueries(s.Registry, "terraform") + if len(queries) == 0 { + return nil + } + + for _, q := range queries { + if err := ctx.Err(); err != nil { + return err + } + + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + endpoint := fmt.Sprintf("%s/v1/modules?q=%s&limit=20", + base, url.QueryEscape(q)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("terraform: build req: %w", err) + } + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(ctx, req) + if err != nil { + continue + } + + var parsed terraformSearchResponse + decErr := json.NewDecoder(resp.Body).Decode(&parsed) + _ = resp.Body.Close() + if decErr != nil { + continue + } + + for _, mod := range parsed.Modules { + if err := ctx.Err(); err != nil { + return err + } + + sourceURL := fmt.Sprintf("https://registry.terraform.io/modules/%s/%s/%s", + mod.Namespace, mod.Name, mod.Provider) + if base != "https://registry.terraform.io" { + sourceURL = fmt.Sprintf("%s/modules/%s/%s/%s", + base, mod.Namespace, mod.Name, mod.Provider) + } + + f := recon.Finding{ + ProviderName: "", + Source: sourceURL, + SourceType: "recon:terraform", + Confidence: "low", + DetectedAt: time.Now(), + } + select { + case out <- f: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil +} + +type terraformSearchResponse struct { + Modules []terraformModule `json:"modules"` +} + +type terraformModule struct { + ID string `json:"id"` + Namespace string `json:"namespace"` + Name string `json:"name"` + Provider string `json:"provider"` + Description string `json:"description"` +} diff --git a/pkg/recon/sources/terraform_test.go b/pkg/recon/sources/terraform_test.go new file mode 100644 index 0000000..a3daf6c --- /dev/null +++ b/pkg/recon/sources/terraform_test.go @@ -0,0 +1,190 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +func terraformStubHandler(t *testing.T, calls *int32) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(calls, 1) + if r.URL.Path != "/v1/modules" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + if r.URL.Query().Get("q") == "" { + t.Errorf("missing q param") + } + body := terraformSearchResponse{ + Modules: []terraformModule{ + { + ID: "hashicorp/openai/aws", + Namespace: "hashicorp", + Name: "openai", + Provider: "aws", + }, + { + ID: "community/llm-gateway/azure", + Namespace: "community", + Name: "llm-gateway", + Provider: "azure", + }, + }, + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(body) + } +} + +func TestTerraform_SweepEmitsFindings(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("terraform", 1000, 100) + + var calls int32 + srv := httptest.NewServer(terraformStubHandler(t, &calls)) + defer srv.Close() + + src := &TerraformSource{ + BaseURL: srv.URL, + Registry: reg, + Limiters: lim, + Client: NewClient(), + } + + out := make(chan recon.Finding, 32) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + done := make(chan error, 1) + go func() { done <- src.Sweep(ctx, "", out); close(out) }() + + var findings []recon.Finding + for f := range out { + findings = append(findings, f) + } + if err := <-done; err != nil { + t.Fatalf("Sweep error: %v", err) + } + + // 2 keywords * 2 modules = 4 findings + if len(findings) != 4 { + t.Fatalf("expected 4 findings, got %d", len(findings)) + } + for _, f := range findings { + if f.SourceType != "recon:terraform" { + t.Errorf("SourceType=%q want recon:terraform", f.SourceType) + } + } + if got := atomic.LoadInt32(&calls); got != 2 { + t.Errorf("expected 2 server calls, got %d", got) + } +} + +func TestTerraform_ModuleURLConstruction(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("terraform", 1000, 100) + + var calls int32 + srv := httptest.NewServer(terraformStubHandler(t, &calls)) + defer srv.Close() + + src := &TerraformSource{ + BaseURL: srv.URL, + Registry: reg, + Limiters: lim, + Client: NewClient(), + } + + out := make(chan recon.Finding, 32) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + done := make(chan error, 1) + go func() { done <- src.Sweep(ctx, "", out); close(out) }() + + var findings []recon.Finding + for f := range out { + findings = append(findings, f) + } + if err := <-done; err != nil { + t.Fatalf("Sweep error: %v", err) + } + + // Verify URL contains namespace/name/provider structure. + hasHashicorp := false + hasCommunity := false + for _, f := range findings { + if contains(f.Source, "/modules/hashicorp/openai/aws") { + hasHashicorp = true + } + if contains(f.Source, "/modules/community/llm-gateway/azure") { + hasCommunity = true + } + } + if !hasHashicorp { + t.Error("expected finding with hashicorp/openai/aws module URL") + } + if !hasCommunity { + t.Error("expected finding with community/llm-gateway/azure module URL") + } +} + +func TestTerraform_EnabledAlwaysTrue(t *testing.T) { + s := &TerraformSource{} + if !s.Enabled(recon.Config{}) { + t.Fatal("expected Enabled=true") + } +} + +func TestTerraform_NameAndRate(t *testing.T) { + s := &TerraformSource{} + if s.Name() != "terraform" { + t.Errorf("unexpected name: %s", s.Name()) + } + if s.Burst() != 2 { + t.Errorf("burst: %d", s.Burst()) + } + if s.RespectsRobots() { + t.Error("expected RespectsRobots=false") + } +} + +func TestTerraform_CtxCancelled(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("terraform", 1000, 100) + + src := &TerraformSource{ + BaseURL: "http://127.0.0.1:1", + Registry: reg, + Limiters: lim, + Client: NewClient(), + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + out := make(chan recon.Finding, 1) + err := src.Sweep(ctx, "", out) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } +} + +func TestTerraform_NilRegistryNoError(t *testing.T) { + src := &TerraformSource{Client: NewClient()} + out := make(chan recon.Finding, 1) + if err := src.Sweep(context.Background(), "", out); err != nil { + t.Fatalf("expected nil, got %v", err) + } +}