From f5d8470aab94b20cb4bcacd740bb2b5fbde7be07 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Mon, 6 Apr 2026 12:23:06 +0300 Subject: [PATCH] feat(12-01): implement Shodan, Censys, ZoomEye recon sources - ShodanSource searches /shodan/host/search with API key auth - CensysSource POSTs to /v2/hosts/search with Basic Auth - ZoomEyeSource searches /host/search with API-KEY header - All use shared Client for retry/backoff, LimiterRegistry for rate limiting --- pkg/recon/sources/censys.go | 170 +++++++++++++++++++++++++++++++++++ pkg/recon/sources/shodan.go | 153 +++++++++++++++++++++++++++++++ pkg/recon/sources/zoomeye.go | 157 ++++++++++++++++++++++++++++++++ 3 files changed, 480 insertions(+) create mode 100644 pkg/recon/sources/censys.go create mode 100644 pkg/recon/sources/shodan.go create mode 100644 pkg/recon/sources/zoomeye.go diff --git a/pkg/recon/sources/censys.go b/pkg/recon/sources/censys.go new file mode 100644 index 0000000..7915c9b --- /dev/null +++ b/pkg/recon/sources/censys.go @@ -0,0 +1,170 @@ +package sources + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// CensysSource implements recon.ReconSource against the Censys v2 /hosts/search +// API. It iterates provider keyword queries and emits a Finding for every hit +// returned (exposed services leaking API keys). +// +// Missing API credentials disable the source without error. +type CensysSource struct { + APIId string + APISecret string + BaseURL string + Registry *providers.Registry + Limiters *recon.LimiterRegistry + client *Client +} + +// Compile-time assertion. +var _ recon.ReconSource = (*CensysSource)(nil) + +// NewCensysSource constructs a CensysSource with the shared retry client. +func NewCensysSource(apiId, apiSecret string, reg *providers.Registry, lim *recon.LimiterRegistry) *CensysSource { + return &CensysSource{ + APIId: apiId, + APISecret: apiSecret, + BaseURL: "https://search.censys.io/api", + Registry: reg, + Limiters: lim, + client: NewClient(), + } +} + +func (s *CensysSource) Name() string { return "censys" } +func (s *CensysSource) RateLimit() rate.Limit { return rate.Every(2500 * time.Millisecond) } +func (s *CensysSource) Burst() int { return 1 } +func (s *CensysSource) RespectsRobots() bool { return false } + +// Enabled returns true only when both APIId and APISecret are configured. +func (s *CensysSource) Enabled(_ recon.Config) bool { + return s.APIId != "" && s.APISecret != "" +} + +// Sweep issues one POST /v2/hosts/search request per provider keyword and +// emits a Finding for every hit returned. +func (s *CensysSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error { + if s.APIId == "" || s.APISecret == "" { + return nil + } + base := s.BaseURL + if base == "" { + base = "https://search.censys.io/api" + } + + queries := BuildQueries(s.Registry, "censys") + kwIndex := censysKeywordIndex(s.Registry) + + for _, q := range queries { + if err := ctx.Err(); err != nil { + return err + } + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + payload, _ := json.Marshal(map[string]any{ + "q": q, + "per_page": 25, + }) + + endpoint := fmt.Sprintf("%s/v2/hosts/search", base) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload)) + if err != nil { + return fmt.Errorf("censys: build request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "keyhunter-recon") + req.SetBasicAuth(s.APIId, s.APISecret) + + resp, err := s.client.Do(ctx, req) + if err != nil { + if errors.Is(err, ErrUnauthorized) { + return err + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + continue + } + + var parsed censysSearchResponse + decErr := json.NewDecoder(resp.Body).Decode(&parsed) + _ = resp.Body.Close() + if decErr != nil { + continue + } + + provName := kwIndex[strings.ToLower(q)] + for _, hit := range parsed.Result.Hits { + f := recon.Finding{ + ProviderName: provName, + Confidence: "low", + Source: fmt.Sprintf("censys://%s", hit.IP), + SourceType: "recon:censys", + DetectedAt: time.Now(), + } + select { + case out <- f: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil +} + +type censysSearchResponse struct { + Result censysResult `json:"result"` +} + +type censysResult struct { + Hits []censysHit `json:"hits"` +} + +type censysHit struct { + IP string `json:"ip"` + Services []censysService `json:"services"` +} + +type censysService struct { + Port int `json:"port"` + ServiceName string `json:"service_name"` +} + +// censysKeywordIndex maps lowercased keywords to provider names. +func censysKeywordIndex(reg *providers.Registry) map[string]string { + m := make(map[string]string) + if reg == nil { + return m + } + for _, p := range reg.List() { + for _, k := range p.Keywords { + kl := strings.ToLower(strings.TrimSpace(k)) + if kl == "" { + continue + } + if _, exists := m[kl]; !exists { + m[kl] = p.Name + } + } + } + return m +} diff --git a/pkg/recon/sources/shodan.go b/pkg/recon/sources/shodan.go new file mode 100644 index 0000000..1de2073 --- /dev/null +++ b/pkg/recon/sources/shodan.go @@ -0,0 +1,153 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// ShodanSource implements recon.ReconSource against the Shodan /shodan/host/search +// REST API. It iterates provider keyword queries and emits a Finding for every +// match returned (exposed LLM endpoints, API keys in banners, etc.). +// +// A missing API key disables the source -- Sweep returns nil and Enabled reports +// false. +type ShodanSource struct { + APIKey string + BaseURL string + Registry *providers.Registry + Limiters *recon.LimiterRegistry + client *Client +} + +// Compile-time assertion. +var _ recon.ReconSource = (*ShodanSource)(nil) + +// NewShodanSource constructs a ShodanSource with the shared retry client. +func NewShodanSource(apiKey string, reg *providers.Registry, lim *recon.LimiterRegistry) *ShodanSource { + return &ShodanSource{ + APIKey: apiKey, + BaseURL: "https://api.shodan.io", + Registry: reg, + Limiters: lim, + client: NewClient(), + } +} + +func (s *ShodanSource) Name() string { return "shodan" } +func (s *ShodanSource) RateLimit() rate.Limit { return rate.Every(1 * time.Second) } +func (s *ShodanSource) Burst() int { return 1 } +func (s *ShodanSource) RespectsRobots() bool { return false } + +// Enabled returns true only when APIKey is configured. +func (s *ShodanSource) Enabled(_ recon.Config) bool { return s.APIKey != "" } + +// Sweep issues one /shodan/host/search request per provider keyword and emits +// a Finding for every match returned. +func (s *ShodanSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error { + if s.APIKey == "" { + return nil + } + base := s.BaseURL + if base == "" { + base = "https://api.shodan.io" + } + + queries := BuildQueries(s.Registry, "shodan") + kwIndex := shodanKeywordIndex(s.Registry) + + for _, q := range queries { + if err := ctx.Err(); err != nil { + return err + } + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + endpoint := fmt.Sprintf("%s/shodan/host/search?key=%s&query=%s", + base, url.QueryEscape(s.APIKey), url.QueryEscape(q)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("shodan: build request: %w", err) + } + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "keyhunter-recon") + + resp, err := s.client.Do(ctx, req) + if err != nil { + if errors.Is(err, ErrUnauthorized) { + return err + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + continue + } + + var parsed shodanSearchResponse + decErr := json.NewDecoder(resp.Body).Decode(&parsed) + _ = resp.Body.Close() + if decErr != nil { + continue + } + + provName := kwIndex[strings.ToLower(q)] + for _, m := range parsed.Matches { + f := recon.Finding{ + ProviderName: provName, + Confidence: "low", + Source: fmt.Sprintf("shodan://%s:%d", m.IPStr, m.Port), + SourceType: "recon:shodan", + DetectedAt: time.Now(), + } + select { + case out <- f: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil +} + +type shodanSearchResponse struct { + Matches []shodanMatch `json:"matches"` +} + +type shodanMatch struct { + IPStr string `json:"ip_str"` + Port int `json:"port"` + Data string `json:"data"` +} + +// shodanKeywordIndex maps lowercased keywords to provider names. +func shodanKeywordIndex(reg *providers.Registry) map[string]string { + m := make(map[string]string) + if reg == nil { + return m + } + for _, p := range reg.List() { + for _, k := range p.Keywords { + kl := strings.ToLower(strings.TrimSpace(k)) + if kl == "" { + continue + } + if _, exists := m[kl]; !exists { + m[kl] = p.Name + } + } + } + return m +} diff --git a/pkg/recon/sources/zoomeye.go b/pkg/recon/sources/zoomeye.go new file mode 100644 index 0000000..abdbd40 --- /dev/null +++ b/pkg/recon/sources/zoomeye.go @@ -0,0 +1,157 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// ZoomEyeSource implements recon.ReconSource against the ZoomEye /host/search +// API. It iterates provider keyword queries and emits a Finding for every match +// returned (device/service key exposure). +// +// A missing API key disables the source without error. +type ZoomEyeSource struct { + APIKey string + BaseURL string + Registry *providers.Registry + Limiters *recon.LimiterRegistry + client *Client +} + +// Compile-time assertion. +var _ recon.ReconSource = (*ZoomEyeSource)(nil) + +// NewZoomEyeSource constructs a ZoomEyeSource with the shared retry client. +func NewZoomEyeSource(apiKey string, reg *providers.Registry, lim *recon.LimiterRegistry) *ZoomEyeSource { + return &ZoomEyeSource{ + APIKey: apiKey, + BaseURL: "https://api.zoomeye.org", + Registry: reg, + Limiters: lim, + client: NewClient(), + } +} + +func (s *ZoomEyeSource) Name() string { return "zoomeye" } +func (s *ZoomEyeSource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) } +func (s *ZoomEyeSource) Burst() int { return 1 } +func (s *ZoomEyeSource) RespectsRobots() bool { return false } + +// Enabled returns true only when APIKey is configured. +func (s *ZoomEyeSource) Enabled(_ recon.Config) bool { return s.APIKey != "" } + +// Sweep issues one /host/search request per provider keyword and emits a +// Finding for every match returned. +func (s *ZoomEyeSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error { + if s.APIKey == "" { + return nil + } + base := s.BaseURL + if base == "" { + base = "https://api.zoomeye.org" + } + + queries := BuildQueries(s.Registry, "zoomeye") + kwIndex := zoomeyeKeywordIndex(s.Registry) + + for _, q := range queries { + if err := ctx.Err(); err != nil { + return err + } + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + endpoint := fmt.Sprintf("%s/host/search?query=%s&page=1", + base, url.QueryEscape(q)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("zoomeye: build request: %w", err) + } + req.Header.Set("API-KEY", s.APIKey) + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "keyhunter-recon") + + resp, err := s.client.Do(ctx, req) + if err != nil { + if errors.Is(err, ErrUnauthorized) { + return err + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + continue + } + + var parsed zoomeyeSearchResponse + decErr := json.NewDecoder(resp.Body).Decode(&parsed) + _ = resp.Body.Close() + if decErr != nil { + continue + } + + provName := kwIndex[strings.ToLower(q)] + for _, m := range parsed.Matches { + f := recon.Finding{ + ProviderName: provName, + Confidence: "low", + Source: fmt.Sprintf("zoomeye://%s:%d", m.IP, m.PortInfo.Port), + SourceType: "recon:zoomeye", + DetectedAt: time.Now(), + } + select { + case out <- f: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil +} + +type zoomeyeSearchResponse struct { + Matches []zoomeyeMatch `json:"matches"` +} + +type zoomeyeMatch struct { + IP string `json:"ip"` + PortInfo zoomeyePortInfo `json:"portinfo"` + Banner string `json:"banner"` +} + +type zoomeyePortInfo struct { + Port int `json:"port"` +} + +// zoomeyeKeywordIndex maps lowercased keywords to provider names. +func zoomeyeKeywordIndex(reg *providers.Registry) map[string]string { + m := make(map[string]string) + if reg == nil { + return m + } + for _, p := range reg.List() { + for _, k := range p.Keywords { + kl := strings.ToLower(strings.TrimSpace(k)) + if kl == "" { + continue + } + if _, exists := m[kl]; !exists { + m[kl] = p.Name + } + } + } + return m +}