From 270bbbfb4902c230775e3bddba4b9cc027f5dac1 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Mon, 6 Apr 2026 12:24:04 +0300 Subject: [PATCH] feat(12-02): implement FOFA, Netlas, BinaryEdge recon sources - FOFASource searches FOFA API with base64-encoded queries (email+key auth) - NetlasSource searches Netlas API with X-API-Key header auth - BinaryEdgeSource searches BinaryEdge API with X-Key header auth - All three implement recon.ReconSource with shared Client retry/backoff Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/recon/sources/binaryedge.go | 147 ++++++++++++++++++++++++++++++++ pkg/recon/sources/fofa.go | 144 +++++++++++++++++++++++++++++++ pkg/recon/sources/netlas.go | 147 ++++++++++++++++++++++++++++++++ 3 files changed, 438 insertions(+) create mode 100644 pkg/recon/sources/binaryedge.go create mode 100644 pkg/recon/sources/fofa.go create mode 100644 pkg/recon/sources/netlas.go diff --git a/pkg/recon/sources/binaryedge.go b/pkg/recon/sources/binaryedge.go new file mode 100644 index 0000000..5b9a3c5 --- /dev/null +++ b/pkg/recon/sources/binaryedge.go @@ -0,0 +1,147 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// BinaryEdgeSource implements recon.ReconSource against the BinaryEdge +// internet data API. It iterates provider keyword queries and emits a Finding +// per result event. +// +// A missing API key disables the source without error. +type BinaryEdgeSource struct { + APIKey string + BaseURL string + Registry *providers.Registry + Limiters *recon.LimiterRegistry + client *Client +} + +// Compile-time assertion. +var _ recon.ReconSource = (*BinaryEdgeSource)(nil) + +func (s *BinaryEdgeSource) Name() string { return "binaryedge" } +func (s *BinaryEdgeSource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) } +func (s *BinaryEdgeSource) Burst() int { return 1 } +func (s *BinaryEdgeSource) RespectsRobots() bool { return false } + +// Enabled returns true only when APIKey is configured. +func (s *BinaryEdgeSource) Enabled(_ recon.Config) bool { return s.APIKey != "" } + +// Sweep issues one BinaryEdge search request per provider keyword and emits +// a Finding for every result event. +func (s *BinaryEdgeSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error { + if s.APIKey == "" { + return nil + } + if s.client == nil { + s.client = NewClient() + } + base := s.BaseURL + if base == "" { + base = "https://api.binaryedge.io" + } + + queries := BuildQueries(s.Registry, "binaryedge") + kwIndex := binaryedgeKeywordIndex(s.Registry) + + for _, q := range queries { + if err := ctx.Err(); err != nil { + return err + } + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + endpoint := fmt.Sprintf("%s/v2/query/search?query=%s&page=1", + base, url.QueryEscape(q)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("binaryedge: build request: %w", err) + } + req.Header.Set("X-Key", s.APIKey) + req.Header.Set("Accept", "application/json") + + resp, err := s.client.Do(ctx, req) + if err != nil { + if errors.Is(err, ErrUnauthorized) { + return err + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + continue + } + + var parsed binaryedgeSearchResponse + decErr := json.NewDecoder(resp.Body).Decode(&parsed) + _ = resp.Body.Close() + if decErr != nil { + continue + } + + provName := kwIndex[strings.ToLower(q)] + for _, ev := range parsed.Events { + f := recon.Finding{ + ProviderName: provName, + Confidence: "low", + Source: fmt.Sprintf("binaryedge://%s:%d", ev.Target.IP, ev.Target.Port), + SourceType: "recon:binaryedge", + DetectedAt: time.Now(), + } + select { + case out <- f: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil +} + +type binaryedgeSearchResponse struct { + Events []binaryedgeEvent `json:"events"` +} + +type binaryedgeEvent struct { + Target binaryedgeTarget `json:"target"` +} + +type binaryedgeTarget struct { + IP string `json:"ip"` + Port int `json:"port"` +} + +// binaryedgeKeywordIndex maps lowercased keywords to provider names. +func binaryedgeKeywordIndex(reg *providers.Registry) map[string]string { + m := make(map[string]string) + if reg == nil { + return m + } + for _, p := range reg.List() { + for _, k := range p.Keywords { + kl := strings.ToLower(strings.TrimSpace(k)) + if kl == "" { + continue + } + if _, exists := m[kl]; !exists { + m[kl] = p.Name + } + } + } + return m +} diff --git a/pkg/recon/sources/fofa.go b/pkg/recon/sources/fofa.go new file mode 100644 index 0000000..2fec8d5 --- /dev/null +++ b/pkg/recon/sources/fofa.go @@ -0,0 +1,144 @@ +package sources + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// FOFASource implements recon.ReconSource against the FOFA internet search +// engine API. It iterates provider keyword queries and emits a Finding per +// result. +// +// A missing Email or API key disables the source without error. +type FOFASource struct { + Email string + APIKey string + BaseURL string + Registry *providers.Registry + Limiters *recon.LimiterRegistry + client *Client +} + +// Compile-time assertion. +var _ recon.ReconSource = (*FOFASource)(nil) + +func (s *FOFASource) Name() string { return "fofa" } +func (s *FOFASource) RateLimit() rate.Limit { return rate.Every(1 * time.Second) } +func (s *FOFASource) Burst() int { return 1 } +func (s *FOFASource) RespectsRobots() bool { return false } + +// Enabled returns true only when both Email and APIKey are configured. +func (s *FOFASource) Enabled(_ recon.Config) bool { return s.Email != "" && s.APIKey != "" } + +// Sweep issues one FOFA search request per provider keyword and emits a +// Finding for every result row. +func (s *FOFASource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error { + if s.Email == "" || s.APIKey == "" { + return nil + } + if s.client == nil { + s.client = NewClient() + } + base := s.BaseURL + if base == "" { + base = "https://fofa.info" + } + + queries := BuildQueries(s.Registry, "fofa") + kwIndex := fofaKeywordIndex(s.Registry) + + for _, q := range queries { + if err := ctx.Err(); err != nil { + return err + } + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + qb64 := base64.StdEncoding.EncodeToString([]byte(q)) + endpoint := fmt.Sprintf("%s/api/v1/search/all?email=%s&key=%s&qbase64=%s&size=100", + base, s.Email, s.APIKey, qb64) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("fofa: build request: %w", err) + } + req.Header.Set("Accept", "application/json") + + resp, err := s.client.Do(ctx, req) + if err != nil { + if errors.Is(err, ErrUnauthorized) { + return err + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + continue + } + + var parsed fofaSearchResponse + decErr := json.NewDecoder(resp.Body).Decode(&parsed) + _ = resp.Body.Close() + if decErr != nil { + continue + } + + provName := kwIndex[strings.ToLower(q)] + for _, row := range parsed.Results { + // Each row is [host, ip, port]. + if len(row) < 3 { + continue + } + f := recon.Finding{ + ProviderName: provName, + Confidence: "low", + Source: fmt.Sprintf("fofa://%s:%s", row[1], row[2]), + SourceType: "recon:fofa", + DetectedAt: time.Now(), + } + select { + case out <- f: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil +} + +type fofaSearchResponse struct { + Results [][]string `json:"results"` + Size int `json:"size"` +} + +// fofaKeywordIndex maps lowercased keywords to provider names. +func fofaKeywordIndex(reg *providers.Registry) map[string]string { + m := make(map[string]string) + if reg == nil { + return m + } + for _, p := range reg.List() { + for _, k := range p.Keywords { + kl := strings.ToLower(strings.TrimSpace(k)) + if kl == "" { + continue + } + if _, exists := m[kl]; !exists { + m[kl] = p.Name + } + } + } + return m +} diff --git a/pkg/recon/sources/netlas.go b/pkg/recon/sources/netlas.go new file mode 100644 index 0000000..017dd1f --- /dev/null +++ b/pkg/recon/sources/netlas.go @@ -0,0 +1,147 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// NetlasSource implements recon.ReconSource against the Netlas internet +// intelligence API. It iterates provider keyword queries and emits a Finding +// per result item. +// +// A missing API key disables the source without error. +type NetlasSource struct { + APIKey string + BaseURL string + Registry *providers.Registry + Limiters *recon.LimiterRegistry + client *Client +} + +// Compile-time assertion. +var _ recon.ReconSource = (*NetlasSource)(nil) + +func (s *NetlasSource) Name() string { return "netlas" } +func (s *NetlasSource) RateLimit() rate.Limit { return rate.Every(1 * time.Second) } +func (s *NetlasSource) Burst() int { return 1 } +func (s *NetlasSource) RespectsRobots() bool { return false } + +// Enabled returns true only when APIKey is configured. +func (s *NetlasSource) Enabled(_ recon.Config) bool { return s.APIKey != "" } + +// Sweep issues one Netlas search request per provider keyword and emits a +// Finding for every result item. +func (s *NetlasSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error { + if s.APIKey == "" { + return nil + } + if s.client == nil { + s.client = NewClient() + } + base := s.BaseURL + if base == "" { + base = "https://app.netlas.io" + } + + queries := BuildQueries(s.Registry, "netlas") + kwIndex := netlasKeywordIndex(s.Registry) + + for _, q := range queries { + if err := ctx.Err(); err != nil { + return err + } + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + endpoint := fmt.Sprintf("%s/api/responses/?q=%s&start=0&indices=", + base, url.QueryEscape(q)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("netlas: build request: %w", err) + } + req.Header.Set("X-API-Key", s.APIKey) + req.Header.Set("Accept", "application/json") + + resp, err := s.client.Do(ctx, req) + if err != nil { + if errors.Is(err, ErrUnauthorized) { + return err + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + continue + } + + var parsed netlasSearchResponse + decErr := json.NewDecoder(resp.Body).Decode(&parsed) + _ = resp.Body.Close() + if decErr != nil { + continue + } + + provName := kwIndex[strings.ToLower(q)] + for _, item := range parsed.Items { + f := recon.Finding{ + ProviderName: provName, + Confidence: "low", + Source: fmt.Sprintf("netlas://%s:%d", item.Data.IP, item.Data.Port), + SourceType: "recon:netlas", + DetectedAt: time.Now(), + } + select { + case out <- f: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil +} + +type netlasSearchResponse struct { + Items []netlasItem `json:"items"` +} + +type netlasItem struct { + Data netlasData `json:"data"` +} + +type netlasData struct { + IP string `json:"ip"` + Port int `json:"port"` +} + +// netlasKeywordIndex maps lowercased keywords to provider names. +func netlasKeywordIndex(reg *providers.Registry) map[string]string { + m := make(map[string]string) + if reg == nil { + return m + } + for _, p := range reg.List() { + for _, k := range p.Keywords { + kl := strings.ToLower(strings.TrimSpace(k)) + if kl == "" { + continue + } + if _, exists := m[kl]; !exists { + m[kl] = p.Name + } + } + } + return m +}