From 270bbbfb4902c230775e3bddba4b9cc027f5dac1 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Mon, 6 Apr 2026 12:24:04 +0300 Subject: [PATCH 1/3] feat(12-02): implement FOFA, Netlas, BinaryEdge recon sources - FOFASource searches FOFA API with base64-encoded queries (email+key auth) - NetlasSource searches Netlas API with X-API-Key header auth - BinaryEdgeSource searches BinaryEdge API with X-Key header auth - All three implement recon.ReconSource with shared Client retry/backoff Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/recon/sources/binaryedge.go | 147 ++++++++++++++++++++++++++++++++ pkg/recon/sources/fofa.go | 144 +++++++++++++++++++++++++++++++ pkg/recon/sources/netlas.go | 147 ++++++++++++++++++++++++++++++++ 3 files changed, 438 insertions(+) create mode 100644 pkg/recon/sources/binaryedge.go create mode 100644 pkg/recon/sources/fofa.go create mode 100644 pkg/recon/sources/netlas.go diff --git a/pkg/recon/sources/binaryedge.go b/pkg/recon/sources/binaryedge.go new file mode 100644 index 0000000..5b9a3c5 --- /dev/null +++ b/pkg/recon/sources/binaryedge.go @@ -0,0 +1,147 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// BinaryEdgeSource implements recon.ReconSource against the BinaryEdge +// internet data API. It iterates provider keyword queries and emits a Finding +// per result event. +// +// A missing API key disables the source without error. +type BinaryEdgeSource struct { + APIKey string + BaseURL string + Registry *providers.Registry + Limiters *recon.LimiterRegistry + client *Client +} + +// Compile-time assertion. +var _ recon.ReconSource = (*BinaryEdgeSource)(nil) + +func (s *BinaryEdgeSource) Name() string { return "binaryedge" } +func (s *BinaryEdgeSource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) } +func (s *BinaryEdgeSource) Burst() int { return 1 } +func (s *BinaryEdgeSource) RespectsRobots() bool { return false } + +// Enabled returns true only when APIKey is configured. +func (s *BinaryEdgeSource) Enabled(_ recon.Config) bool { return s.APIKey != "" } + +// Sweep issues one BinaryEdge search request per provider keyword and emits +// a Finding for every result event. +func (s *BinaryEdgeSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error { + if s.APIKey == "" { + return nil + } + if s.client == nil { + s.client = NewClient() + } + base := s.BaseURL + if base == "" { + base = "https://api.binaryedge.io" + } + + queries := BuildQueries(s.Registry, "binaryedge") + kwIndex := binaryedgeKeywordIndex(s.Registry) + + for _, q := range queries { + if err := ctx.Err(); err != nil { + return err + } + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + endpoint := fmt.Sprintf("%s/v2/query/search?query=%s&page=1", + base, url.QueryEscape(q)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("binaryedge: build request: %w", err) + } + req.Header.Set("X-Key", s.APIKey) + req.Header.Set("Accept", "application/json") + + resp, err := s.client.Do(ctx, req) + if err != nil { + if errors.Is(err, ErrUnauthorized) { + return err + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + continue + } + + var parsed binaryedgeSearchResponse + decErr := json.NewDecoder(resp.Body).Decode(&parsed) + _ = resp.Body.Close() + if decErr != nil { + continue + } + + provName := kwIndex[strings.ToLower(q)] + for _, ev := range parsed.Events { + f := recon.Finding{ + ProviderName: provName, + Confidence: "low", + Source: fmt.Sprintf("binaryedge://%s:%d", ev.Target.IP, ev.Target.Port), + SourceType: "recon:binaryedge", + DetectedAt: time.Now(), + } + select { + case out <- f: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil +} + +type binaryedgeSearchResponse struct { + Events []binaryedgeEvent `json:"events"` +} + +type binaryedgeEvent struct { + Target binaryedgeTarget `json:"target"` +} + +type binaryedgeTarget struct { + IP string `json:"ip"` + Port int `json:"port"` +} + +// binaryedgeKeywordIndex maps lowercased keywords to provider names. +func binaryedgeKeywordIndex(reg *providers.Registry) map[string]string { + m := make(map[string]string) + if reg == nil { + return m + } + for _, p := range reg.List() { + for _, k := range p.Keywords { + kl := strings.ToLower(strings.TrimSpace(k)) + if kl == "" { + continue + } + if _, exists := m[kl]; !exists { + m[kl] = p.Name + } + } + } + return m +} diff --git a/pkg/recon/sources/fofa.go b/pkg/recon/sources/fofa.go new file mode 100644 index 0000000..2fec8d5 --- /dev/null +++ b/pkg/recon/sources/fofa.go @@ -0,0 +1,144 @@ +package sources + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// FOFASource implements recon.ReconSource against the FOFA internet search +// engine API. It iterates provider keyword queries and emits a Finding per +// result. +// +// A missing Email or API key disables the source without error. +type FOFASource struct { + Email string + APIKey string + BaseURL string + Registry *providers.Registry + Limiters *recon.LimiterRegistry + client *Client +} + +// Compile-time assertion. +var _ recon.ReconSource = (*FOFASource)(nil) + +func (s *FOFASource) Name() string { return "fofa" } +func (s *FOFASource) RateLimit() rate.Limit { return rate.Every(1 * time.Second) } +func (s *FOFASource) Burst() int { return 1 } +func (s *FOFASource) RespectsRobots() bool { return false } + +// Enabled returns true only when both Email and APIKey are configured. +func (s *FOFASource) Enabled(_ recon.Config) bool { return s.Email != "" && s.APIKey != "" } + +// Sweep issues one FOFA search request per provider keyword and emits a +// Finding for every result row. +func (s *FOFASource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error { + if s.Email == "" || s.APIKey == "" { + return nil + } + if s.client == nil { + s.client = NewClient() + } + base := s.BaseURL + if base == "" { + base = "https://fofa.info" + } + + queries := BuildQueries(s.Registry, "fofa") + kwIndex := fofaKeywordIndex(s.Registry) + + for _, q := range queries { + if err := ctx.Err(); err != nil { + return err + } + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + qb64 := base64.StdEncoding.EncodeToString([]byte(q)) + endpoint := fmt.Sprintf("%s/api/v1/search/all?email=%s&key=%s&qbase64=%s&size=100", + base, s.Email, s.APIKey, qb64) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("fofa: build request: %w", err) + } + req.Header.Set("Accept", "application/json") + + resp, err := s.client.Do(ctx, req) + if err != nil { + if errors.Is(err, ErrUnauthorized) { + return err + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + continue + } + + var parsed fofaSearchResponse + decErr := json.NewDecoder(resp.Body).Decode(&parsed) + _ = resp.Body.Close() + if decErr != nil { + continue + } + + provName := kwIndex[strings.ToLower(q)] + for _, row := range parsed.Results { + // Each row is [host, ip, port]. + if len(row) < 3 { + continue + } + f := recon.Finding{ + ProviderName: provName, + Confidence: "low", + Source: fmt.Sprintf("fofa://%s:%s", row[1], row[2]), + SourceType: "recon:fofa", + DetectedAt: time.Now(), + } + select { + case out <- f: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil +} + +type fofaSearchResponse struct { + Results [][]string `json:"results"` + Size int `json:"size"` +} + +// fofaKeywordIndex maps lowercased keywords to provider names. +func fofaKeywordIndex(reg *providers.Registry) map[string]string { + m := make(map[string]string) + if reg == nil { + return m + } + for _, p := range reg.List() { + for _, k := range p.Keywords { + kl := strings.ToLower(strings.TrimSpace(k)) + if kl == "" { + continue + } + if _, exists := m[kl]; !exists { + m[kl] = p.Name + } + } + } + return m +} diff --git a/pkg/recon/sources/netlas.go b/pkg/recon/sources/netlas.go new file mode 100644 index 0000000..017dd1f --- /dev/null +++ b/pkg/recon/sources/netlas.go @@ -0,0 +1,147 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/time/rate" + + "github.com/salvacybersec/keyhunter/pkg/providers" + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +// NetlasSource implements recon.ReconSource against the Netlas internet +// intelligence API. It iterates provider keyword queries and emits a Finding +// per result item. +// +// A missing API key disables the source without error. +type NetlasSource struct { + APIKey string + BaseURL string + Registry *providers.Registry + Limiters *recon.LimiterRegistry + client *Client +} + +// Compile-time assertion. +var _ recon.ReconSource = (*NetlasSource)(nil) + +func (s *NetlasSource) Name() string { return "netlas" } +func (s *NetlasSource) RateLimit() rate.Limit { return rate.Every(1 * time.Second) } +func (s *NetlasSource) Burst() int { return 1 } +func (s *NetlasSource) RespectsRobots() bool { return false } + +// Enabled returns true only when APIKey is configured. +func (s *NetlasSource) Enabled(_ recon.Config) bool { return s.APIKey != "" } + +// Sweep issues one Netlas search request per provider keyword and emits a +// Finding for every result item. +func (s *NetlasSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error { + if s.APIKey == "" { + return nil + } + if s.client == nil { + s.client = NewClient() + } + base := s.BaseURL + if base == "" { + base = "https://app.netlas.io" + } + + queries := BuildQueries(s.Registry, "netlas") + kwIndex := netlasKeywordIndex(s.Registry) + + for _, q := range queries { + if err := ctx.Err(); err != nil { + return err + } + if s.Limiters != nil { + if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil { + return err + } + } + + endpoint := fmt.Sprintf("%s/api/responses/?q=%s&start=0&indices=", + base, url.QueryEscape(q)) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("netlas: build request: %w", err) + } + req.Header.Set("X-API-Key", s.APIKey) + req.Header.Set("Accept", "application/json") + + resp, err := s.client.Do(ctx, req) + if err != nil { + if errors.Is(err, ErrUnauthorized) { + return err + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + continue + } + + var parsed netlasSearchResponse + decErr := json.NewDecoder(resp.Body).Decode(&parsed) + _ = resp.Body.Close() + if decErr != nil { + continue + } + + provName := kwIndex[strings.ToLower(q)] + for _, item := range parsed.Items { + f := recon.Finding{ + ProviderName: provName, + Confidence: "low", + Source: fmt.Sprintf("netlas://%s:%d", item.Data.IP, item.Data.Port), + SourceType: "recon:netlas", + DetectedAt: time.Now(), + } + select { + case out <- f: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return nil +} + +type netlasSearchResponse struct { + Items []netlasItem `json:"items"` +} + +type netlasItem struct { + Data netlasData `json:"data"` +} + +type netlasData struct { + IP string `json:"ip"` + Port int `json:"port"` +} + +// netlasKeywordIndex maps lowercased keywords to provider names. +func netlasKeywordIndex(reg *providers.Registry) map[string]string { + m := make(map[string]string) + if reg == nil { + return m + } + for _, p := range reg.List() { + for _, k := range p.Keywords { + kl := strings.ToLower(strings.TrimSpace(k)) + if kl == "" { + continue + } + if _, exists := m[kl]; !exists { + m[kl] = p.Name + } + } + } + return m +} From d6c35f4f14931a7bc572ed33df3544d69e016109 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Mon, 6 Apr 2026 12:24:11 +0300 Subject: [PATCH 2/3] test(12-02): add httptest tests for FOFA, Netlas, BinaryEdge sources - FOFA: mock JSON with 2 results, credential validation, context cancellation - Netlas: mock JSON with 2 items, X-API-Key header check, context cancellation - BinaryEdge: mock JSON with 2 events, X-Key header check, context cancellation - All verify correct finding count, source type, and disabled state Co-Authored-By: Claude Opus 4.6 (1M context) --- pkg/recon/sources/binaryedge_test.go | 117 ++++++++++++++++++++++++ pkg/recon/sources/fofa_test.go | 130 +++++++++++++++++++++++++++ pkg/recon/sources/netlas_test.go | 117 ++++++++++++++++++++++++ 3 files changed, 364 insertions(+) create mode 100644 pkg/recon/sources/binaryedge_test.go create mode 100644 pkg/recon/sources/fofa_test.go create mode 100644 pkg/recon/sources/netlas_test.go diff --git a/pkg/recon/sources/binaryedge_test.go b/pkg/recon/sources/binaryedge_test.go new file mode 100644 index 0000000..e003a01 --- /dev/null +++ b/pkg/recon/sources/binaryedge_test.go @@ -0,0 +1,117 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +func binaryedgeStubHandler(t *testing.T, calls *int32) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(calls, 1) + if !strings.HasPrefix(r.URL.Path, "/v2/query/search") { + t.Errorf("unexpected path: %s", r.URL.Path) + } + if got := r.Header.Get("X-Key"); got != "testkey" { + t.Errorf("missing X-Key header: %q", got) + } + body := binaryedgeSearchResponse{ + Events: []binaryedgeEvent{ + {Target: binaryedgeTarget{IP: "192.168.1.1", Port: 80}}, + {Target: binaryedgeTarget{IP: "192.168.1.2", Port: 443}}, + }, + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(body) + } +} + +func TestBinaryEdgeSource_EnabledRequiresAPIKey(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + + s := &BinaryEdgeSource{APIKey: "", Registry: reg, Limiters: lim} + if s.Enabled(recon.Config{}) { + t.Error("expected Enabled=false with empty key") + } + s = &BinaryEdgeSource{APIKey: "key", Registry: reg, Limiters: lim} + if !s.Enabled(recon.Config{}) { + t.Error("expected Enabled=true with key") + } +} + +func TestBinaryEdgeSource_SweepEmitsFindings(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("binaryedge", 1000, 100) + + var calls int32 + srv := httptest.NewServer(binaryedgeStubHandler(t, &calls)) + defer srv.Close() + + s := &BinaryEdgeSource{ + APIKey: "testkey", + BaseURL: srv.URL, + Registry: reg, + Limiters: lim, + } + + out := make(chan recon.Finding, 32) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + done := make(chan error, 1) + go func() { done <- s.Sweep(ctx, "", out); close(out) }() + + var findings []recon.Finding + for f := range out { + findings = append(findings, f) + } + if err := <-done; err != nil { + t.Fatalf("Sweep error: %v", err) + } + + // 2 keywords * 2 events = 4 findings + if len(findings) != 4 { + t.Fatalf("expected 4 findings, got %d", len(findings)) + } + for _, f := range findings { + if f.SourceType != "recon:binaryedge" { + t.Errorf("SourceType=%q want recon:binaryedge", f.SourceType) + } + } + if got := atomic.LoadInt32(&calls); got != 2 { + t.Errorf("expected 2 API calls, got %d", got) + } +} + +func TestBinaryEdgeSource_CtxCancelled(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("binaryedge", 1000, 100) + + s := &BinaryEdgeSource{ + APIKey: "key", + BaseURL: "http://127.0.0.1:1", + Registry: reg, + Limiters: lim, + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + out := make(chan recon.Finding, 1) + err := s.Sweep(ctx, "", out) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } +} diff --git a/pkg/recon/sources/fofa_test.go b/pkg/recon/sources/fofa_test.go new file mode 100644 index 0000000..e17497d --- /dev/null +++ b/pkg/recon/sources/fofa_test.go @@ -0,0 +1,130 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +func fofaStubHandler(t *testing.T, calls *int32) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(calls, 1) + if r.URL.Path != "/api/v1/search/all" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + if got := r.URL.Query().Get("email"); got != "test@example.com" { + t.Errorf("missing email param: %q", got) + } + if got := r.URL.Query().Get("key"); got != "testkey" { + t.Errorf("missing key param: %q", got) + } + body := fofaSearchResponse{ + Results: [][]string{ + {"example.com", "1.2.3.4", "443"}, + {"test.org", "5.6.7.8", "8080"}, + }, + Size: 2, + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(body) + } +} + +func TestFOFASource_EnabledRequiresCredentials(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + + s := &FOFASource{Email: "", APIKey: "", Registry: reg, Limiters: lim} + if s.Enabled(recon.Config{}) { + t.Error("expected Enabled=false with empty credentials") + } + s = &FOFASource{Email: "a@b.com", APIKey: "", Registry: reg, Limiters: lim} + if s.Enabled(recon.Config{}) { + t.Error("expected Enabled=false with empty APIKey") + } + s = &FOFASource{Email: "", APIKey: "key", Registry: reg, Limiters: lim} + if s.Enabled(recon.Config{}) { + t.Error("expected Enabled=false with empty Email") + } + s = &FOFASource{Email: "a@b.com", APIKey: "key", Registry: reg, Limiters: lim} + if !s.Enabled(recon.Config{}) { + t.Error("expected Enabled=true with both credentials") + } +} + +func TestFOFASource_SweepEmitsFindings(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("fofa", 1000, 100) + + var calls int32 + srv := httptest.NewServer(fofaStubHandler(t, &calls)) + defer srv.Close() + + s := &FOFASource{ + Email: "test@example.com", + APIKey: "testkey", + BaseURL: srv.URL, + Registry: reg, + Limiters: lim, + } + + out := make(chan recon.Finding, 32) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + done := make(chan error, 1) + go func() { done <- s.Sweep(ctx, "", out); close(out) }() + + var findings []recon.Finding + for f := range out { + findings = append(findings, f) + } + if err := <-done; err != nil { + t.Fatalf("Sweep error: %v", err) + } + + // 2 keywords * 2 results = 4 findings + if len(findings) != 4 { + t.Fatalf("expected 4 findings, got %d", len(findings)) + } + for _, f := range findings { + if f.SourceType != "recon:fofa" { + t.Errorf("SourceType=%q want recon:fofa", f.SourceType) + } + } + if got := atomic.LoadInt32(&calls); got != 2 { + t.Errorf("expected 2 API calls, got %d", got) + } +} + +func TestFOFASource_CtxCancelled(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("fofa", 1000, 100) + + s := &FOFASource{ + Email: "a@b.com", + APIKey: "key", + BaseURL: "http://127.0.0.1:1", + Registry: reg, + Limiters: lim, + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + out := make(chan recon.Finding, 1) + err := s.Sweep(ctx, "", out) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } +} diff --git a/pkg/recon/sources/netlas_test.go b/pkg/recon/sources/netlas_test.go new file mode 100644 index 0000000..ddc337a --- /dev/null +++ b/pkg/recon/sources/netlas_test.go @@ -0,0 +1,117 @@ +package sources + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/salvacybersec/keyhunter/pkg/recon" +) + +func netlasStubHandler(t *testing.T, calls *int32) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(calls, 1) + if !strings.HasPrefix(r.URL.Path, "/api/responses/") { + t.Errorf("unexpected path: %s", r.URL.Path) + } + if got := r.Header.Get("X-API-Key"); got != "testkey" { + t.Errorf("missing X-API-Key header: %q", got) + } + body := netlasSearchResponse{ + Items: []netlasItem{ + {Data: netlasData{IP: "10.0.0.1", Port: 443}}, + {Data: netlasData{IP: "10.0.0.2", Port: 8443}}, + }, + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(body) + } +} + +func TestNetlasSource_EnabledRequiresAPIKey(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + + s := &NetlasSource{APIKey: "", Registry: reg, Limiters: lim} + if s.Enabled(recon.Config{}) { + t.Error("expected Enabled=false with empty key") + } + s = &NetlasSource{APIKey: "key", Registry: reg, Limiters: lim} + if !s.Enabled(recon.Config{}) { + t.Error("expected Enabled=true with key") + } +} + +func TestNetlasSource_SweepEmitsFindings(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("netlas", 1000, 100) + + var calls int32 + srv := httptest.NewServer(netlasStubHandler(t, &calls)) + defer srv.Close() + + s := &NetlasSource{ + APIKey: "testkey", + BaseURL: srv.URL, + Registry: reg, + Limiters: lim, + } + + out := make(chan recon.Finding, 32) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + done := make(chan error, 1) + go func() { done <- s.Sweep(ctx, "", out); close(out) }() + + var findings []recon.Finding + for f := range out { + findings = append(findings, f) + } + if err := <-done; err != nil { + t.Fatalf("Sweep error: %v", err) + } + + // 2 keywords * 2 items = 4 findings + if len(findings) != 4 { + t.Fatalf("expected 4 findings, got %d", len(findings)) + } + for _, f := range findings { + if f.SourceType != "recon:netlas" { + t.Errorf("SourceType=%q want recon:netlas", f.SourceType) + } + } + if got := atomic.LoadInt32(&calls); got != 2 { + t.Errorf("expected 2 API calls, got %d", got) + } +} + +func TestNetlasSource_CtxCancelled(t *testing.T) { + reg := syntheticRegistry() + lim := recon.NewLimiterRegistry() + _ = lim.For("netlas", 1000, 100) + + s := &NetlasSource{ + APIKey: "key", + BaseURL: "http://127.0.0.1:1", + Registry: reg, + Limiters: lim, + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + out := make(chan recon.Finding, 1) + err := s.Sweep(ctx, "", out) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got %v", err) + } +} From 6ab411cda230324bc12c5b65ecec5921b17aaa1a Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Mon, 6 Apr 2026 12:25:06 +0300 Subject: [PATCH 3/3] docs(12-02): complete FOFA, Netlas, BinaryEdge plan Co-Authored-By: Claude Opus 4.6 (1M context) --- .planning/REQUIREMENTS.md | 6 +- .planning/ROADMAP.md | 4 +- .planning/STATE.md | 16 +-- .../12-02-SUMMARY.md | 103 ++++++++++++++++++ 4 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 .planning/phases/12-osint_iot_cloud_storage/12-02-SUMMARY.md diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md index ff5647d..39fc0c5 100644 --- a/.planning/REQUIREMENTS.md +++ b/.planning/REQUIREMENTS.md @@ -96,9 +96,9 @@ Requirements for initial release. Each maps to roadmap phases. - [ ] **RECON-IOT-01**: Shodan API search and dorking - [ ] **RECON-IOT-02**: Censys API search - [ ] **RECON-IOT-03**: ZoomEye API search -- [ ] **RECON-IOT-04**: FOFA API search -- [ ] **RECON-IOT-05**: Netlas API search -- [ ] **RECON-IOT-06**: BinaryEdge API search +- [x] **RECON-IOT-04**: FOFA API search +- [x] **RECON-IOT-05**: Netlas API search +- [x] **RECON-IOT-06**: BinaryEdge API search ### OSINT/Recon — Code Hosting & Snippets diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 99468e1..61c517e 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -256,7 +256,7 @@ Plans: Plans: - [ ] 12-01-PLAN.md — ShodanSource + CensysSource + ZoomEyeSource (RECON-IOT-01, RECON-IOT-02, RECON-IOT-03) -- [ ] 12-02-PLAN.md — FOFASource + NetlasSource + BinaryEdgeSource (RECON-IOT-04, RECON-IOT-05, RECON-IOT-06) +- [x] 12-02-PLAN.md — FOFASource + NetlasSource + BinaryEdgeSource (RECON-IOT-04, RECON-IOT-05, RECON-IOT-06) - [ ] 12-03-PLAN.md — S3Scanner + GCSScanner + AzureBlobScanner + DOSpacesScanner (RECON-CLOUD-01, RECON-CLOUD-02, RECON-CLOUD-03, RECON-CLOUD-04) - [ ] 12-04-PLAN.md — RegisterAll wiring + cmd/recon.go credentials + integration test (all Phase 12 reqs) @@ -349,7 +349,7 @@ Phases execute in numeric order: 1 → 2 → 3 → ... → 18 | 9. OSINT Infrastructure | 2/6 | In Progress| | | 10. OSINT Code Hosting | 9/9 | Complete | 2026-04-06 | | 11. OSINT Search & Paste | 3/3 | Complete | 2026-04-06 | -| 12. OSINT IoT & Cloud Storage | 0/? | Not started | - | +| 12. OSINT IoT & Cloud Storage | 1/4 | In Progress| | | 13. OSINT Package Registries & Container/IaC | 0/? | Not started | - | | 14. OSINT CI/CD Logs, Web Archives & Frontend Leaks | 0/? | Not started | - | | 15. OSINT Forums, Collaboration & Log Aggregators | 0/? | Not started | - | diff --git a/.planning/STATE.md b/.planning/STATE.md index fe584be..0066fcd 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -3,14 +3,14 @@ gsd_state_version: 1.0 milestone: v1.0 milestone_name: milestone status: completed -stopped_at: Completed 11-03-PLAN.md -last_updated: "2026-04-06T09:09:48.100Z" +stopped_at: Completed 12-02-PLAN.md +last_updated: "2026-04-06T09:24:57.655Z" last_activity: 2026-04-06 progress: total_phases: 18 - completed_phases: 11 - total_plans: 65 - completed_plans: 66 + completed_phases: 10 + total_plans: 64 + completed_plans: 67 percent: 20 --- @@ -91,6 +91,7 @@ Progress: [██░░░░░░░░] 20% | Phase 10 P09 | 12min | 2 tasks | 5 files | | Phase 11 P03 | 6min | 2 tasks | 4 files | | Phase 11 P01 | 3min | 2 tasks | 11 files | +| Phase 12 P02 | 2min | 2 tasks | 6 files | ## Accumulated Context @@ -131,6 +132,7 @@ Recent decisions affecting current work: - [Phase 11]: RegisterAll extended to 18 sources (10 Phase 10 + 8 Phase 11); paste sources use BaseURL prefix in integration test to avoid /search path collision - [Phase 11]: Integration test uses injected test platforms for PasteSites (same pattern as SandboxesSource) - [Phase 11]: All five search sources use dork query format to focus on paste/code hosting leak sites +- [Phase 12]: FOFA uses base64-encoded qbase64 param; Netlas uses X-API-Key header; BinaryEdge uses X-Key header ### Pending Todos @@ -145,6 +147,6 @@ None yet. ## Session Continuity -Last session: 2026-04-06T09:07:51.980Z -Stopped at: Completed 11-03-PLAN.md +Last session: 2026-04-06T09:24:57.651Z +Stopped at: Completed 12-02-PLAN.md Resume file: None diff --git a/.planning/phases/12-osint_iot_cloud_storage/12-02-SUMMARY.md b/.planning/phases/12-osint_iot_cloud_storage/12-02-SUMMARY.md new file mode 100644 index 0000000..ec4a2ee --- /dev/null +++ b/.planning/phases/12-osint_iot_cloud_storage/12-02-SUMMARY.md @@ -0,0 +1,103 @@ +--- +phase: 12-osint_iot_cloud_storage +plan: 02 +subsystem: recon +tags: [fofa, netlas, binaryedge, iot, osint, httptest] + +requires: + - phase: 09-osint-infrastructure + provides: LimiterRegistry, shared Client retry/backoff HTTP + - phase: 10-osint-code-hosting + provides: ReconSource interface pattern, BuildQueries, keywordIndex helpers +provides: + - FOFASource implementing recon.ReconSource for FOFA internet search + - NetlasSource implementing recon.ReconSource for Netlas intelligence API + - BinaryEdgeSource implementing recon.ReconSource for BinaryEdge data API +affects: [12-osint_iot_cloud_storage, cmd/recon] + +tech-stack: + added: [] + patterns: [base64-encoded query params for FOFA, X-API-Key header auth for Netlas, X-Key header auth for BinaryEdge] + +key-files: + created: + - pkg/recon/sources/fofa.go + - pkg/recon/sources/fofa_test.go + - pkg/recon/sources/netlas.go + - pkg/recon/sources/netlas_test.go + - pkg/recon/sources/binaryedge.go + - pkg/recon/sources/binaryedge_test.go + modified: [] + +key-decisions: + - "FOFA uses base64-encoded qbase64 param with email+key auth in query string" + - "Netlas uses X-API-Key header; BinaryEdge uses X-Key header for auth" + - "All three sources use bare keyword queries (default formatQuery path)" + +patterns-established: + - "IoT scanner source pattern: struct with APIKey/BaseURL/Registry/Limiters + lazy client init" + +requirements-completed: [RECON-IOT-04, RECON-IOT-05, RECON-IOT-06] + +duration: 2min +completed: 2026-04-06 +--- + +# Phase 12 Plan 02: FOFA, Netlas, BinaryEdge Sources Summary + +**Three IoT/device scanner recon sources (FOFA, Netlas, BinaryEdge) with httptest-based unit tests covering sweep, auth, and cancellation** + +## Performance + +- **Duration:** 2 min +- **Started:** 2026-04-06T09:22:18Z +- **Completed:** 2026-04-06T09:24:22Z +- **Tasks:** 2 +- **Files modified:** 6 + +## Accomplishments +- FOFASource searches FOFA API with base64-encoded queries and email+key authentication +- NetlasSource searches Netlas API with X-API-Key header authentication +- BinaryEdgeSource searches BinaryEdge API with X-Key header authentication +- All three sources follow established Phase 10 pattern with shared Client, LimiterRegistry, BuildQueries + +## Task Commits + +Each task was committed atomically: + +1. **Task 1: Implement FOFASource, NetlasSource, BinaryEdgeSource** - `270bbbf` (feat) +2. **Task 2: Unit tests for FOFA, Netlas, BinaryEdge sources** - `d6c35f4` (test) + +## Files Created/Modified +- `pkg/recon/sources/fofa.go` - FOFASource with base64 query encoding and dual-credential auth +- `pkg/recon/sources/fofa_test.go` - httptest tests for FOFA sweep, credentials, cancellation +- `pkg/recon/sources/netlas.go` - NetlasSource with X-API-Key header auth +- `pkg/recon/sources/netlas_test.go` - httptest tests for Netlas sweep, credentials, cancellation +- `pkg/recon/sources/binaryedge.go` - BinaryEdgeSource with X-Key header auth +- `pkg/recon/sources/binaryedge_test.go` - httptest tests for BinaryEdge sweep, credentials, cancellation + +## Decisions Made +- FOFA uses base64-encoded qbase64 query parameter (matching FOFA API spec) with email+key in query string +- Netlas uses X-API-Key header; BinaryEdge uses X-Key header (matching their respective API specs) +- All three use bare keyword queries via default formatQuery path (no source-specific query formatting needed) + +## Deviations from Plan + +None - plan executed exactly as written. + +## Issues Encountered +None + +## Known Stubs +None + +## User Setup Required +None - no external service configuration required. + +## Next Phase Readiness +- Three IoT scanner sources ready for RegisterAll wiring +- FOFA requires email + API key; Netlas and BinaryEdge require API key only + +--- +*Phase: 12-osint_iot_cloud_storage* +*Completed: 2026-04-06*