keyhunter/pkg/recon/sources/intelligencex.go

package sources

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"

	"golang.org/x/time/rate"

	"github.com/salvacybersec/keyhunter/pkg/providers"
	"github.com/salvacybersec/keyhunter/pkg/recon"
)

// IntelligenceXSource searches the IntelligenceX archive for leaked credentials.
// IX indexes breached databases, paste sites, and dark web content, making it
// a high-value source for discovering leaked API keys.
type IntelligenceXSource struct {
	APIKey   string
	BaseURL  string
	Registry *providers.Registry
	Limiters *recon.LimiterRegistry
	Client   *Client
}

var _ recon.ReconSource = (*IntelligenceXSource)(nil)

func (s *IntelligenceXSource) Name() string         { return "intelligencex" }
func (s *IntelligenceXSource) RateLimit() rate.Limit { return rate.Every(5 * time.Second) }
func (s *IntelligenceXSource) Burst() int            { return 3 }
func (s *IntelligenceXSource) RespectsRobots() bool   { return false }
func (s *IntelligenceXSource) Enabled(_ recon.Config) bool {
	return s.APIKey != ""
}

// ixSearchRequest is the JSON body for the IX search endpoint.
type ixSearchRequest struct {
	Term       string `json:"term"`
	MaxResults int    `json:"maxresults"`
	Media      int    `json:"media"`
	Timeout    int    `json:"timeout"`
}

// ixSearchResponse is the response from the IX search initiation endpoint.
type ixSearchResponse struct {
	ID     string `json:"id"`
	Status int    `json:"status"`
}

// ixResultResponse is the response from the IX search results endpoint.
type ixResultResponse struct {
	Records []ixRecord `json:"records"`
}

// ixRecord is a single record in the IX search results.
type ixRecord struct {
	SystemID  string `json:"systemid"`
	Name      string `json:"name"`
	StorageID string `json:"storageid"`
	Bucket    string `json:"bucket"`
}

func (s *IntelligenceXSource) Sweep(ctx context.Context, query string, out chan<- recon.Finding) error {
	base := s.BaseURL
	if base == "" {
		base = "https://2.intelx.io"
	}
	client := s.Client
	if client == nil {
		client = NewClient()
	}

	queries := BuildQueries(s.Registry, "intelligencex")
	if len(queries) == 0 {
		return nil
	}

	for _, q := range queries {
		if err := ctx.Err(); err != nil {
			return err
		}

		if s.Limiters != nil {
			if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
				return err
			}
		}

		// Step 1: Initiate search.
		searchBody, _ := json.Marshal(ixSearchRequest{
			Term:       q,
			MaxResults: 10,
			Media:      0,
			Timeout:    5,
		})

		searchURL := fmt.Sprintf("%s/intelligent/search", base)
		req, err := http.NewRequestWithContext(ctx, http.MethodPost, searchURL, bytes.NewReader(searchBody))
		if err != nil {
			continue
		}
		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("x-key", s.APIKey)

		resp, err := client.Do(ctx, req)
		if err != nil {
			continue
		}

		respData, err := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
		_ = resp.Body.Close()
		if err != nil {
			continue
		}

		var searchResp ixSearchResponse
		if err := json.Unmarshal(respData, &searchResp); err != nil {
			continue
		}
		if searchResp.ID == "" {
			continue
		}

		// Step 2: Fetch results.
		if s.Limiters != nil {
			if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
				return err
			}
		}

		resultURL := fmt.Sprintf("%s/intelligent/search/result?id=%s&limit=10", base, searchResp.ID)
		resReq, err := http.NewRequestWithContext(ctx, http.MethodGet, resultURL, nil)
		if err != nil {
			continue
		}
		resReq.Header.Set("x-key", s.APIKey)

		resResp, err := client.Do(ctx, resReq)
		if err != nil {
			continue
		}

		resData, err := io.ReadAll(io.LimitReader(resResp.Body, 512*1024))
		_ = resResp.Body.Close()
		if err != nil {
			continue
		}

		var resultResp ixResultResponse
		if err := json.Unmarshal(resData, &resultResp); err != nil {
			continue
		}

		// Step 3: Fetch content for each record and check for keys.
		for _, rec := range resultResp.Records {
			if err := ctx.Err(); err != nil {
				return err
			}

			if s.Limiters != nil {
				if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
					return err
				}
			}

			fileURL := fmt.Sprintf(
				"%s/file/read?type=0&storageid=%s&bucket=%s",
				base, rec.StorageID, rec.Bucket,
			)
			fileReq, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil)
			if err != nil {
				continue
			}
			fileReq.Header.Set("x-key", s.APIKey)

			fileResp, err := client.Do(ctx, fileReq)
			if err != nil {
				continue
			}

			fileData, err := io.ReadAll(io.LimitReader(fileResp.Body, 512*1024))
			_ = fileResp.Body.Close()
			if err != nil {
				continue
			}

			if ciLogKeyPattern.Match(fileData) {
				out <- recon.Finding{
					ProviderName: q,
					Source:       fmt.Sprintf("%s/file/read?storageid=%s", base, rec.StorageID),
					SourceType:   "recon:intelligencex",
					Confidence:   "medium",
					DetectedAt:   time.Now(),
				}
			}
		}
	}
	return nil
}