keyhunter/pkg/recon/sources/crtsh.go

package sources

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	"golang.org/x/time/rate"

	"github.com/salvacybersec/keyhunter/pkg/providers"
	"github.com/salvacybersec/keyhunter/pkg/recon"
)

// CrtShSource discovers subdomains via certificate transparency logs (crt.sh)
// and probes their config endpoints (/.env, /api/config, /actuator/env) for
// leaked API keys.
type CrtShSource struct {
	BaseURL  string
	Registry *providers.Registry
	Limiters *recon.LimiterRegistry
	Client   *Client

	// ProbeBaseURL overrides the scheme+host used when probing discovered
	// subdomains. Tests set this to the httptest server URL.
	ProbeBaseURL string
}

var _ recon.ReconSource = (*CrtShSource)(nil)

func (s *CrtShSource) Name() string               { return "crtsh" }
func (s *CrtShSource) RateLimit() rate.Limit       { return rate.Every(3 * time.Second) }
func (s *CrtShSource) Burst() int                  { return 3 }
func (s *CrtShSource) RespectsRobots() bool        { return false }
func (s *CrtShSource) Enabled(_ recon.Config) bool { return true }

// crtshEntry represents one row from the crt.sh JSON API.
type crtshEntry struct {
	NameValue  string `json:"name_value"`
	CommonName string `json:"common_name"`
}

// configProbeEndpoints are the well-known config endpoints probed on each
// discovered subdomain.
var configProbeEndpoints = []string{
	"/.env",
	"/api/config",
	"/actuator/env",
}

func (s *CrtShSource) Sweep(ctx context.Context, query string, out chan<- recon.Finding) error {
	base := s.BaseURL
	if base == "" {
		base = "https://crt.sh"
	}
	client := s.Client
	if client == nil {
		client = NewClient()
	}

	// query should be a domain. Skip keyword-like queries (no dots).
	if query == "" || !strings.Contains(query, ".") {
		return nil
	}

	if s.Limiters != nil {
		if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
			return err
		}
	}

	// Fetch subdomains from crt.sh.
	crtURL := fmt.Sprintf("%s/?q=%%25.%s&output=json", base, url.QueryEscape(query))
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, crtURL, nil)
	if err != nil {
		return err
	}

	resp, err := client.Do(ctx, req)
	if err != nil {
		return nil // non-fatal: crt.sh may be down
	}

	data, err := io.ReadAll(io.LimitReader(resp.Body, 1024*1024))
	_ = resp.Body.Close()
	if err != nil {
		return nil
	}

	var entries []crtshEntry
	if err := json.Unmarshal(data, &entries); err != nil {
		return nil
	}

	// Deduplicate name_value entries.
	seen := make(map[string]struct{})
	var subdomains []string
	for _, e := range entries {
		// name_value can contain multiple names separated by newlines.
		for _, name := range strings.Split(e.NameValue, "\n") {
			name = strings.TrimSpace(name)
			if name == "" {
				continue
			}
			// Remove wildcard prefix.
			name = strings.TrimPrefix(name, "*.")
			if _, ok := seen[name]; ok {
				continue
			}
			seen[name] = struct{}{}
			subdomains = append(subdomains, name)
			if len(subdomains) >= 20 {
				break
			}
		}
		if len(subdomains) >= 20 {
			break
		}
	}

	// Probe config endpoints on each subdomain.
	probeClient := &http.Client{Timeout: 5 * time.Second}
	for _, sub := range subdomains {
		if err := ctx.Err(); err != nil {
			return err
		}
		s.probeSubdomain(ctx, probeClient, sub, out)
	}
	return nil
}

// probeSubdomain checks well-known config endpoints for key patterns.
func (s *CrtShSource) probeSubdomain(ctx context.Context, probeClient *http.Client, subdomain string, out chan<- recon.Finding) {
	for _, ep := range configProbeEndpoints {
		if err := ctx.Err(); err != nil {
			return
		}

		var probeURL string
		if s.ProbeBaseURL != "" {
			// Test mode: use the mock server URL with subdomain as a header/path hint.
			probeURL = s.ProbeBaseURL + "/" + subdomain + ep
		} else {
			probeURL = "https://" + subdomain + ep
		}

		req, err := http.NewRequestWithContext(ctx, http.MethodGet, probeURL, nil)
		if err != nil {
			continue
		}

		resp, err := probeClient.Do(req)
		if err != nil {
			continue
		}

		body, err := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
		_ = resp.Body.Close()
		if err != nil {
			continue
		}

		if resp.StatusCode == http.StatusOK && ciLogKeyPattern.Match(body) {
			out <- recon.Finding{
				ProviderName: subdomain,
				Source:       probeURL,
				SourceType:   "recon:crtsh",
				Confidence:   "high",
				DetectedAt:   time.Now(),
			}
		}
	}
}