merge: phase 14-04 register wiring

pkg/recon/sources/wayback.go

@@ -1,12 +1,20 @@
 package sources
 
 import (
+<<<<<<< HEAD
 	"bufio"
 	"context"
 	"fmt"
 	"net/http"
 	"net/url"
 	"strings"
+=======
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+>>>>>>> worktree-agent-adad8c10
 	"time"
 
 	"golang.org/x/time/rate"
@@ -15,6 +23,7 @@ import (
 	"github.com/salvacybersec/keyhunter/pkg/recon"
 )
 
+<<<<<<< HEAD
 // WaybackMachineSource implements recon.ReconSource against the Wayback Machine
 // CDX Server API. It queries web.archive.org/cdx/search/cdx for historical
 // snapshots of pages matching provider keywords (e.g. domains known to host
@@ -48,6 +57,26 @@ func (s *WaybackMachineSource) Enabled(_ recon.Config) bool { return true }
 // a Finding for every archived snapshot URL returned. The CDX API returns
 // plain-text lines with space-separated fields; we extract the original URL
 // and timestamp to construct the full Wayback snapshot link.
+=======
+// WaybackMachineSource searches the Internet Archive's Wayback Machine CDX API
+// for archived pages that may contain leaked API keys. Developers sometimes
+// remove secrets from live pages but cached versions persist in web archives.
+type WaybackMachineSource struct {
+	BaseURL  string
+	Registry *providers.Registry
+	Limiters *recon.LimiterRegistry
+	Client   *Client
+}
+
+var _ recon.ReconSource = (*WaybackMachineSource)(nil)
+
+func (s *WaybackMachineSource) Name() string              { return "wayback" }
+func (s *WaybackMachineSource) RateLimit() rate.Limit      { return rate.Every(5 * time.Second) }
+func (s *WaybackMachineSource) Burst() int                 { return 1 }
+func (s *WaybackMachineSource) RespectsRobots() bool       { return true }
+func (s *WaybackMachineSource) Enabled(_ recon.Config) bool { return true }
+
+>>>>>>> worktree-agent-adad8c10
 func (s *WaybackMachineSource) Sweep(ctx context.Context, _ string, out chan<- recon.Finding) error {
 	base := s.BaseURL
 	if base == "" {
@@ -74,6 +103,7 @@ func (s *WaybackMachineSource) Sweep(ctx context.Context, _ string, out chan<- r
 			}
 		}
 
+<<<<<<< HEAD
 		// CDX API: output=text, fl=timestamp,original limits response to two fields per line.
 		// limit=50 keeps the response bounded per keyword.
 		endpoint := fmt.Sprintf("%s/cdx/search/cdx?url=*&output=text&fl=timestamp,original&limit=50&matchType=prefix&filter=statuscode:200&query=%s",
@@ -121,6 +151,79 @@ func (s *WaybackMachineSource) Sweep(ctx context.Context, _ string, out chan<- r
 			}
 		}
 		_ = resp.Body.Close()
+=======
+		// CDX API: search for archived URLs matching the query.
+		// Filter for .env, config, and JS files that commonly contain keys.
+		cdxURL := fmt.Sprintf("%s/cdx/search/cdx?url=*%s*&output=json&limit=10&fl=url,timestamp,statuscode&filter=statuscode:200", base, q)
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, cdxURL, nil)
+		if err != nil {
+			continue
+		}
+		req.Header.Set("Accept", "application/json")
+
+		resp, err := client.Do(ctx, req)
+		if err != nil {
+			continue
+		}
+
+		var rows [][]string
+		if err := json.NewDecoder(resp.Body).Decode(&rows); err != nil {
+			_ = resp.Body.Close()
+			continue
+		}
+		_ = resp.Body.Close()
+
+		// Skip the header row if present.
+		start := 0
+		if len(rows) > 0 && len(rows[0]) > 0 && rows[0][0] == "url" {
+			start = 1
+		}
+
+		for _, row := range rows[start:] {
+			if err := ctx.Err(); err != nil {
+				return err
+			}
+			if len(row) < 2 {
+				continue
+			}
+			archivedURL := row[0]
+			timestamp := row[1]
+
+			if s.Limiters != nil {
+				if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
+					return err
+				}
+			}
+
+			// Fetch the archived page content.
+			snapshotURL := fmt.Sprintf("%s/web/%sid_/%s", base, timestamp, archivedURL)
+			snapReq, err := http.NewRequestWithContext(ctx, http.MethodGet, snapshotURL, nil)
+			if err != nil {
+				continue
+			}
+
+			snapResp, err := client.Do(ctx, snapReq)
+			if err != nil {
+				continue
+			}
+
+			body, err := io.ReadAll(io.LimitReader(snapResp.Body, 256*1024))
+			_ = snapResp.Body.Close()
+			if err != nil {
+				continue
+			}
+
+			if apiKeyPattern.Match(body) {
+				out <- recon.Finding{
+					ProviderName: q,
+					Source:       snapshotURL,
+					SourceType:   "recon:wayback",
+					Confidence:   "medium",
+					DetectedAt:   time.Now(),
+				}
+			}
+		}
+>>>>>>> worktree-agent-adad8c10
 	}
 	return nil
 }