feat(10-07): add SandboxesSource aggregator (codepen/jsfiddle/stackblitz/glitch/observable)
- Single ReconSource umbrella iterating per-platform HTML or JSON search endpoints - Per-platform failures logged and skipped (log-and-continue); ctx cancel aborts fast - Sub-platform identifier encoded in Finding.KeyMasked as 'platform=<name>' (pragmatic slot) - Gitpod intentionally omitted (no public search) - 5 httptest-backed tests covering HTML+JSON extraction, platform-failure tolerance, ctx cancel
This commit is contained in:
salvacybersec
180
pkg/recon/sources/sandboxes_test.go
Normal file
180
pkg/recon/sources/sandboxes_test.go
Normal file
@@ -0,0 +1,180 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
func sandboxesTestRegistry() *providers.Registry {
|
||||
return providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
}
|
||||
|
||||
// sandboxesTestServer serves:
|
||||
// - /codepen-search : HTML with pen anchors
|
||||
// - /jsfiddle-search : JSON with results
|
||||
// - /fail-search : 500 to exercise per-platform failure tolerance
|
||||
func sandboxesTestServer(t *testing.T) *httptest.Server {
|
||||
t.Helper()
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/codepen-search", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write([]byte(`<html><body>
|
||||
<a href="/alice/pen/AbCdEf123">one</a>
|
||||
<a href="/bob/pen/ZzZz9999">two</a>
|
||||
<a href="/nope">skip</a>
|
||||
</body></html>`))
|
||||
})
|
||||
mux.HandleFunc("/jsfiddle-search", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"results":[
|
||||
{"url":"https://jsfiddle.net/u/abcd1234/"},
|
||||
{"url":"https://jsfiddle.net/u/wxyz5678/"}
|
||||
]}`))
|
||||
})
|
||||
mux.HandleFunc("/fail-search", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = w.Write([]byte("boom"))
|
||||
})
|
||||
return httptest.NewServer(mux)
|
||||
}
|
||||
|
||||
func newSandboxesTestSource(srvURL string, plats []subPlatform) *SandboxesSource {
|
||||
return &SandboxesSource{
|
||||
Platforms: plats,
|
||||
Registry: sandboxesTestRegistry(),
|
||||
Limiters: recon.NewLimiterRegistry(),
|
||||
Client: NewClient(),
|
||||
BaseURL: srvURL,
|
||||
}
|
||||
}
|
||||
|
||||
func TestSandboxes_Sweep_HTMLAndJSON(t *testing.T) {
|
||||
srv := sandboxesTestServer(t)
|
||||
defer srv.Close()
|
||||
|
||||
plats := []subPlatform{
|
||||
{Name: "codepen", SearchPath: "/codepen-search?q=%s", ResultLinkRegex: `^/[^/]+/pen/[a-zA-Z0-9]+$`, IsJSON: false},
|
||||
{Name: "jsfiddle", SearchPath: "/jsfiddle-search?q=%s", IsJSON: true, JSONItemsKey: "results", JSONURLKey: "url"},
|
||||
}
|
||||
src := newSandboxesTestSource(srv.URL, plats)
|
||||
|
||||
out := make(chan recon.Finding, 32)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := src.Sweep(ctx, "", out); err != nil {
|
||||
t.Fatalf("Sweep err: %v", err)
|
||||
}
|
||||
close(out)
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
// codepen: 2 hits, jsfiddle: 2 hits
|
||||
if len(findings) != 4 {
|
||||
t.Fatalf("expected 4 findings, got %d: %+v", len(findings), findings)
|
||||
}
|
||||
|
||||
platforms := map[string]int{}
|
||||
for _, f := range findings {
|
||||
if f.SourceType != "recon:sandboxes" {
|
||||
t.Errorf("unexpected SourceType: %s", f.SourceType)
|
||||
}
|
||||
// sub-platform identifier is encoded into KeyMasked as "platform=<name>"
|
||||
platforms[f.KeyMasked]++
|
||||
}
|
||||
if platforms["platform=codepen"] != 2 {
|
||||
t.Errorf("expected 2 codepen findings, got %d", platforms["platform=codepen"])
|
||||
}
|
||||
if platforms["platform=jsfiddle"] != 2 {
|
||||
t.Errorf("expected 2 jsfiddle findings, got %d", platforms["platform=jsfiddle"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSandboxes_Sweep_FailingPlatformDoesNotAbortOthers(t *testing.T) {
|
||||
srv := sandboxesTestServer(t)
|
||||
defer srv.Close()
|
||||
|
||||
plats := []subPlatform{
|
||||
{Name: "broken", SearchPath: "/fail-search?q=%s", ResultLinkRegex: `^/x$`, IsJSON: false},
|
||||
{Name: "codepen", SearchPath: "/codepen-search?q=%s", ResultLinkRegex: `^/[^/]+/pen/[a-zA-Z0-9]+$`, IsJSON: false},
|
||||
}
|
||||
src := newSandboxesTestSource(srv.URL, plats)
|
||||
|
||||
out := make(chan recon.Finding, 32)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := src.Sweep(ctx, "", out); err != nil {
|
||||
t.Fatalf("Sweep err (should be nil, log-and-continue): %v", err)
|
||||
}
|
||||
close(out)
|
||||
|
||||
var n int
|
||||
for f := range out {
|
||||
if f.KeyMasked != "platform=codepen" {
|
||||
t.Errorf("unexpected platform: %s", f.KeyMasked)
|
||||
}
|
||||
n++
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("expected 2 codepen findings after broken platform skipped, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSandboxes_RespectsRobotsAndName(t *testing.T) {
|
||||
s := &SandboxesSource{}
|
||||
if !s.RespectsRobots() {
|
||||
t.Fatal("expected RespectsRobots=true")
|
||||
}
|
||||
if s.Name() != "sandboxes" {
|
||||
t.Fatalf("unexpected name: %s", s.Name())
|
||||
}
|
||||
if !s.Enabled(recon.Config{}) {
|
||||
t.Fatal("expected Enabled=true")
|
||||
}
|
||||
if s.Burst() != 1 {
|
||||
t.Fatal("expected Burst=1")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSandboxes_Sweep_CtxCancelled(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
plats := []subPlatform{
|
||||
{Name: "codepen", SearchPath: "/s?q=%s", ResultLinkRegex: `^/x$`, IsJSON: false},
|
||||
}
|
||||
src := newSandboxesTestSource(srv.URL, plats)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
out := make(chan recon.Finding, 4)
|
||||
if err := src.Sweep(ctx, "", out); err == nil {
|
||||
t.Fatal("expected ctx error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSandboxes_DefaultPlatformsListed(t *testing.T) {
|
||||
// Sanity check: defaultPlatforms should contain the five documented sub-platforms.
|
||||
want := map[string]bool{"codepen": true, "jsfiddle": true, "stackblitz": true, "glitch": true, "observable": true}
|
||||
got := map[string]bool{}
|
||||
for _, p := range defaultPlatforms {
|
||||
got[p.Name] = true
|
||||
}
|
||||
for k := range want {
|
||||
if !got[k] {
|
||||
t.Errorf("missing default platform: %s", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user