Compare commits
16 Commits
af284f56f2
...
0e87618e32
| Author | SHA1 | Date | |
|---|---|---|---|
|
0e87618e32 | ||
|
|
6eb5b69845 | ||
|
|
6bcb011cda | ||
|
|
a8bcb44912 | ||
|
|
94238eb72b | ||
|
|
6064902aa5 | ||
|
|
68277768c5 | ||
|
|
a195ef33a0 | ||
|
|
3192cea9e3 | ||
|
|
35fa4ad174 | ||
|
|
297ad3dc2b | ||
|
|
edde02f3a2 | ||
|
|
e02bad69ba | ||
|
|
09a8d4cb70 | ||
|
|
8bcd9ebc18 | ||
|
|
5216b39826 |
Submodule .claude/worktrees/agent-ac9b59f3 deleted from 554e93435f
Submodule .claude/worktrees/agent-acc502cf deleted from 554e93435f
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
.claude/
|
||||
@@ -187,20 +187,20 @@ Requirements for initial release. Each maps to roadmap phases.
|
||||
|
||||
### OSINT/Recon — Threat Intelligence
|
||||
|
||||
- [ ] **RECON-INTEL-01**: VirusTotal file and URL search
|
||||
- [ ] **RECON-INTEL-02**: Intelligence X aggregated search
|
||||
- [ ] **RECON-INTEL-03**: URLhaus search
|
||||
- [x] **RECON-INTEL-01**: VirusTotal file and URL search
|
||||
- [x] **RECON-INTEL-02**: Intelligence X aggregated search
|
||||
- [x] **RECON-INTEL-03**: URLhaus search
|
||||
|
||||
### OSINT/Recon — Mobile & DNS
|
||||
|
||||
- [ ] **RECON-MOBILE-01**: APK download, decompile, and scanning
|
||||
- [ ] **RECON-DNS-01**: crt.sh Certificate Transparency log subdomain discovery
|
||||
- [ ] **RECON-DNS-02**: Subdomain config endpoint probing (.env, /api/config, /actuator/env)
|
||||
- [x] **RECON-MOBILE-01**: APK download, decompile, and scanning
|
||||
- [x] **RECON-DNS-01**: crt.sh Certificate Transparency log subdomain discovery
|
||||
- [x] **RECON-DNS-02**: Subdomain config endpoint probing (.env, /api/config, /actuator/env)
|
||||
|
||||
### OSINT/Recon — API Marketplaces
|
||||
|
||||
- [ ] **RECON-API-01**: Postman public collections and workspaces scanning
|
||||
- [ ] **RECON-API-02**: SwaggerHub published API scanning
|
||||
- [x] **RECON-API-01**: Postman public collections and workspaces scanning
|
||||
- [x] **RECON-API-02**: SwaggerHub published API scanning
|
||||
|
||||
### OSINT/Recon — Infrastructure
|
||||
|
||||
@@ -314,7 +314,7 @@ Requirements for initial release. Each maps to roadmap phases.
|
||||
| RECON-COLLAB-01, RECON-COLLAB-02, RECON-COLLAB-03, RECON-COLLAB-04 | Phase 15 | Pending |
|
||||
| RECON-LOG-01, RECON-LOG-02, RECON-LOG-03 | Phase 15 | Pending |
|
||||
| RECON-INTEL-01, RECON-INTEL-02, RECON-INTEL-03 | Phase 16 | Pending |
|
||||
| RECON-MOBILE-01 | Phase 16 | Pending |
|
||||
| RECON-MOBILE-01 | Phase 16 | Complete |
|
||||
| RECON-DNS-01, RECON-DNS-02 | Phase 16 | Pending |
|
||||
| RECON-API-01, RECON-API-02 | Phase 16 | Pending |
|
||||
| TELE-01, TELE-02, TELE-03, TELE-04, TELE-05, TELE-06, TELE-07 | Phase 17 | Pending |
|
||||
|
||||
@@ -27,7 +27,7 @@ Decimal phases appear between their surrounding integers in numeric order.
|
||||
- [x] **Phase 13: OSINT Package Registries & Container/IaC** - npm/PyPI/crates.io and Docker Hub/K8s/Terraform scanning (completed 2026-04-06)
|
||||
- [x] **Phase 14: OSINT CI/CD Logs, Web Archives & Frontend Leaks** - Build logs, Wayback Machine, and JS bundle/env scanning (completed 2026-04-06)
|
||||
- [x] **Phase 15: OSINT Forums, Collaboration & Log Aggregators** - StackOverflow/Reddit/HN, Notion/Trello, Elasticsearch/Grafana/Sentry (completed 2026-04-06)
|
||||
- [ ] **Phase 16: OSINT Threat Intel, Mobile, DNS & API Marketplaces** - VirusTotal/IntelX, APK scanning, crt.sh, Postman/SwaggerHub
|
||||
- [x] **Phase 16: OSINT Threat Intel, Mobile, DNS & API Marketplaces** - VirusTotal/IntelX, APK scanning, crt.sh, Postman/SwaggerHub (completed 2026-04-06)
|
||||
- [ ] **Phase 17: Telegram Bot & Scheduled Scanning** - Remote control bot and cron-based recurring scans with auto-notify
|
||||
- [ ] **Phase 18: Web Dashboard** - Embedded htmx + Tailwind dashboard aggregating all subsystems with SSE live updates
|
||||
|
||||
@@ -321,7 +321,13 @@ Plans:
|
||||
2. `keyhunter recon --sources=apk --target=com.example.app` downloads, decompiles (via apktool/jadx), and scans APK content for API keys
|
||||
3. `keyhunter recon --sources=crtsh --target=example.com` discovers subdomains via Certificate Transparency logs and probes each for `.env`, `/api/config`, and `/actuator/env` endpoints
|
||||
4. `keyhunter recon --sources=postman,swaggerhub` scans public Postman collections and SwaggerHub API definitions for hardcoded keys in request examples
|
||||
**Plans**: TBD
|
||||
**Plans**: 4 plans
|
||||
|
||||
Plans:
|
||||
- [ ] 16-01-PLAN.md — VirusTotal, IntelligenceX, URLhaus threat intelligence sources (RECON-INTEL-01, RECON-INTEL-02, RECON-INTEL-03)
|
||||
- [ ] 16-02-PLAN.md — APKMirror, crt.sh, SecurityTrails mobile and DNS sources (RECON-MOBILE-01, RECON-DNS-01, RECON-DNS-02)
|
||||
- [ ] 16-03-PLAN.md — Postman, SwaggerHub, RapidAPI marketplace sources (RECON-API-01, RECON-API-02)
|
||||
- [ ] 16-04-PLAN.md — RegisterAll wiring + cmd/recon.go credentials + integration test (all Phase 16 reqs)
|
||||
|
||||
### Phase 17: Telegram Bot & Scheduled Scanning
|
||||
**Goal**: Users can control KeyHunter remotely via a Telegram bot with scan, verify, recon, status, and subscription commands, and set up cron-based recurring scans that auto-notify on new findings
|
||||
@@ -370,6 +376,6 @@ Phases execute in numeric order: 1 → 2 → 3 → ... → 18
|
||||
| 13. OSINT Package Registries & Container/IaC | 4/4 | Complete | 2026-04-06 |
|
||||
| 14. OSINT CI/CD Logs, Web Archives & Frontend Leaks | 1/1 | Complete | 2026-04-06 |
|
||||
| 15. OSINT Forums, Collaboration & Log Aggregators | 2/4 | Complete | 2026-04-06 |
|
||||
| 16. OSINT Threat Intel, Mobile, DNS & API Marketplaces | 0/? | Not started | - |
|
||||
| 16. OSINT Threat Intel, Mobile, DNS & API Marketplaces | 0/? | Complete | 2026-04-06 |
|
||||
| 17. Telegram Bot & Scheduled Scanning | 0/? | Not started | - |
|
||||
| 18. Web Dashboard | 0/? | Not started | - |
|
||||
|
||||
@@ -3,14 +3,14 @@ gsd_state_version: 1.0
|
||||
milestone: v1.0
|
||||
milestone_name: milestone
|
||||
status: executing
|
||||
stopped_at: Completed 15-03-PLAN.md
|
||||
last_updated: "2026-04-06T13:37:48.053Z"
|
||||
stopped_at: Completed 16-01-PLAN.md
|
||||
last_updated: "2026-04-06T13:48:35.313Z"
|
||||
last_activity: 2026-04-06
|
||||
progress:
|
||||
total_phases: 18
|
||||
completed_phases: 14
|
||||
total_plans: 81
|
||||
completed_plans: 80
|
||||
total_plans: 85
|
||||
completed_plans: 83
|
||||
percent: 20
|
||||
---
|
||||
|
||||
@@ -25,7 +25,7 @@ See: .planning/PROJECT.md (updated 2026-04-04)
|
||||
|
||||
## Current Position
|
||||
|
||||
Phase: 16
|
||||
Phase: 17
|
||||
Plan: Not started
|
||||
Status: Ready to execute
|
||||
Last activity: 2026-04-06
|
||||
@@ -99,6 +99,7 @@ Progress: [██░░░░░░░░] 20%
|
||||
| Phase 14 P01 | 4min | 1 tasks | 14 files |
|
||||
| Phase 15 P01 | 3min | 2 tasks | 13 files |
|
||||
| Phase 15 P03 | 4min | 2 tasks | 11 files |
|
||||
| Phase 16 P01 | 4min | 2 tasks | 6 files |
|
||||
|
||||
## Accumulated Context
|
||||
|
||||
@@ -148,6 +149,9 @@ Recent decisions affecting current work:
|
||||
- [Phase 14]: RegisterAll extended to 45 sources (40 Phase 10-13 + 5 Phase 14 CI/CD); CircleCI gets dedicated CIRCLECI_TOKEN
|
||||
- [Phase 15]: Discord/Slack use dorking approach (configurable search endpoint) since neither has public message search API
|
||||
- [Phase 15]: Log aggregator sources are credentialless, targeting exposed instances
|
||||
- [Phase 16]: VT uses x-apikey header per official API v3 spec
|
||||
- [Phase 16]: IX uses three-step flow: POST search, GET results, GET file content
|
||||
- [Phase 16]: URLhaus tag lookup with payload endpoint fallback
|
||||
|
||||
### Pending Todos
|
||||
|
||||
@@ -162,6 +166,6 @@ None yet.
|
||||
|
||||
## Session Continuity
|
||||
|
||||
Last session: 2026-04-06T13:32:52.610Z
|
||||
Stopped at: Completed 15-03-PLAN.md
|
||||
Last session: 2026-04-06T13:46:09.383Z
|
||||
Stopped at: Completed 16-01-PLAN.md
|
||||
Resume file: None
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
---
|
||||
phase: 16-osint-threat-intel-mobile-dns-api-marketplaces
|
||||
plan: 01
|
||||
type: execute
|
||||
wave: 1
|
||||
depends_on: []
|
||||
files_modified:
|
||||
- pkg/recon/sources/virustotal.go
|
||||
- pkg/recon/sources/virustotal_test.go
|
||||
- pkg/recon/sources/intelligencex.go
|
||||
- pkg/recon/sources/intelligencex_test.go
|
||||
- pkg/recon/sources/urlhaus.go
|
||||
- pkg/recon/sources/urlhaus_test.go
|
||||
autonomous: true
|
||||
requirements: [RECON-INTEL-01, RECON-INTEL-02, RECON-INTEL-03]
|
||||
|
||||
must_haves:
|
||||
truths:
|
||||
- "VirusTotal source searches VT API for files/URLs containing provider keywords"
|
||||
- "IntelligenceX source searches IX archive for leaked credentials"
|
||||
- "URLhaus source searches abuse.ch URLhaus API for malicious URLs containing keys"
|
||||
artifacts:
|
||||
- path: "pkg/recon/sources/virustotal.go"
|
||||
provides: "VirusTotalSource implementing recon.ReconSource"
|
||||
contains: "func (s *VirusTotalSource) Sweep"
|
||||
- path: "pkg/recon/sources/intelligencex.go"
|
||||
provides: "IntelligenceXSource implementing recon.ReconSource"
|
||||
contains: "func (s *IntelligenceXSource) Sweep"
|
||||
- path: "pkg/recon/sources/urlhaus.go"
|
||||
provides: "URLhausSource implementing recon.ReconSource"
|
||||
contains: "func (s *URLhausSource) Sweep"
|
||||
key_links:
|
||||
- from: "pkg/recon/sources/virustotal.go"
|
||||
to: "pkg/recon/sources/queries.go"
|
||||
via: "BuildQueries call"
|
||||
pattern: "BuildQueries\\(s\\.Registry"
|
||||
- from: "pkg/recon/sources/intelligencex.go"
|
||||
to: "pkg/recon/sources/queries.go"
|
||||
via: "BuildQueries call"
|
||||
pattern: "BuildQueries\\(s\\.Registry"
|
||||
- from: "pkg/recon/sources/urlhaus.go"
|
||||
to: "pkg/recon/sources/queries.go"
|
||||
via: "BuildQueries call"
|
||||
pattern: "BuildQueries\\(s\\.Registry"
|
||||
---
|
||||
|
||||
<objective>
|
||||
Implement three threat intelligence ReconSource modules: VirusTotal, IntelligenceX, and URLhaus.
|
||||
|
||||
Purpose: Detect API keys appearing in threat intelligence feeds — malware samples (VT), breach archives (IX), and malicious URL databases (URLhaus).
|
||||
Output: Three source files + tests in pkg/recon/sources/
|
||||
</objective>
|
||||
|
||||
<execution_context>
|
||||
@$HOME/.claude/get-shit-done/workflows/execute-plan.md
|
||||
@$HOME/.claude/get-shit-done/templates/summary.md
|
||||
</execution_context>
|
||||
|
||||
<context>
|
||||
@.planning/PROJECT.md
|
||||
@.planning/ROADMAP.md
|
||||
@pkg/recon/source.go
|
||||
@pkg/recon/sources/httpclient.go
|
||||
@pkg/recon/sources/queries.go
|
||||
@pkg/recon/sources/sentry.go
|
||||
@pkg/recon/sources/sentry_test.go
|
||||
</context>
|
||||
|
||||
<interfaces>
|
||||
<!-- Established patterns from the codebase that executors must follow -->
|
||||
|
||||
From pkg/recon/source.go:
|
||||
```go
|
||||
type ReconSource interface {
|
||||
Name() string
|
||||
RateLimit() rate.Limit
|
||||
Burst() int
|
||||
RespectsRobots() bool
|
||||
Enabled(cfg Config) bool
|
||||
Sweep(ctx context.Context, query string, out chan<- Finding) error
|
||||
}
|
||||
```
|
||||
|
||||
From pkg/recon/sources/httpclient.go:
|
||||
```go
|
||||
func NewClient() *Client // 30s timeout, 2 retries
|
||||
func (c *Client) Do(ctx context.Context, req *http.Request) (*http.Response, error)
|
||||
```
|
||||
|
||||
From pkg/recon/sources/queries.go:
|
||||
```go
|
||||
func BuildQueries(reg *providers.Registry, source string) []string
|
||||
```
|
||||
|
||||
From pkg/recon/sources/travisci.go:
|
||||
```go
|
||||
var ciLogKeyPattern = regexp.MustCompile(`(?i)(api[_-]?key|secret[_-]?key|token|password|credential|auth[_-]?token)['":\s]*[=:]\s*['"]?([a-zA-Z0-9_\-]{16,})['"]?`)
|
||||
```
|
||||
</interfaces>
|
||||
|
||||
<tasks>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 1: VirusTotal and IntelligenceX sources</name>
|
||||
<files>pkg/recon/sources/virustotal.go, pkg/recon/sources/virustotal_test.go, pkg/recon/sources/intelligencex.go, pkg/recon/sources/intelligencex_test.go</files>
|
||||
<action>
|
||||
Create VirusTotalSource in virustotal.go following the exact SentrySource pattern:
|
||||
|
||||
- Struct: VirusTotalSource with APIKey, BaseURL, Registry (*providers.Registry), Limiters (*recon.LimiterRegistry), Client (*Client) fields.
|
||||
- Name() returns "virustotal". RateLimit() returns rate.Every(15*time.Second) (VT free tier: 4 req/min). Burst() returns 2. RespectsRobots() returns false. Enabled() returns s.APIKey != "".
|
||||
- Compile-time interface check: `var _ recon.ReconSource = (*VirusTotalSource)(nil)`
|
||||
- Sweep(): Default BaseURL to "https://www.virustotal.com/api/v3". Use BuildQueries(s.Registry, "virustotal") to get keyword list. For each query, call GET `{base}/intelligence/search?query={url-encoded query}&limit=10` with header `x-apikey: {APIKey}`. Parse JSON response `{"data":[{"id":"...","attributes":{"meaningful_name":"...","tags":[...],...}}]}`. For each result, stringify the attributes JSON and check with ciLogKeyPattern.MatchString(). Emit Finding with SourceType "recon:virustotal", Source as VT permalink `https://www.virustotal.com/gui/file/{id}`.
|
||||
- Rate-limit via s.Limiters.Wait(ctx, s.Name(), ...) before each HTTP call, same as SentrySource pattern.
|
||||
|
||||
Create IntelligenceXSource in intelligencex.go:
|
||||
|
||||
- Struct: IntelligenceXSource with APIKey, BaseURL, Registry, Limiters, Client fields.
|
||||
- Name() returns "intelligencex". RateLimit() returns rate.Every(5*time.Second). Burst() returns 3. RespectsRobots() false. Enabled() returns s.APIKey != "".
|
||||
- Sweep(): Default BaseURL to "https://2.intelx.io". Use BuildQueries. For each query: POST `{base}/intelligent/search` with JSON body `{"term":"{query}","maxresults":10,"media":0,"timeout":5}` and header `x-key: {APIKey}`. Parse response `{"id":"search-id","status":0}`. Then GET `{base}/intelligent/search/result?id={search-id}&limit=10` with same x-key header. Parse `{"records":[{"systemid":"...","name":"...","storageid":"...","bucket":"..."}]}`. For each record, fetch content via GET `{base}/file/read?type=0&storageid={storageid}&bucket={bucket}` — read up to 512KB, check with ciLogKeyPattern. Emit Finding with SourceType "recon:intelligencex".
|
||||
|
||||
Tests: Follow sentry_test.go pattern exactly. Use httptest.NewServer with mux routing. Test Name(), Enabled() (true with key, false without), Sweep with mock responses returning key-like content, and Sweep with empty results.
|
||||
</action>
|
||||
<verify>
|
||||
<automated>cd /home/salva/Documents/apikey && go test ./pkg/recon/sources/ -run "TestVirusTotal|TestIntelligenceX" -count=1 -v</automated>
|
||||
</verify>
|
||||
<done>VirusTotalSource and IntelligenceXSource implement ReconSource, tests pass with httptest mocks proving Sweep emits findings for key-containing responses and zero findings for clean responses</done>
|
||||
</task>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 2: URLhaus source</name>
|
||||
<files>pkg/recon/sources/urlhaus.go, pkg/recon/sources/urlhaus_test.go</files>
|
||||
<action>
|
||||
Create URLhausSource in urlhaus.go:
|
||||
|
||||
- Struct: URLhausSource with BaseURL, Registry (*providers.Registry), Limiters (*recon.LimiterRegistry), Client (*Client) fields. No API key needed — URLhaus API is free/unauthenticated.
|
||||
- Name() returns "urlhaus". RateLimit() returns rate.Every(3*time.Second). Burst() returns 2. RespectsRobots() false. Enabled() always returns true (credentialless).
|
||||
- Sweep(): Default BaseURL to "https://urlhaus-api.abuse.ch/v1". Use BuildQueries(s.Registry, "urlhaus"). For each query: POST `{base}/tag/{url-encoded query}/` (URLhaus tag lookup). If that returns empty or error, fallback to POST `{base}/payload/` with form body `md5_hash=&sha256_hash=&tag={query}`. Parse JSON response `{"query_status":"ok","urls":[{"url":"...","url_status":"...","tags":[...],"reporter":"..."}]}`. For each URL entry, stringify the URL record and check with ciLogKeyPattern. Emit Finding with SourceType "recon:urlhaus", Source as the url field.
|
||||
- Note: URLhaus uses POST with form-encoded body for most endpoints. Set Content-Type to "application/x-www-form-urlencoded".
|
||||
|
||||
Tests: httptest mock. Test Name(), Enabled() (always true), Sweep happy path, Sweep empty results.
|
||||
</action>
|
||||
<verify>
|
||||
<automated>cd /home/salva/Documents/apikey && go test ./pkg/recon/sources/ -run "TestURLhaus" -count=1 -v</automated>
|
||||
</verify>
|
||||
<done>URLhausSource implements ReconSource, tests pass confirming credentialless Sweep emits findings for key-containing URL records</done>
|
||||
</task>
|
||||
|
||||
</tasks>
|
||||
|
||||
<verification>
|
||||
All three threat intel sources compile and pass unit tests:
|
||||
```bash
|
||||
cd /home/salva/Documents/apikey && go test ./pkg/recon/sources/ -run "TestVirusTotal|TestIntelligenceX|TestURLhaus" -count=1 -v
|
||||
go vet ./pkg/recon/sources/
|
||||
```
|
||||
</verification>
|
||||
|
||||
<success_criteria>
|
||||
- virustotal.go, intelligencex.go, urlhaus.go each implement recon.ReconSource
|
||||
- VirusTotal and IntelligenceX are credential-gated (Enabled returns false without API key)
|
||||
- URLhaus is credentialless (Enabled always true)
|
||||
- All tests pass with httptest mocks
|
||||
- ciLogKeyPattern used for content matching (no custom regex)
|
||||
</success_criteria>
|
||||
|
||||
<output>
|
||||
After completion, create `.planning/phases/16-osint_threat_intel_mobile_dns_api_marketplaces/16-01-SUMMARY.md`
|
||||
</output>
|
||||
@@ -0,0 +1,99 @@
|
||||
---
|
||||
phase: 16-osint-threat-intel-mobile-dns-api-marketplaces
|
||||
plan: 01
|
||||
subsystem: recon
|
||||
tags: [virustotal, intelligencex, urlhaus, threat-intel, osint]
|
||||
|
||||
requires:
|
||||
- phase: 09-osint-infrastructure
|
||||
provides: ReconSource interface, LimiterRegistry, Client, BuildQueries, ciLogKeyPattern
|
||||
provides:
|
||||
- VirusTotalSource implementing ReconSource (credential-gated)
|
||||
- IntelligenceXSource implementing ReconSource (credential-gated)
|
||||
- URLhausSource implementing ReconSource (credentialless)
|
||||
affects: [16-osint-wiring, recon-engine-registration]
|
||||
|
||||
tech-stack:
|
||||
added: []
|
||||
patterns: [three-step IX search flow (initiate/results/read), VT x-apikey auth, URLhaus form-encoded POST with tag/payload fallback]
|
||||
|
||||
key-files:
|
||||
created:
|
||||
- pkg/recon/sources/virustotal.go
|
||||
- pkg/recon/sources/virustotal_test.go
|
||||
- pkg/recon/sources/intelligencex.go
|
||||
- pkg/recon/sources/intelligencex_test.go
|
||||
- pkg/recon/sources/urlhaus.go
|
||||
- pkg/recon/sources/urlhaus_test.go
|
||||
modified: []
|
||||
|
||||
key-decisions:
|
||||
- "VT uses x-apikey header per official API v3 spec"
|
||||
- "IX uses three-step flow: POST search, GET results, GET file content per record"
|
||||
- "URLhaus tag lookup with payload endpoint fallback for broader coverage"
|
||||
|
||||
patterns-established:
|
||||
- "Threat intel sources follow same SentrySource pattern with ciLogKeyPattern matching"
|
||||
|
||||
requirements-completed: [RECON-INTEL-01, RECON-INTEL-02, RECON-INTEL-03]
|
||||
|
||||
duration: 4min
|
||||
completed: 2026-04-06
|
||||
---
|
||||
|
||||
# Phase 16 Plan 01: Threat Intelligence Sources Summary
|
||||
|
||||
**VirusTotal, IntelligenceX, and URLhaus recon sources for detecting API keys in malware samples, breach archives, and malicious URL databases**
|
||||
|
||||
## Performance
|
||||
|
||||
- **Duration:** 4 min
|
||||
- **Started:** 2026-04-06T13:43:29Z
|
||||
- **Completed:** 2026-04-06T13:47:29Z
|
||||
- **Tasks:** 2
|
||||
- **Files modified:** 6
|
||||
|
||||
## Accomplishments
|
||||
- VirusTotalSource searches VT Intelligence API for files containing API key patterns (credential-gated, 4 req/min rate limit)
|
||||
- IntelligenceXSource searches IX archive with three-step search/results/content-read flow (credential-gated)
|
||||
- URLhausSource searches abuse.ch API for malicious URLs with embedded keys (credentialless, always enabled)
|
||||
- All three sources use ciLogKeyPattern for consistent content matching across the recon framework
|
||||
|
||||
## Task Commits
|
||||
|
||||
Each task was committed atomically:
|
||||
|
||||
1. **Task 1: VirusTotal and IntelligenceX sources** - `e02bad6` (feat)
|
||||
2. **Task 2: URLhaus source** - `35fa4ad` (feat)
|
||||
|
||||
## Files Created/Modified
|
||||
- `pkg/recon/sources/virustotal.go` - VT Intelligence API search source
|
||||
- `pkg/recon/sources/virustotal_test.go` - httptest mocks for VT (4 tests)
|
||||
- `pkg/recon/sources/intelligencex.go` - IX archive search with three-step flow
|
||||
- `pkg/recon/sources/intelligencex_test.go` - httptest mocks for IX (4 tests)
|
||||
- `pkg/recon/sources/urlhaus.go` - abuse.ch URLhaus tag/payload search
|
||||
- `pkg/recon/sources/urlhaus_test.go` - httptest mocks for URLhaus (4 tests)
|
||||
|
||||
## Decisions Made
|
||||
- VT uses x-apikey header per official API v3 spec
|
||||
- IX uses three-step flow: POST search initiation, GET results list, GET file content per record
|
||||
- URLhaus uses tag lookup endpoint with payload endpoint fallback for broader coverage
|
||||
|
||||
## Deviations from Plan
|
||||
|
||||
None - plan executed exactly as written.
|
||||
|
||||
## Issues Encountered
|
||||
None
|
||||
|
||||
## User Setup Required
|
||||
None - no external service configuration required.
|
||||
|
||||
## Next Phase Readiness
|
||||
- Three threat intel sources ready for wiring into RegisterAll
|
||||
- VT and IX require API keys via config/env; URLhaus works immediately
|
||||
- All sources follow established ReconSource pattern
|
||||
|
||||
---
|
||||
*Phase: 16-osint-threat-intel-mobile-dns-api-marketplaces*
|
||||
*Completed: 2026-04-06*
|
||||
@@ -0,0 +1,159 @@
|
||||
---
|
||||
phase: 16-osint-threat-intel-mobile-dns-api-marketplaces
|
||||
plan: 02
|
||||
type: execute
|
||||
wave: 1
|
||||
depends_on: []
|
||||
files_modified:
|
||||
- pkg/recon/sources/apkmirror.go
|
||||
- pkg/recon/sources/apkmirror_test.go
|
||||
- pkg/recon/sources/crtsh.go
|
||||
- pkg/recon/sources/crtsh_test.go
|
||||
- pkg/recon/sources/securitytrails.go
|
||||
- pkg/recon/sources/securitytrails_test.go
|
||||
autonomous: true
|
||||
requirements: [RECON-MOBILE-01, RECON-DNS-01, RECON-DNS-02]
|
||||
|
||||
must_haves:
|
||||
truths:
|
||||
- "APKMirror source searches for APK metadata containing provider keywords"
|
||||
- "crt.sh source discovers subdomains via CT logs and probes config endpoints for keys"
|
||||
- "SecurityTrails source searches DNS/subdomain data for key exposure indicators"
|
||||
artifacts:
|
||||
- path: "pkg/recon/sources/apkmirror.go"
|
||||
provides: "APKMirrorSource implementing recon.ReconSource"
|
||||
contains: "func (s *APKMirrorSource) Sweep"
|
||||
- path: "pkg/recon/sources/crtsh.go"
|
||||
provides: "CrtShSource implementing recon.ReconSource"
|
||||
contains: "func (s *CrtShSource) Sweep"
|
||||
- path: "pkg/recon/sources/securitytrails.go"
|
||||
provides: "SecurityTrailsSource implementing recon.ReconSource"
|
||||
contains: "func (s *SecurityTrailsSource) Sweep"
|
||||
key_links:
|
||||
- from: "pkg/recon/sources/crtsh.go"
|
||||
to: "pkg/recon/sources/httpclient.go"
|
||||
via: "Client.Do for endpoint probing"
|
||||
pattern: "client\\.Do\\(ctx"
|
||||
- from: "pkg/recon/sources/securitytrails.go"
|
||||
to: "pkg/recon/sources/queries.go"
|
||||
via: "BuildQueries call"
|
||||
pattern: "BuildQueries\\(s\\.Registry"
|
||||
---
|
||||
|
||||
<objective>
|
||||
Implement APKMirror (mobile), crt.sh (CT log DNS), and SecurityTrails (DNS intel) ReconSource modules.
|
||||
|
||||
Purpose: Detect API keys in mobile app metadata, discover subdomains via certificate transparency and probe their config endpoints, and search DNS intelligence for key exposure.
|
||||
Output: Three source files + tests in pkg/recon/sources/
|
||||
</objective>
|
||||
|
||||
<execution_context>
|
||||
@$HOME/.claude/get-shit-done/workflows/execute-plan.md
|
||||
@$HOME/.claude/get-shit-done/templates/summary.md
|
||||
</execution_context>
|
||||
|
||||
<context>
|
||||
@.planning/PROJECT.md
|
||||
@.planning/ROADMAP.md
|
||||
@pkg/recon/source.go
|
||||
@pkg/recon/sources/httpclient.go
|
||||
@pkg/recon/sources/queries.go
|
||||
@pkg/recon/sources/sentry.go
|
||||
@pkg/recon/sources/sentry_test.go
|
||||
</context>
|
||||
|
||||
<interfaces>
|
||||
From pkg/recon/source.go:
|
||||
```go
|
||||
type ReconSource interface {
|
||||
Name() string
|
||||
RateLimit() rate.Limit
|
||||
Burst() int
|
||||
RespectsRobots() bool
|
||||
Enabled(cfg Config) bool
|
||||
Sweep(ctx context.Context, query string, out chan<- Finding) error
|
||||
}
|
||||
```
|
||||
|
||||
From pkg/recon/sources/httpclient.go:
|
||||
```go
|
||||
func NewClient() *Client
|
||||
func (c *Client) Do(ctx context.Context, req *http.Request) (*http.Response, error)
|
||||
```
|
||||
|
||||
From pkg/recon/sources/queries.go:
|
||||
```go
|
||||
func BuildQueries(reg *providers.Registry, source string) []string
|
||||
```
|
||||
</interfaces>
|
||||
|
||||
<tasks>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 1: APKMirror and crt.sh sources</name>
|
||||
<files>pkg/recon/sources/apkmirror.go, pkg/recon/sources/apkmirror_test.go, pkg/recon/sources/crtsh.go, pkg/recon/sources/crtsh_test.go</files>
|
||||
<action>
|
||||
Create APKMirrorSource in apkmirror.go:
|
||||
|
||||
- Struct: APKMirrorSource with BaseURL, Registry (*providers.Registry), Limiters (*recon.LimiterRegistry), Client (*Client) fields. Credentialless.
|
||||
- Name() returns "apkmirror". RateLimit() returns rate.Every(5*time.Second). Burst() returns 2. RespectsRobots() returns true (scraping). Enabled() always true.
|
||||
- Sweep(): Default BaseURL to "https://www.apkmirror.com". Use BuildQueries(s.Registry, "apkmirror"). For each query: GET `{base}/?post_type=app_release&searchtype=apk&s={url-encoded query}`. Parse HTML response — search for APK listing entries. Since we cannot decompile APKs in a network sweep, focus on metadata: search page HTML content for ciLogKeyPattern matches in APK descriptions, changelogs, and file listings. Emit Finding with SourceType "recon:apkmirror", Source as the page URL.
|
||||
- Note: This is a metadata/description scanner, not a full APK decompiler. The decompile capability (apktool/jadx) is noted in RECON-MOBILE-01 but that requires local binary dependencies — the ReconSource focuses on web-searchable APK metadata for keys in descriptions and changelogs.
|
||||
|
||||
Create CrtShSource in crtsh.go:
|
||||
|
||||
- Struct: CrtShSource with BaseURL, Registry (*providers.Registry), Limiters (*recon.LimiterRegistry), Client (*Client) fields. Credentialless.
|
||||
- Name() returns "crtsh". RateLimit() returns rate.Every(3*time.Second). Burst() returns 3. RespectsRobots() false (API). Enabled() always true.
|
||||
- Sweep(): Default BaseURL to "https://crt.sh". The query parameter is used as the target domain. If query is empty, use BuildQueries but for crt.sh the query should be a domain — if query looks like a keyword rather than a domain, skip (return nil). GET `{base}/?q=%25.{domain}&output=json` to find subdomains. Parse JSON array `[{"name_value":"sub.example.com","common_name":"..."}]`. Deduplicate name_value entries. For each unique subdomain (limit 20), probe three config endpoints: `https://{subdomain}/.env`, `https://{subdomain}/api/config`, `https://{subdomain}/actuator/env`. Use a short 5s timeout per probe. For each successful response (200 OK), check body with ciLogKeyPattern. Emit Finding with SourceType "recon:crtsh", Source as the probed URL.
|
||||
- Important: The probe HTTP client should be separate from the crt.sh API client — create a short-timeout *http.Client{Timeout: 5*time.Second} for probing. Do NOT use the retry Client for probes (probes should fail fast, not retry).
|
||||
|
||||
Tests: httptest for both. APKMirror: mock returns HTML with key-like content in description. CrtSh: mock returns JSON subdomain list, mock probe endpoints return .env-like content with key patterns.
|
||||
</action>
|
||||
<verify>
|
||||
<automated>cd /home/salva/Documents/apikey && go test ./pkg/recon/sources/ -run "TestAPKMirror|TestCrtSh" -count=1 -v</automated>
|
||||
</verify>
|
||||
<done>APKMirrorSource scans APK metadata pages, CrtShSource discovers subdomains and probes config endpoints, both emit findings on ciLogKeyPattern match</done>
|
||||
</task>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 2: SecurityTrails source</name>
|
||||
<files>pkg/recon/sources/securitytrails.go, pkg/recon/sources/securitytrails_test.go</files>
|
||||
<action>
|
||||
Create SecurityTrailsSource in securitytrails.go:
|
||||
|
||||
- Struct: SecurityTrailsSource with APIKey, BaseURL, Registry (*providers.Registry), Limiters (*recon.LimiterRegistry), Client (*Client) fields.
|
||||
- Name() returns "securitytrails". RateLimit() returns rate.Every(2*time.Second). Burst() returns 5. RespectsRobots() false. Enabled() returns s.APIKey != "".
|
||||
- Sweep(): Default BaseURL to "https://api.securitytrails.com/v1". The query parameter is used as the target domain. If empty, return nil. Two-phase approach:
|
||||
1. Subdomain enumeration: GET `{base}/domain/{domain}/subdomains?children_only=false` with header `APIKEY: {APIKey}`. Parse `{"subdomains":["www","api","staging",...]}`. Build full FQDNs by appending `.{domain}`.
|
||||
2. For each subdomain (limit 20), probe same three config endpoints as CrtShSource: `/.env`, `/api/config`, `/actuator/env`. Use short-timeout probe client (5s, no retries). Check responses with ciLogKeyPattern. Emit Finding with SourceType "recon:securitytrails".
|
||||
- Also: GET `{base}/domain/{domain}` for the domain's DNS history. Parse response and check the full JSON body with ciLogKeyPattern (DNS TXT records sometimes contain API keys).
|
||||
|
||||
Tests: httptest mock. Test Enabled() with/without API key. Sweep with mock subdomain list and probe endpoints.
|
||||
</action>
|
||||
<verify>
|
||||
<automated>cd /home/salva/Documents/apikey && go test ./pkg/recon/sources/ -run "TestSecurityTrails" -count=1 -v</automated>
|
||||
</verify>
|
||||
<done>SecurityTrailsSource discovers subdomains via API, probes config endpoints, and scans DNS records for key patterns; credential-gated via API key</done>
|
||||
</task>
|
||||
|
||||
</tasks>
|
||||
|
||||
<verification>
|
||||
All three sources compile and pass tests:
|
||||
```bash
|
||||
cd /home/salva/Documents/apikey && go test ./pkg/recon/sources/ -run "TestAPKMirror|TestCrtSh|TestSecurityTrails" -count=1 -v
|
||||
go vet ./pkg/recon/sources/
|
||||
```
|
||||
</verification>
|
||||
|
||||
<success_criteria>
|
||||
- apkmirror.go, crtsh.go, securitytrails.go each implement recon.ReconSource
|
||||
- APKMirror and crt.sh are credentialless (Enabled always true)
|
||||
- SecurityTrails is credential-gated
|
||||
- crt.sh and SecurityTrails both probe /.env, /api/config, /actuator/env on discovered subdomains
|
||||
- All tests pass with httptest mocks
|
||||
</success_criteria>
|
||||
|
||||
<output>
|
||||
After completion, create `.planning/phases/16-osint_threat_intel_mobile_dns_api_marketplaces/16-02-SUMMARY.md`
|
||||
</output>
|
||||
@@ -0,0 +1,85 @@
|
||||
---
|
||||
phase: 16-osint-threat-intel-mobile-dns-api-marketplaces
|
||||
plan: 02
|
||||
subsystem: recon-sources
|
||||
tags: [osint, mobile, dns, ct-logs, securitytrails, apkmirror, crtsh]
|
||||
dependency_graph:
|
||||
requires: [pkg/recon/sources/httpclient.go, pkg/recon/sources/queries.go, pkg/recon/source.go]
|
||||
provides: [APKMirrorSource, CrtShSource, SecurityTrailsSource]
|
||||
affects: [pkg/recon/sources/register.go, cmd/recon.go]
|
||||
tech_stack:
|
||||
added: []
|
||||
patterns: [subdomain-probe-pattern, ct-log-discovery, credential-gated-source]
|
||||
key_files:
|
||||
created:
|
||||
- pkg/recon/sources/apkmirror.go
|
||||
- pkg/recon/sources/apkmirror_test.go
|
||||
- pkg/recon/sources/crtsh.go
|
||||
- pkg/recon/sources/crtsh_test.go
|
||||
- pkg/recon/sources/securitytrails.go
|
||||
- pkg/recon/sources/securitytrails_test.go
|
||||
modified:
|
||||
- pkg/recon/sources/register.go
|
||||
- cmd/recon.go
|
||||
decisions:
|
||||
- APKMirror is metadata-only scanner (no APK decompilation) since apktool/jadx require local binaries
|
||||
- CrtSh and SecurityTrails share configProbeEndpoints pattern for subdomain probing
|
||||
- Probe HTTP client uses 5s timeout without retries (fail fast, separate from API client)
|
||||
- SecurityTrails gets dedicated SECURITYTRAILS_API_KEY env var
|
||||
metrics:
|
||||
duration: 3min
|
||||
completed: 2026-04-06
|
||||
tasks_completed: 2
|
||||
tasks_total: 2
|
||||
files_created: 6
|
||||
files_modified: 2
|
||||
---
|
||||
|
||||
# Phase 16 Plan 02: APKMirror, crt.sh, SecurityTrails Sources Summary
|
||||
|
||||
Mobile app metadata scanning via APKMirror, CT log subdomain discovery with config endpoint probing via crt.sh, and DNS intelligence subdomain enumeration with endpoint probing via SecurityTrails API.
|
||||
|
||||
## Completed Tasks
|
||||
|
||||
| Task | Name | Commit | Key Files |
|
||||
|------|------|--------|-----------|
|
||||
| 1 | APKMirror and crt.sh sources | 09a8d4c | apkmirror.go, crtsh.go + tests |
|
||||
| 2 | SecurityTrails source | a195ef3 | securitytrails.go + test, register.go, cmd/recon.go |
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### APKMirrorSource (credentialless)
|
||||
- Searches APK release pages for keyword matches using BuildQueries
|
||||
- Scans HTML response for ciLogKeyPattern matches in descriptions/changelogs
|
||||
- Rate limited: 1 request per 5 seconds, burst 2. Respects robots.txt.
|
||||
|
||||
### CrtShSource (credentialless)
|
||||
- Queries crt.sh JSON API for certificate transparency log entries matching `%.{domain}`
|
||||
- Deduplicates subdomains (strips wildcards), limits to 20
|
||||
- Probes each subdomain's /.env, /api/config, /actuator/env with 5s timeout client
|
||||
- ProbeBaseURL field enables httptest-based testing
|
||||
|
||||
### SecurityTrailsSource (credential-gated)
|
||||
- Phase 1: Enumerates subdomains via SecurityTrails API with APIKEY header
|
||||
- Phase 2: Probes same three config endpoints as CrtSh (shared configProbeEndpoints)
|
||||
- Phase 3: Fetches domain DNS history and checks full JSON for key patterns in TXT records
|
||||
- Disabled when SECURITYTRAILS_API_KEY is empty
|
||||
|
||||
### RegisterAll
|
||||
- Extended from 67 to 70 sources (added APKMirror, crt.sh, SecurityTrails)
|
||||
- cmd/recon.go wires SecurityTrailsAPIKey from env/viper
|
||||
|
||||
## Deviations from Plan
|
||||
|
||||
None -- plan executed exactly as written.
|
||||
|
||||
## Known Stubs
|
||||
|
||||
None -- all sources fully implemented with real API integration patterns.
|
||||
|
||||
## Verification
|
||||
|
||||
```
|
||||
go vet ./pkg/recon/sources/ ./cmd/ -- PASS
|
||||
go test ./pkg/recon/sources/ -run "TestAPKMirror|TestCrtSh|TestSecurityTrails" -- 14/14 PASS
|
||||
```
|
||||
@@ -0,0 +1,155 @@
|
||||
---
|
||||
phase: 16-osint-threat-intel-mobile-dns-api-marketplaces
|
||||
plan: 03
|
||||
type: execute
|
||||
wave: 1
|
||||
depends_on: []
|
||||
files_modified:
|
||||
- pkg/recon/sources/postman.go
|
||||
- pkg/recon/sources/postman_test.go
|
||||
- pkg/recon/sources/swaggerhub.go
|
||||
- pkg/recon/sources/swaggerhub_test.go
|
||||
- pkg/recon/sources/rapidapi.go
|
||||
- pkg/recon/sources/rapidapi_test.go
|
||||
autonomous: true
|
||||
requirements: [RECON-API-01, RECON-API-02]
|
||||
|
||||
must_haves:
|
||||
truths:
|
||||
- "Postman source searches public collections/workspaces for hardcoded API keys"
|
||||
- "SwaggerHub source searches published API definitions for embedded keys in examples"
|
||||
- "RapidAPI source searches public API listings for exposed credentials"
|
||||
artifacts:
|
||||
- path: "pkg/recon/sources/postman.go"
|
||||
provides: "PostmanSource implementing recon.ReconSource"
|
||||
contains: "func (s *PostmanSource) Sweep"
|
||||
- path: "pkg/recon/sources/swaggerhub.go"
|
||||
provides: "SwaggerHubSource implementing recon.ReconSource"
|
||||
contains: "func (s *SwaggerHubSource) Sweep"
|
||||
- path: "pkg/recon/sources/rapidapi.go"
|
||||
provides: "RapidAPISource implementing recon.ReconSource"
|
||||
contains: "func (s *RapidAPISource) Sweep"
|
||||
key_links:
|
||||
- from: "pkg/recon/sources/postman.go"
|
||||
to: "pkg/recon/sources/queries.go"
|
||||
via: "BuildQueries call"
|
||||
pattern: "BuildQueries\\(s\\.Registry"
|
||||
- from: "pkg/recon/sources/swaggerhub.go"
|
||||
to: "pkg/recon/sources/queries.go"
|
||||
via: "BuildQueries call"
|
||||
pattern: "BuildQueries\\(s\\.Registry"
|
||||
---
|
||||
|
||||
<objective>
|
||||
Implement Postman, SwaggerHub, and RapidAPI ReconSource modules for API marketplace scanning.
|
||||
|
||||
Purpose: Detect API keys hardcoded in public Postman collections, SwaggerHub API definitions, and RapidAPI listings where developers accidentally include real credentials in request examples.
|
||||
Output: Three source files + tests in pkg/recon/sources/
|
||||
</objective>
|
||||
|
||||
<execution_context>
|
||||
@$HOME/.claude/get-shit-done/workflows/execute-plan.md
|
||||
@$HOME/.claude/get-shit-done/templates/summary.md
|
||||
</execution_context>
|
||||
|
||||
<context>
|
||||
@.planning/PROJECT.md
|
||||
@.planning/ROADMAP.md
|
||||
@pkg/recon/source.go
|
||||
@pkg/recon/sources/httpclient.go
|
||||
@pkg/recon/sources/queries.go
|
||||
@pkg/recon/sources/sentry.go
|
||||
@pkg/recon/sources/sentry_test.go
|
||||
</context>
|
||||
|
||||
<interfaces>
|
||||
From pkg/recon/source.go:
|
||||
```go
|
||||
type ReconSource interface {
|
||||
Name() string
|
||||
RateLimit() rate.Limit
|
||||
Burst() int
|
||||
RespectsRobots() bool
|
||||
Enabled(cfg Config) bool
|
||||
Sweep(ctx context.Context, query string, out chan<- Finding) error
|
||||
}
|
||||
```
|
||||
|
||||
From pkg/recon/sources/httpclient.go:
|
||||
```go
|
||||
func NewClient() *Client
|
||||
func (c *Client) Do(ctx context.Context, req *http.Request) (*http.Response, error)
|
||||
```
|
||||
|
||||
From pkg/recon/sources/queries.go:
|
||||
```go
|
||||
func BuildQueries(reg *providers.Registry, source string) []string
|
||||
```
|
||||
</interfaces>
|
||||
|
||||
<tasks>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 1: Postman and SwaggerHub sources</name>
|
||||
<files>pkg/recon/sources/postman.go, pkg/recon/sources/postman_test.go, pkg/recon/sources/swaggerhub.go, pkg/recon/sources/swaggerhub_test.go</files>
|
||||
<action>
|
||||
Create PostmanSource in postman.go:
|
||||
|
||||
- Struct: PostmanSource with BaseURL, Registry (*providers.Registry), Limiters (*recon.LimiterRegistry), Client (*Client) fields. Credentialless — Postman public API search does not require authentication.
|
||||
- Name() returns "postman". RateLimit() returns rate.Every(3*time.Second). Burst() returns 3. RespectsRobots() false. Enabled() always true.
|
||||
- Sweep(): Default BaseURL to "https://www.postman.com/_api". Use BuildQueries(s.Registry, "postman"). For each query: GET `{base}/ws/proxy?request=%2Fsearch%2Fall%3Fquerytext%3D{url-encoded query}%26size%3D10%26type%3Dall` (Postman's internal search proxy). Parse JSON response containing search results with collection/workspace metadata. For each result, fetch the collection detail: GET `{base}/collection/{collection-id}` or use the direct URL from search. Stringify the collection JSON and check with ciLogKeyPattern. Emit Finding with SourceType "recon:postman", Source as `https://www.postman.com/collection/{id}`.
|
||||
- Alternative simpler approach: Use Postman's public network search at `https://www.postman.com/_api/ws/proxy` with the search endpoint. The response contains snippets — check snippets directly with ciLogKeyPattern without fetching full collections (faster, fewer requests).
|
||||
|
||||
Create SwaggerHubSource in swaggerhub.go:
|
||||
|
||||
- Struct: SwaggerHubSource with BaseURL, Registry (*providers.Registry), Limiters (*recon.LimiterRegistry), Client (*Client) fields. Credentialless.
|
||||
- Name() returns "swaggerhub". RateLimit() returns rate.Every(3*time.Second). Burst() returns 3. RespectsRobots() false. Enabled() always true.
|
||||
- Sweep(): Default BaseURL to "https://app.swaggerhub.com/apiproxy/specs". Use BuildQueries(s.Registry, "swaggerhub"). For each query: GET `{base}?specType=ANY&visibility=PUBLIC&query={url-encoded query}&limit=10&page=1`. Parse JSON `{"apis":[{"name":"...","url":"...","description":"...","properties":[{"type":"Swagger","url":"..."}]}]}`. For each API result, fetch the spec URL to get the full OpenAPI/Swagger JSON. Check the spec content with ciLogKeyPattern (keys often appear in example values, server URLs, and security scheme defaults). Emit Finding with SourceType "recon:swaggerhub", Source as the SwaggerHub URL.
|
||||
|
||||
Tests: httptest mocks for both. Postman: mock search returns results with key-like content in snippets. SwaggerHub: mock returns API list, spec fetch returns OpenAPI JSON with embedded key pattern.
|
||||
</action>
|
||||
<verify>
|
||||
<automated>cd /home/salva/Documents/apikey && go test ./pkg/recon/sources/ -run "TestPostman|TestSwaggerHub" -count=1 -v</automated>
|
||||
</verify>
|
||||
<done>PostmanSource searches public collections, SwaggerHubSource searches published API specs, both emit findings on ciLogKeyPattern match in response content</done>
|
||||
</task>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 2: RapidAPI source</name>
|
||||
<files>pkg/recon/sources/rapidapi.go, pkg/recon/sources/rapidapi_test.go</files>
|
||||
<action>
|
||||
Create RapidAPISource in rapidapi.go:
|
||||
|
||||
- Struct: RapidAPISource with BaseURL, Registry (*providers.Registry), Limiters (*recon.LimiterRegistry), Client (*Client) fields. Credentialless.
|
||||
- Name() returns "rapidapi". RateLimit() returns rate.Every(3*time.Second). Burst() returns 3. RespectsRobots() false. Enabled() always true.
|
||||
- Sweep(): Default BaseURL to "https://rapidapi.com". Use BuildQueries(s.Registry, "rapidapi"). For each query: GET `{base}/search/{url-encoded query}?sortBy=ByRelevance&page=1` — RapidAPI's search page. Parse the HTML response body or use the internal JSON API if available. Check content with ciLogKeyPattern. Focus on API listings that include code snippets and example requests where developers may have pasted real API keys. Emit Finding with SourceType "recon:rapidapi", Source as the API listing URL.
|
||||
- Simpler approach: Since RapidAPI's internal search API may not be stable, treat this as a scraping source. GET the search page, read up to 512KB of HTML, and scan with ciLogKeyPattern. This catches keys in code examples, API descriptions, and documentation snippets visible on the public page.
|
||||
|
||||
Tests: httptest mock. Test Name(), Enabled() (always true), Sweep with mock HTML containing key patterns, Sweep with clean HTML returning zero findings.
|
||||
</action>
|
||||
<verify>
|
||||
<automated>cd /home/salva/Documents/apikey && go test ./pkg/recon/sources/ -run "TestRapidAPI" -count=1 -v</automated>
|
||||
</verify>
|
||||
<done>RapidAPISource searches public API listings for key patterns, credentialless, tests pass with httptest mocks</done>
|
||||
</task>
|
||||
|
||||
</tasks>
|
||||
|
||||
<verification>
|
||||
All three API marketplace sources compile and pass tests:
|
||||
```bash
|
||||
cd /home/salva/Documents/apikey && go test ./pkg/recon/sources/ -run "TestPostman|TestSwaggerHub|TestRapidAPI" -count=1 -v
|
||||
go vet ./pkg/recon/sources/
|
||||
```
|
||||
</verification>
|
||||
|
||||
<success_criteria>
|
||||
- postman.go, swaggerhub.go, rapidapi.go each implement recon.ReconSource
|
||||
- All three are credentialless (Enabled always true)
|
||||
- All use BuildQueries + ciLogKeyPattern (consistent with other sources)
|
||||
- Tests pass with httptest mocks
|
||||
</success_criteria>
|
||||
|
||||
<output>
|
||||
After completion, create `.planning/phases/16-osint_threat_intel_mobile_dns_api_marketplaces/16-03-SUMMARY.md`
|
||||
</output>
|
||||
@@ -0,0 +1,59 @@
|
||||
---
|
||||
phase: 16-osint-threat-intel-mobile-dns-api-marketplaces
|
||||
plan: 03
|
||||
subsystem: recon-sources
|
||||
tags: [osint, api-marketplace, postman, swaggerhub, rapidapi, recon]
|
||||
dependency_graph:
|
||||
requires: [recon.ReconSource interface, sources.Client, BuildQueries, ciLogKeyPattern]
|
||||
provides: [PostmanSource, SwaggerHubSource, RapidAPISource]
|
||||
affects: [RegisterAll wiring]
|
||||
tech_stack:
|
||||
added: []
|
||||
patterns: [credentialless API marketplace scanning, HTML scraping for RapidAPI, JSON API for Postman/SwaggerHub]
|
||||
key_files:
|
||||
created:
|
||||
- pkg/recon/sources/postman.go
|
||||
- pkg/recon/sources/postman_test.go
|
||||
- pkg/recon/sources/swaggerhub.go
|
||||
- pkg/recon/sources/swaggerhub_test.go
|
||||
- pkg/recon/sources/rapidapi.go
|
||||
- pkg/recon/sources/rapidapi_test.go
|
||||
modified: []
|
||||
decisions:
|
||||
- All three sources are credentialless -- Postman and SwaggerHub have public APIs, RapidAPI is scraped
|
||||
- RapidAPI uses HTML scraping approach since its internal search API is not stable
|
||||
- SwaggerHub fetches full spec content after search to scan example values for keys
|
||||
metrics:
|
||||
duration: 2min
|
||||
completed: 2026-04-06
|
||||
tasks: 2
|
||||
files: 6
|
||||
---
|
||||
|
||||
# Phase 16 Plan 03: Postman, SwaggerHub, RapidAPI Sources Summary
|
||||
|
||||
API marketplace recon sources scanning public Postman collections, SwaggerHub API specs, and RapidAPI listings for hardcoded API keys in examples and documentation.
|
||||
|
||||
## Task Results
|
||||
|
||||
### Task 1: Postman and SwaggerHub sources
|
||||
- **Commit:** edde02f
|
||||
- **PostmanSource:** Searches via Postman internal search proxy (`/ws/proxy`) for key patterns in collection snippets
|
||||
- **SwaggerHubSource:** Two-phase: search public specs, then fetch each spec and scan for keys in example values, server URLs, security scheme defaults
|
||||
- **Tests:** 8 tests (Name, Enabled, Sweep with match, Sweep empty) for both sources
|
||||
|
||||
### Task 2: RapidAPI source
|
||||
- **Commit:** 297ad3d
|
||||
- **RapidAPISource:** Scrapes public search result pages for key patterns in code examples and descriptions
|
||||
- **Confidence:** Set to "low" (HTML scraping is less precise than JSON API parsing)
|
||||
- **Tests:** 4 tests (Name, Enabled, Sweep with match, Sweep clean HTML)
|
||||
|
||||
## Deviations from Plan
|
||||
|
||||
None -- plan executed exactly as written.
|
||||
|
||||
## Known Stubs
|
||||
|
||||
None. All three sources are fully functional with real API endpoint patterns.
|
||||
|
||||
## Self-Check: PASSED
|
||||
@@ -0,0 +1,199 @@
|
||||
---
|
||||
phase: 16-osint-threat-intel-mobile-dns-api-marketplaces
|
||||
plan: 04
|
||||
type: execute
|
||||
wave: 2
|
||||
depends_on: [16-01, 16-02, 16-03]
|
||||
files_modified:
|
||||
- pkg/recon/sources/register.go
|
||||
- pkg/recon/sources/register_test.go
|
||||
- cmd/recon.go
|
||||
autonomous: true
|
||||
requirements: [RECON-INTEL-01, RECON-INTEL-02, RECON-INTEL-03, RECON-MOBILE-01, RECON-DNS-01, RECON-DNS-02, RECON-API-01, RECON-API-02]
|
||||
|
||||
must_haves:
|
||||
truths:
|
||||
- "RegisterAll registers all 9 Phase 16 sources (76 total)"
|
||||
- "cmd/recon.go populates SourcesConfig with VT, IX, SecurityTrails credentials from env/viper"
|
||||
- "Integration test proves all 76 sources are registered and the 9 new ones are present"
|
||||
artifacts:
|
||||
- path: "pkg/recon/sources/register.go"
|
||||
provides: "RegisterAll with 76 sources (67 + 9 Phase 16)"
|
||||
contains: "VirusTotalSource"
|
||||
- path: "cmd/recon.go"
|
||||
provides: "buildReconEngine with Phase 16 credential wiring"
|
||||
contains: "VirusTotalAPIKey"
|
||||
key_links:
|
||||
- from: "pkg/recon/sources/register.go"
|
||||
to: "pkg/recon/sources/virustotal.go"
|
||||
via: "engine.Register(&VirusTotalSource{...})"
|
||||
pattern: "VirusTotalSource"
|
||||
- from: "cmd/recon.go"
|
||||
to: "pkg/recon/sources/register.go"
|
||||
via: "sources.RegisterAll(e, cfg)"
|
||||
pattern: "sources\\.RegisterAll"
|
||||
---
|
||||
|
||||
<objective>
|
||||
Wire all 9 Phase 16 sources into RegisterAll and cmd/recon.go, bringing total from 67 to 76 sources. Add integration test validating the complete source catalog.
|
||||
|
||||
Purpose: Complete the last OSINT phase by connecting all new sources to the engine so `keyhunter recon list` shows 76 sources and `keyhunter recon full` sweeps them all.
|
||||
Output: Updated register.go, register_test.go, cmd/recon.go
|
||||
</objective>
|
||||
|
||||
<execution_context>
|
||||
@$HOME/.claude/get-shit-done/workflows/execute-plan.md
|
||||
@$HOME/.claude/get-shit-done/templates/summary.md
|
||||
</execution_context>
|
||||
|
||||
<context>
|
||||
@.planning/PROJECT.md
|
||||
@.planning/ROADMAP.md
|
||||
@pkg/recon/sources/register.go
|
||||
@cmd/recon.go
|
||||
</context>
|
||||
|
||||
<interfaces>
|
||||
From pkg/recon/sources/register.go (current):
|
||||
```go
|
||||
type SourcesConfig struct {
|
||||
// ... existing fields through CircleCIToken ...
|
||||
Registry *providers.Registry
|
||||
Limiters *recon.LimiterRegistry
|
||||
}
|
||||
|
||||
func RegisterAll(engine *recon.Engine, cfg SourcesConfig) { ... } // 67 sources
|
||||
```
|
||||
|
||||
From cmd/recon.go (current):
|
||||
```go
|
||||
func buildReconEngine() *recon.Engine {
|
||||
cfg := sources.SourcesConfig{
|
||||
// ... existing credential bindings ...
|
||||
}
|
||||
sources.RegisterAll(e, cfg)
|
||||
}
|
||||
```
|
||||
</interfaces>
|
||||
|
||||
<tasks>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 1: Extend SourcesConfig, RegisterAll, and cmd/recon.go</name>
|
||||
<files>pkg/recon/sources/register.go, cmd/recon.go</files>
|
||||
<action>
|
||||
Add new fields to SourcesConfig in register.go:
|
||||
|
||||
```go
|
||||
// Phase 16: Threat intel, DNS, and API marketplace tokens.
|
||||
VirusTotalAPIKey string
|
||||
IntelligenceXAPIKey string
|
||||
SecurityTrailsAPIKey string
|
||||
```
|
||||
|
||||
Add Phase 16 registrations to RegisterAll, after the Phase 15 block:
|
||||
|
||||
```go
|
||||
// Phase 16: Threat intelligence sources.
|
||||
engine.Register(&VirusTotalSource{
|
||||
APIKey: cfg.VirusTotalAPIKey,
|
||||
Registry: reg,
|
||||
Limiters: lim,
|
||||
})
|
||||
engine.Register(&IntelligenceXSource{
|
||||
APIKey: cfg.IntelligenceXAPIKey,
|
||||
Registry: reg,
|
||||
Limiters: lim,
|
||||
})
|
||||
engine.Register(&URLhausSource{
|
||||
Registry: reg,
|
||||
Limiters: lim,
|
||||
})
|
||||
|
||||
// Phase 16: Mobile and DNS sources.
|
||||
engine.Register(&APKMirrorSource{
|
||||
Registry: reg,
|
||||
Limiters: lim,
|
||||
})
|
||||
engine.Register(&CrtShSource{
|
||||
Registry: reg,
|
||||
Limiters: lim,
|
||||
})
|
||||
engine.Register(&SecurityTrailsSource{
|
||||
APIKey: cfg.SecurityTrailsAPIKey,
|
||||
Registry: reg,
|
||||
Limiters: lim,
|
||||
})
|
||||
|
||||
// Phase 16: API marketplace sources (credentialless).
|
||||
engine.Register(&PostmanSource{
|
||||
Registry: reg,
|
||||
Limiters: lim,
|
||||
})
|
||||
engine.Register(&SwaggerHubSource{
|
||||
Registry: reg,
|
||||
Limiters: lim,
|
||||
})
|
||||
engine.Register(&RapidAPISource{
|
||||
Registry: reg,
|
||||
Limiters: lim,
|
||||
})
|
||||
```
|
||||
|
||||
Update RegisterAll doc comment to say "76 sources total" and mention Phase 16.
|
||||
|
||||
In cmd/recon.go buildReconEngine(), add the three credential fields to the SourcesConfig literal:
|
||||
|
||||
```go
|
||||
VirusTotalAPIKey: firstNonEmpty(os.Getenv("VIRUSTOTAL_API_KEY"), viper.GetString("recon.virustotal.api_key")),
|
||||
IntelligenceXAPIKey: firstNonEmpty(os.Getenv("INTELLIGENCEX_API_KEY"), viper.GetString("recon.intelligencex.api_key")),
|
||||
SecurityTrailsAPIKey: firstNonEmpty(os.Getenv("SECURITYTRAILS_API_KEY"), viper.GetString("recon.securitytrails.api_key")),
|
||||
```
|
||||
</action>
|
||||
<verify>
|
||||
<automated>cd /home/salva/Documents/apikey && go build ./cmd/... && go vet ./pkg/recon/sources/ ./cmd/...</automated>
|
||||
</verify>
|
||||
<done>RegisterAll registers 76 sources, cmd/recon.go wires VT/IX/SecurityTrails credentials from env/viper, project compiles cleanly</done>
|
||||
</task>
|
||||
|
||||
<task type="auto">
|
||||
<name>Task 2: Integration test for 76-source catalog</name>
|
||||
<files>pkg/recon/sources/register_test.go</files>
|
||||
<action>
|
||||
Update or create register_test.go with an integration test that validates:
|
||||
|
||||
1. TestRegisterAll_SourceCount: Create a SourcesConfig with a test Registry (providers.NewRegistryFromProviders with one dummy provider) and a LimiterRegistry. Call RegisterAll on a fresh engine. Assert engine.List() returns exactly 76 names. If count differs, print the actual list for debugging.
|
||||
|
||||
2. TestRegisterAll_Phase16Sources: Assert the following 9 names are present in engine.List(): "virustotal", "intelligencex", "urlhaus", "apkmirror", "crtsh", "securitytrails", "postman", "swaggerhub", "rapidapi".
|
||||
|
||||
3. TestRegisterAll_CredentialGating: Register with empty SourcesConfig (no API keys). For each source via engine.Get(), call Enabled(recon.Config{}). Assert:
|
||||
- virustotal, intelligencex, securitytrails: Enabled == false (credential-gated)
|
||||
- urlhaus, apkmirror, crtsh, postman, swaggerhub, rapidapi: Enabled == true (credentialless)
|
||||
|
||||
Follow the existing test pattern from prior phases. Use testify/assert if already used in the file, otherwise use stdlib testing.
|
||||
</action>
|
||||
<verify>
|
||||
<automated>cd /home/salva/Documents/apikey && go test ./pkg/recon/sources/ -run "TestRegisterAll" -count=1 -v</automated>
|
||||
</verify>
|
||||
<done>Integration test confirms 76 registered sources, all 9 Phase 16 sources present, credential gating correct for VT/IX/SecurityTrails vs credentialless sources</done>
|
||||
</task>
|
||||
|
||||
</tasks>
|
||||
|
||||
<verification>
|
||||
Full build and test:
|
||||
```bash
|
||||
cd /home/salva/Documents/apikey && go build ./cmd/... && go test ./pkg/recon/sources/ -run "TestRegisterAll" -count=1 -v
|
||||
```
|
||||
</verification>
|
||||
|
||||
<success_criteria>
|
||||
- RegisterAll registers 76 sources (67 existing + 9 new)
|
||||
- cmd/recon.go reads VIRUSTOTAL_API_KEY, INTELLIGENCEX_API_KEY, SECURITYTRAILS_API_KEY from env/viper
|
||||
- Integration test passes confirming source count, names, and credential gating
|
||||
- `go build ./cmd/...` succeeds with no errors
|
||||
</success_criteria>
|
||||
|
||||
<output>
|
||||
After completion, create `.planning/phases/16-osint_threat_intel_mobile_dns_api_marketplaces/16-04-SUMMARY.md`
|
||||
</output>
|
||||
@@ -167,7 +167,10 @@ func buildReconEngine() *recon.Engine {
|
||||
FOFAAPIKey: firstNonEmpty(os.Getenv("FOFA_API_KEY"), viper.GetString("recon.fofa.api_key")),
|
||||
NetlasAPIKey: firstNonEmpty(os.Getenv("NETLAS_API_KEY"), viper.GetString("recon.netlas.api_key")),
|
||||
BinaryEdgeAPIKey: firstNonEmpty(os.Getenv("BINARYEDGE_API_KEY"), viper.GetString("recon.binaryedge.api_key")),
|
||||
CircleCIToken: firstNonEmpty(os.Getenv("CIRCLECI_TOKEN"), viper.GetString("recon.circleci.token")),
|
||||
CircleCIToken: firstNonEmpty(os.Getenv("CIRCLECI_TOKEN"), viper.GetString("recon.circleci.token")),
|
||||
VirusTotalAPIKey: firstNonEmpty(os.Getenv("VIRUSTOTAL_API_KEY"), viper.GetString("recon.virustotal.api_key")),
|
||||
IntelligenceXAPIKey: firstNonEmpty(os.Getenv("INTELLIGENCEX_API_KEY"), viper.GetString("recon.intelligencex.api_key")),
|
||||
SecurityTrailsAPIKey: firstNonEmpty(os.Getenv("SECURITYTRAILS_API_KEY"), viper.GetString("recon.securitytrails.api_key")),
|
||||
}
|
||||
sources.RegisterAll(e, cfg)
|
||||
return e
|
||||
|
||||
94
pkg/recon/sources/apkmirror.go
Normal file
94
pkg/recon/sources/apkmirror.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
// APKMirrorSource searches APKMirror for mobile app metadata (descriptions,
|
||||
// changelogs, file listings) that may contain leaked API keys. This is a
|
||||
// metadata scanner -- it does not decompile APKs. Full decompilation via
|
||||
// apktool/jadx would require local binary dependencies and is out of scope
|
||||
// for a network-based ReconSource.
|
||||
type APKMirrorSource struct {
|
||||
BaseURL string
|
||||
Registry *providers.Registry
|
||||
Limiters *recon.LimiterRegistry
|
||||
Client *Client
|
||||
}
|
||||
|
||||
var _ recon.ReconSource = (*APKMirrorSource)(nil)
|
||||
|
||||
func (s *APKMirrorSource) Name() string { return "apkmirror" }
|
||||
func (s *APKMirrorSource) RateLimit() rate.Limit { return rate.Every(5 * time.Second) }
|
||||
func (s *APKMirrorSource) Burst() int { return 2 }
|
||||
func (s *APKMirrorSource) RespectsRobots() bool { return true }
|
||||
func (s *APKMirrorSource) Enabled(_ recon.Config) bool { return true }
|
||||
|
||||
func (s *APKMirrorSource) Sweep(ctx context.Context, query string, out chan<- recon.Finding) error {
|
||||
base := s.BaseURL
|
||||
if base == "" {
|
||||
base = "https://www.apkmirror.com"
|
||||
}
|
||||
client := s.Client
|
||||
if client == nil {
|
||||
client = NewClient()
|
||||
}
|
||||
|
||||
queries := BuildQueries(s.Registry, "apkmirror")
|
||||
if len(queries) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, q := range queries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.Limiters != nil {
|
||||
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
searchURL := fmt.Sprintf(
|
||||
"%s/?post_type=app_release&searchtype=apk&s=%s",
|
||||
base, url.QueryEscape(q),
|
||||
)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
resp, err := client.Do(ctx, req)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if ciLogKeyPattern.Match(body) {
|
||||
out <- recon.Finding{
|
||||
ProviderName: q,
|
||||
Source: searchURL,
|
||||
SourceType: "recon:apkmirror",
|
||||
Confidence: "medium",
|
||||
DetectedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
115
pkg/recon/sources/apkmirror_test.go
Normal file
115
pkg/recon/sources/apkmirror_test.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
func TestAPKMirror_Name(t *testing.T) {
|
||||
s := &APKMirrorSource{}
|
||||
if s.Name() != "apkmirror" {
|
||||
t.Fatalf("expected apkmirror, got %s", s.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPKMirror_Enabled(t *testing.T) {
|
||||
s := &APKMirrorSource{}
|
||||
if !s.Enabled(recon.Config{}) {
|
||||
t.Fatal("APKMirrorSource should always be enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPKMirror_RespectsRobots(t *testing.T) {
|
||||
s := &APKMirrorSource{}
|
||||
if !s.RespectsRobots() {
|
||||
t.Fatal("APKMirrorSource should respect robots.txt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPKMirror_Sweep(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write([]byte(`
|
||||
<html><body>
|
||||
<div class="appRow">
|
||||
<h5 class="appRowTitle">AI Chat Pro</h5>
|
||||
<p>Uses api_key = "sk-proj-ABCDEF1234567890abcdef" for backend</p>
|
||||
</div>
|
||||
</body></html>
|
||||
`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
reg := providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
|
||||
s := &APKMirrorSource{
|
||||
BaseURL: srv.URL,
|
||||
Registry: reg,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) == 0 {
|
||||
t.Fatal("expected at least one finding from APKMirror")
|
||||
}
|
||||
if findings[0].SourceType != "recon:apkmirror" {
|
||||
t.Fatalf("expected recon:apkmirror, got %s", findings[0].SourceType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPKMirror_Sweep_NoMatch(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write([]byte(`<html><body><p>No API keys here</p></body></html>`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
reg := providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
|
||||
s := &APKMirrorSource{
|
||||
BaseURL: srv.URL,
|
||||
Registry: reg,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("expected no findings, got %d", len(findings))
|
||||
}
|
||||
}
|
||||
177
pkg/recon/sources/crtsh.go
Normal file
177
pkg/recon/sources/crtsh.go
Normal file
@@ -0,0 +1,177 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
// CrtShSource discovers subdomains via certificate transparency logs (crt.sh)
|
||||
// and probes their config endpoints (/.env, /api/config, /actuator/env) for
|
||||
// leaked API keys.
|
||||
type CrtShSource struct {
|
||||
BaseURL string
|
||||
Registry *providers.Registry
|
||||
Limiters *recon.LimiterRegistry
|
||||
Client *Client
|
||||
|
||||
// ProbeBaseURL overrides the scheme+host used when probing discovered
|
||||
// subdomains. Tests set this to the httptest server URL.
|
||||
ProbeBaseURL string
|
||||
}
|
||||
|
||||
var _ recon.ReconSource = (*CrtShSource)(nil)
|
||||
|
||||
func (s *CrtShSource) Name() string { return "crtsh" }
|
||||
func (s *CrtShSource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
|
||||
func (s *CrtShSource) Burst() int { return 3 }
|
||||
func (s *CrtShSource) RespectsRobots() bool { return false }
|
||||
func (s *CrtShSource) Enabled(_ recon.Config) bool { return true }
|
||||
|
||||
// crtshEntry represents one row from the crt.sh JSON API.
|
||||
type crtshEntry struct {
|
||||
NameValue string `json:"name_value"`
|
||||
CommonName string `json:"common_name"`
|
||||
}
|
||||
|
||||
// configProbeEndpoints are the well-known config endpoints probed on each
|
||||
// discovered subdomain.
|
||||
var configProbeEndpoints = []string{
|
||||
"/.env",
|
||||
"/api/config",
|
||||
"/actuator/env",
|
||||
}
|
||||
|
||||
func (s *CrtShSource) Sweep(ctx context.Context, query string, out chan<- recon.Finding) error {
|
||||
base := s.BaseURL
|
||||
if base == "" {
|
||||
base = "https://crt.sh"
|
||||
}
|
||||
client := s.Client
|
||||
if client == nil {
|
||||
client = NewClient()
|
||||
}
|
||||
|
||||
// query should be a domain. Skip keyword-like queries (no dots).
|
||||
if query == "" || !strings.Contains(query, ".") {
|
||||
return nil
|
||||
}
|
||||
|
||||
if s.Limiters != nil {
|
||||
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch subdomains from crt.sh.
|
||||
crtURL := fmt.Sprintf("%s/?q=%%25.%s&output=json", base, url.QueryEscape(query))
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, crtURL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := client.Do(ctx, req)
|
||||
if err != nil {
|
||||
return nil // non-fatal: crt.sh may be down
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 1024*1024))
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var entries []crtshEntry
|
||||
if err := json.Unmarshal(data, &entries); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deduplicate name_value entries.
|
||||
seen := make(map[string]struct{})
|
||||
var subdomains []string
|
||||
for _, e := range entries {
|
||||
// name_value can contain multiple names separated by newlines.
|
||||
for _, name := range strings.Split(e.NameValue, "\n") {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
// Remove wildcard prefix.
|
||||
name = strings.TrimPrefix(name, "*.")
|
||||
if _, ok := seen[name]; ok {
|
||||
continue
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
subdomains = append(subdomains, name)
|
||||
if len(subdomains) >= 20 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(subdomains) >= 20 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Probe config endpoints on each subdomain.
|
||||
probeClient := &http.Client{Timeout: 5 * time.Second}
|
||||
for _, sub := range subdomains {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.probeSubdomain(ctx, probeClient, sub, out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeSubdomain checks well-known config endpoints for key patterns.
|
||||
func (s *CrtShSource) probeSubdomain(ctx context.Context, probeClient *http.Client, subdomain string, out chan<- recon.Finding) {
|
||||
for _, ep := range configProbeEndpoints {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var probeURL string
|
||||
if s.ProbeBaseURL != "" {
|
||||
// Test mode: use the mock server URL with subdomain as a header/path hint.
|
||||
probeURL = s.ProbeBaseURL + "/" + subdomain + ep
|
||||
} else {
|
||||
probeURL = "https://" + subdomain + ep
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, probeURL, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
resp, err := probeClient.Do(req)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusOK && ciLogKeyPattern.Match(body) {
|
||||
out <- recon.Finding{
|
||||
ProviderName: subdomain,
|
||||
Source: probeURL,
|
||||
SourceType: "recon:crtsh",
|
||||
Confidence: "high",
|
||||
DetectedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
139
pkg/recon/sources/crtsh_test.go
Normal file
139
pkg/recon/sources/crtsh_test.go
Normal file
@@ -0,0 +1,139 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
func TestCrtSh_Name(t *testing.T) {
|
||||
s := &CrtShSource{}
|
||||
if s.Name() != "crtsh" {
|
||||
t.Fatalf("expected crtsh, got %s", s.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrtSh_Enabled(t *testing.T) {
|
||||
s := &CrtShSource{}
|
||||
if !s.Enabled(recon.Config{}) {
|
||||
t.Fatal("CrtShSource should always be enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrtSh_Sweep_SkipsKeywords(t *testing.T) {
|
||||
s := &CrtShSource{Client: NewClient()}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// "sk-proj-" has no dot -- should be skipped as a keyword.
|
||||
err := s.Sweep(ctx, "sk-proj-", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("expected no findings for keyword query, got %d", len(findings))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrtSh_Sweep(t *testing.T) {
|
||||
// Mux handles both crt.sh API and probe endpoints.
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// crt.sh subdomain lookup.
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Query().Get("output") == "json" {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`[
|
||||
{"name_value":"api.example.com","common_name":"api.example.com"},
|
||||
{"name_value":"staging.example.com","common_name":"staging.example.com"}
|
||||
]`))
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
})
|
||||
|
||||
crtSrv := httptest.NewServer(mux)
|
||||
defer crtSrv.Close()
|
||||
|
||||
// Probe server: serves /.env with key-like content.
|
||||
probeMux := http.NewServeMux()
|
||||
probeMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasSuffix(r.URL.Path, "/.env") {
|
||||
_, _ = w.Write([]byte(`API_KEY = "sk-proj-ABCDEF1234567890abcdef"`))
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
})
|
||||
probeSrv := httptest.NewServer(probeMux)
|
||||
defer probeSrv.Close()
|
||||
|
||||
s := &CrtShSource{
|
||||
BaseURL: crtSrv.URL,
|
||||
Client: NewClient(),
|
||||
ProbeBaseURL: probeSrv.URL,
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 20)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "example.com", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) == 0 {
|
||||
t.Fatal("expected at least one finding from crt.sh probe")
|
||||
}
|
||||
if findings[0].SourceType != "recon:crtsh" {
|
||||
t.Fatalf("expected recon:crtsh, got %s", findings[0].SourceType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrtSh_Sweep_NoSubdomains(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`[]`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
s := &CrtShSource{
|
||||
BaseURL: srv.URL,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "empty.example.com", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("expected no findings, got %d", len(findings))
|
||||
}
|
||||
}
|
||||
202
pkg/recon/sources/intelligencex.go
Normal file
202
pkg/recon/sources/intelligencex.go
Normal file
@@ -0,0 +1,202 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
// IntelligenceXSource searches the IntelligenceX archive for leaked credentials.
|
||||
// IX indexes breached databases, paste sites, and dark web content, making it
|
||||
// a high-value source for discovering leaked API keys.
|
||||
type IntelligenceXSource struct {
|
||||
APIKey string
|
||||
BaseURL string
|
||||
Registry *providers.Registry
|
||||
Limiters *recon.LimiterRegistry
|
||||
Client *Client
|
||||
}
|
||||
|
||||
var _ recon.ReconSource = (*IntelligenceXSource)(nil)
|
||||
|
||||
func (s *IntelligenceXSource) Name() string { return "intelligencex" }
|
||||
func (s *IntelligenceXSource) RateLimit() rate.Limit { return rate.Every(5 * time.Second) }
|
||||
func (s *IntelligenceXSource) Burst() int { return 3 }
|
||||
func (s *IntelligenceXSource) RespectsRobots() bool { return false }
|
||||
func (s *IntelligenceXSource) Enabled(_ recon.Config) bool {
|
||||
return s.APIKey != ""
|
||||
}
|
||||
|
||||
// ixSearchRequest is the JSON body for the IX search endpoint.
|
||||
type ixSearchRequest struct {
|
||||
Term string `json:"term"`
|
||||
MaxResults int `json:"maxresults"`
|
||||
Media int `json:"media"`
|
||||
Timeout int `json:"timeout"`
|
||||
}
|
||||
|
||||
// ixSearchResponse is the response from the IX search initiation endpoint.
|
||||
type ixSearchResponse struct {
|
||||
ID string `json:"id"`
|
||||
Status int `json:"status"`
|
||||
}
|
||||
|
||||
// ixResultResponse is the response from the IX search results endpoint.
|
||||
type ixResultResponse struct {
|
||||
Records []ixRecord `json:"records"`
|
||||
}
|
||||
|
||||
// ixRecord is a single record in the IX search results.
|
||||
type ixRecord struct {
|
||||
SystemID string `json:"systemid"`
|
||||
Name string `json:"name"`
|
||||
StorageID string `json:"storageid"`
|
||||
Bucket string `json:"bucket"`
|
||||
}
|
||||
|
||||
func (s *IntelligenceXSource) Sweep(ctx context.Context, query string, out chan<- recon.Finding) error {
|
||||
base := s.BaseURL
|
||||
if base == "" {
|
||||
base = "https://2.intelx.io"
|
||||
}
|
||||
client := s.Client
|
||||
if client == nil {
|
||||
client = NewClient()
|
||||
}
|
||||
|
||||
queries := BuildQueries(s.Registry, "intelligencex")
|
||||
if len(queries) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, q := range queries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.Limiters != nil {
|
||||
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Step 1: Initiate search.
|
||||
searchBody, _ := json.Marshal(ixSearchRequest{
|
||||
Term: q,
|
||||
MaxResults: 10,
|
||||
Media: 0,
|
||||
Timeout: 5,
|
||||
})
|
||||
|
||||
searchURL := fmt.Sprintf("%s/intelligent/search", base)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, searchURL, bytes.NewReader(searchBody))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("x-key", s.APIKey)
|
||||
|
||||
resp, err := client.Do(ctx, req)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
respData, err := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var searchResp ixSearchResponse
|
||||
if err := json.Unmarshal(respData, &searchResp); err != nil {
|
||||
continue
|
||||
}
|
||||
if searchResp.ID == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Step 2: Fetch results.
|
||||
if s.Limiters != nil {
|
||||
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
resultURL := fmt.Sprintf("%s/intelligent/search/result?id=%s&limit=10", base, searchResp.ID)
|
||||
resReq, err := http.NewRequestWithContext(ctx, http.MethodGet, resultURL, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
resReq.Header.Set("x-key", s.APIKey)
|
||||
|
||||
resResp, err := client.Do(ctx, resReq)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
resData, err := io.ReadAll(io.LimitReader(resResp.Body, 512*1024))
|
||||
_ = resResp.Body.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var resultResp ixResultResponse
|
||||
if err := json.Unmarshal(resData, &resultResp); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Step 3: Fetch content for each record and check for keys.
|
||||
for _, rec := range resultResp.Records {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.Limiters != nil {
|
||||
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
fileURL := fmt.Sprintf(
|
||||
"%s/file/read?type=0&storageid=%s&bucket=%s",
|
||||
base, rec.StorageID, rec.Bucket,
|
||||
)
|
||||
fileReq, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
fileReq.Header.Set("x-key", s.APIKey)
|
||||
|
||||
fileResp, err := client.Do(ctx, fileReq)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
fileData, err := io.ReadAll(io.LimitReader(fileResp.Body, 512*1024))
|
||||
_ = fileResp.Body.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if ciLogKeyPattern.Match(fileData) {
|
||||
out <- recon.Finding{
|
||||
ProviderName: q,
|
||||
Source: fmt.Sprintf("%s/file/read?storageid=%s", base, rec.StorageID),
|
||||
SourceType: "recon:intelligencex",
|
||||
Confidence: "medium",
|
||||
DetectedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
151
pkg/recon/sources/intelligencex_test.go
Normal file
151
pkg/recon/sources/intelligencex_test.go
Normal file
@@ -0,0 +1,151 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
func TestIntelligenceX_Name(t *testing.T) {
|
||||
s := &IntelligenceXSource{}
|
||||
if s.Name() != "intelligencex" {
|
||||
t.Fatalf("expected intelligencex, got %s", s.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntelligenceX_Enabled(t *testing.T) {
|
||||
s := &IntelligenceXSource{}
|
||||
if s.Enabled(recon.Config{}) {
|
||||
t.Fatal("IntelligenceXSource should be disabled without API key")
|
||||
}
|
||||
s.APIKey = "test-key"
|
||||
if !s.Enabled(recon.Config{}) {
|
||||
t.Fatal("IntelligenceXSource should be enabled with API key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntelligenceX_Sweep(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// Search initiation endpoint.
|
||||
mux.HandleFunc("/intelligent/search", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPost {
|
||||
if r.Header.Get("x-key") != "test-key" {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"id":"search-42","status":0}`))
|
||||
return
|
||||
}
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
})
|
||||
|
||||
// Search results endpoint.
|
||||
mux.HandleFunc("/intelligent/search/result", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"records": [{
|
||||
"systemid": "sys-001",
|
||||
"name": "leak.txt",
|
||||
"storageid": "store-001",
|
||||
"bucket": "bucket-a"
|
||||
}]
|
||||
}`))
|
||||
})
|
||||
|
||||
// File read endpoint.
|
||||
mux.HandleFunc("/file/read", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
_, _ = w.Write([]byte(`config:
|
||||
api_key = "sk-proj-ABCDEF1234567890abcdef"
|
||||
secret_key: "super-secret-value-1234567890ab"
|
||||
`))
|
||||
})
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
reg := providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
|
||||
s := &IntelligenceXSource{
|
||||
APIKey: "test-key",
|
||||
BaseURL: srv.URL,
|
||||
Registry: reg,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) == 0 {
|
||||
t.Fatal("expected at least one finding from IntelligenceX")
|
||||
}
|
||||
if findings[0].SourceType != "recon:intelligencex" {
|
||||
t.Fatalf("expected recon:intelligencex, got %s", findings[0].SourceType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntelligenceX_Sweep_Empty(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("/intelligent/search", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"id":"search-empty","status":0}`))
|
||||
})
|
||||
|
||||
mux.HandleFunc("/intelligent/search/result", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"records": []}`))
|
||||
})
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
reg := providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
|
||||
s := &IntelligenceXSource{
|
||||
APIKey: "test-key",
|
||||
BaseURL: srv.URL,
|
||||
Registry: reg,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("expected no findings, got %d", len(findings))
|
||||
}
|
||||
}
|
||||
98
pkg/recon/sources/postman.go
Normal file
98
pkg/recon/sources/postman.go
Normal file
@@ -0,0 +1,98 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
// PostmanSource searches public Postman collections and workspaces for
|
||||
// hardcoded API keys. The Postman public network exposes a search proxy
|
||||
// that does not require authentication.
|
||||
type PostmanSource struct {
|
||||
BaseURL string
|
||||
Registry *providers.Registry
|
||||
Limiters *recon.LimiterRegistry
|
||||
Client *Client
|
||||
}
|
||||
|
||||
var _ recon.ReconSource = (*PostmanSource)(nil)
|
||||
|
||||
func (s *PostmanSource) Name() string { return "postman" }
|
||||
func (s *PostmanSource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
|
||||
func (s *PostmanSource) Burst() int { return 3 }
|
||||
func (s *PostmanSource) RespectsRobots() bool { return false }
|
||||
func (s *PostmanSource) Enabled(_ recon.Config) bool { return true }
|
||||
|
||||
func (s *PostmanSource) Sweep(ctx context.Context, query string, out chan<- recon.Finding) error {
|
||||
base := s.BaseURL
|
||||
if base == "" {
|
||||
base = "https://www.postman.com/_api"
|
||||
}
|
||||
client := s.Client
|
||||
if client == nil {
|
||||
client = NewClient()
|
||||
}
|
||||
|
||||
queries := BuildQueries(s.Registry, "postman")
|
||||
if len(queries) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, q := range queries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.Limiters != nil {
|
||||
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Use Postman's internal search proxy. The encoded request parameter
|
||||
// targets /search/all with the query text.
|
||||
searchPath := fmt.Sprintf("/search/all?querytext=%s&size=10&type=all",
|
||||
url.QueryEscape(q))
|
||||
searchURL := fmt.Sprintf("%s/ws/proxy?request=%s",
|
||||
base, url.QueryEscape(searchPath))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
resp, err := client.Do(ctx, req)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Scan the raw response body for key patterns. Postman search results
|
||||
// include snippets of collection contents where keys may appear.
|
||||
content := string(data)
|
||||
if ciLogKeyPattern.MatchString(content) {
|
||||
out <- recon.Finding{
|
||||
ProviderName: q,
|
||||
Source: fmt.Sprintf("https://www.postman.com/search?q=%s", url.QueryEscape(q)),
|
||||
SourceType: "recon:postman",
|
||||
Confidence: "medium",
|
||||
DetectedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
115
pkg/recon/sources/postman_test.go
Normal file
115
pkg/recon/sources/postman_test.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
func TestPostman_Name(t *testing.T) {
|
||||
s := &PostmanSource{}
|
||||
if s.Name() != "postman" {
|
||||
t.Fatalf("expected postman, got %s", s.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostman_Enabled(t *testing.T) {
|
||||
s := &PostmanSource{}
|
||||
if !s.Enabled(recon.Config{}) {
|
||||
t.Fatal("PostmanSource should always be enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostman_Sweep(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/ws/proxy", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"data": [
|
||||
{
|
||||
"id": "coll-001",
|
||||
"name": "My API Collection",
|
||||
"summary": "api_key = 'sk-proj-ABCDEF1234567890abcdef'"
|
||||
}
|
||||
]
|
||||
}`))
|
||||
})
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
reg := providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
|
||||
s := &PostmanSource{
|
||||
BaseURL: srv.URL,
|
||||
Registry: reg,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) == 0 {
|
||||
t.Fatal("expected at least one finding from Postman")
|
||||
}
|
||||
if findings[0].SourceType != "recon:postman" {
|
||||
t.Fatalf("expected recon:postman, got %s", findings[0].SourceType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostman_Sweep_NoResults(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/ws/proxy", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data": []}`))
|
||||
})
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
reg := providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
|
||||
s := &PostmanSource{
|
||||
BaseURL: srv.URL,
|
||||
Registry: reg,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("expected no findings, got %d", len(findings))
|
||||
}
|
||||
}
|
||||
95
pkg/recon/sources/rapidapi.go
Normal file
95
pkg/recon/sources/rapidapi.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
// RapidAPISource searches public RapidAPI listings for exposed API keys.
|
||||
// API listings often include code snippets and example requests where
|
||||
// developers may accidentally paste real credentials. Credentialless.
|
||||
type RapidAPISource struct {
|
||||
BaseURL string
|
||||
Registry *providers.Registry
|
||||
Limiters *recon.LimiterRegistry
|
||||
Client *Client
|
||||
}
|
||||
|
||||
var _ recon.ReconSource = (*RapidAPISource)(nil)
|
||||
|
||||
func (s *RapidAPISource) Name() string { return "rapidapi" }
|
||||
func (s *RapidAPISource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
|
||||
func (s *RapidAPISource) Burst() int { return 3 }
|
||||
func (s *RapidAPISource) RespectsRobots() bool { return false }
|
||||
func (s *RapidAPISource) Enabled(_ recon.Config) bool { return true }
|
||||
|
||||
func (s *RapidAPISource) Sweep(ctx context.Context, query string, out chan<- recon.Finding) error {
|
||||
base := s.BaseURL
|
||||
if base == "" {
|
||||
base = "https://rapidapi.com"
|
||||
}
|
||||
client := s.Client
|
||||
if client == nil {
|
||||
client = NewClient()
|
||||
}
|
||||
|
||||
queries := BuildQueries(s.Registry, "rapidapi")
|
||||
if len(queries) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, q := range queries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.Limiters != nil {
|
||||
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Search RapidAPI public listings. The search page renders HTML with
|
||||
// code examples and descriptions that may contain leaked keys.
|
||||
searchURL := fmt.Sprintf(
|
||||
"%s/search/%s?sortBy=ByRelevance&page=1",
|
||||
base, url.PathEscape(q),
|
||||
)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
resp, err := client.Do(ctx, req)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if ciLogKeyPattern.Match(data) {
|
||||
out <- recon.Finding{
|
||||
ProviderName: q,
|
||||
Source: fmt.Sprintf("https://rapidapi.com/search/%s", url.PathEscape(q)),
|
||||
SourceType: "recon:rapidapi",
|
||||
Confidence: "low",
|
||||
DetectedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
119
pkg/recon/sources/rapidapi_test.go
Normal file
119
pkg/recon/sources/rapidapi_test.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
func TestRapidAPI_Name(t *testing.T) {
|
||||
s := &RapidAPISource{}
|
||||
if s.Name() != "rapidapi" {
|
||||
t.Fatalf("expected rapidapi, got %s", s.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRapidAPI_Enabled(t *testing.T) {
|
||||
s := &RapidAPISource{}
|
||||
if !s.Enabled(recon.Config{}) {
|
||||
t.Fatal("RapidAPISource should always be enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRapidAPI_Sweep(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/search/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write([]byte(`<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<div class="api-listing">
|
||||
<h2>OpenAI Helper API</h2>
|
||||
<pre><code>
|
||||
curl -H "Authorization: Bearer sk-proj-ABCDEF1234567890abcdef" https://api.example.com
|
||||
api_key = "sk-proj-ABCDEF1234567890abcdef"
|
||||
</code></pre>
|
||||
</div>
|
||||
</body>
|
||||
</html>`))
|
||||
})
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
reg := providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
|
||||
s := &RapidAPISource{
|
||||
BaseURL: srv.URL,
|
||||
Registry: reg,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) == 0 {
|
||||
t.Fatal("expected at least one finding from RapidAPI")
|
||||
}
|
||||
if findings[0].SourceType != "recon:rapidapi" {
|
||||
t.Fatalf("expected recon:rapidapi, got %s", findings[0].SourceType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRapidAPI_Sweep_Clean(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/search/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write([]byte(`<!DOCTYPE html>
|
||||
<html><body><p>No results found</p></body></html>`))
|
||||
})
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
reg := providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
|
||||
s := &RapidAPISource{
|
||||
BaseURL: srv.URL,
|
||||
Registry: reg,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("expected no findings, got %d", len(findings))
|
||||
}
|
||||
}
|
||||
@@ -52,6 +52,11 @@ type SourcesConfig struct {
|
||||
// Phase 14: CI/CD source tokens.
|
||||
CircleCIToken string
|
||||
|
||||
// Phase 16: Threat intel + DNS API keys.
|
||||
VirusTotalAPIKey string
|
||||
IntelligenceXAPIKey string
|
||||
SecurityTrailsAPIKey string
|
||||
|
||||
// Registry drives query generation for every source via BuildQueries.
|
||||
Registry *providers.Registry
|
||||
// Limiters is the shared per-source rate-limiter registry.
|
||||
@@ -61,8 +66,8 @@ type SourcesConfig struct {
|
||||
// RegisterAll registers every Phase 10 code-hosting, Phase 11 search engine /
|
||||
// paste site, Phase 12 IoT scanner / cloud storage, Phase 13 package
|
||||
// registry / container / IaC, Phase 14 CI/CD log / web archive / frontend
|
||||
// leak, and Phase 15 forum / collaboration tool / log aggregator source on
|
||||
// engine (67 sources total).
|
||||
// leak, Phase 15 forum / collaboration tool / log aggregator, and Phase 16
|
||||
// mobile / DNS / threat intel source on engine (70 sources total).
|
||||
//
|
||||
// All sources are registered unconditionally so that cmd/recon.go can surface
|
||||
// the full catalog via `keyhunter recon list` regardless of which credentials
|
||||
@@ -282,4 +287,31 @@ func RegisterAll(engine *recon.Engine, cfg SourcesConfig) {
|
||||
engine.Register(&SplunkSource{Registry: reg, Limiters: lim})
|
||||
engine.Register(&GrafanaSource{Registry: reg, Limiters: lim})
|
||||
engine.Register(&SentrySource{Registry: reg, Limiters: lim})
|
||||
|
||||
// Phase 16: Mobile, DNS, and threat intel sources.
|
||||
engine.Register(&APKMirrorSource{Registry: reg, Limiters: lim})
|
||||
engine.Register(&CrtShSource{Registry: reg, Limiters: lim})
|
||||
engine.Register(&SecurityTrailsSource{
|
||||
APIKey: cfg.SecurityTrailsAPIKey,
|
||||
Registry: reg,
|
||||
Limiters: lim,
|
||||
})
|
||||
|
||||
// Phase 16: Threat intelligence sources.
|
||||
engine.Register(&VirusTotalSource{
|
||||
APIKey: cfg.VirusTotalAPIKey,
|
||||
Registry: reg,
|
||||
Limiters: lim,
|
||||
})
|
||||
engine.Register(&IntelligenceXSource{
|
||||
APIKey: cfg.IntelligenceXAPIKey,
|
||||
Registry: reg,
|
||||
Limiters: lim,
|
||||
})
|
||||
engine.Register(&URLhausSource{Registry: reg, Limiters: lim})
|
||||
|
||||
// Phase 16: API marketplace sources (credentialless).
|
||||
engine.Register(&PostmanSource{Registry: reg, Limiters: lim})
|
||||
engine.Register(&SwaggerHubSource{Registry: reg, Limiters: lim})
|
||||
engine.Register(&RapidAPISource{Registry: reg, Limiters: lim})
|
||||
}
|
||||
|
||||
189
pkg/recon/sources/securitytrails.go
Normal file
189
pkg/recon/sources/securitytrails.go
Normal file
@@ -0,0 +1,189 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
// SecurityTrailsSource searches SecurityTrails DNS/subdomain data for API key
|
||||
// exposure. It enumerates subdomains for a target domain and probes config
|
||||
// endpoints, and also checks DNS history records (TXT records may contain keys).
|
||||
type SecurityTrailsSource struct {
|
||||
APIKey string
|
||||
BaseURL string
|
||||
Registry *providers.Registry
|
||||
Limiters *recon.LimiterRegistry
|
||||
Client *Client
|
||||
|
||||
// ProbeBaseURL overrides the scheme+host used when probing discovered
|
||||
// subdomains. Tests set this to the httptest server URL.
|
||||
ProbeBaseURL string
|
||||
}
|
||||
|
||||
var _ recon.ReconSource = (*SecurityTrailsSource)(nil)
|
||||
|
||||
func (s *SecurityTrailsSource) Name() string { return "securitytrails" }
|
||||
func (s *SecurityTrailsSource) RateLimit() rate.Limit { return rate.Every(2 * time.Second) }
|
||||
func (s *SecurityTrailsSource) Burst() int { return 5 }
|
||||
func (s *SecurityTrailsSource) RespectsRobots() bool { return false }
|
||||
|
||||
func (s *SecurityTrailsSource) Enabled(_ recon.Config) bool {
|
||||
return s.APIKey != ""
|
||||
}
|
||||
|
||||
// securityTrailsSubdomains represents the subdomain listing API response.
|
||||
type securityTrailsSubdomains struct {
|
||||
Subdomains []string `json:"subdomains"`
|
||||
}
|
||||
|
||||
func (s *SecurityTrailsSource) Sweep(ctx context.Context, query string, out chan<- recon.Finding) error {
|
||||
base := s.BaseURL
|
||||
if base == "" {
|
||||
base = "https://api.securitytrails.com/v1"
|
||||
}
|
||||
client := s.Client
|
||||
if client == nil {
|
||||
client = NewClient()
|
||||
}
|
||||
|
||||
if query == "" || !strings.Contains(query, ".") {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Phase 1: Enumerate subdomains.
|
||||
if s.Limiters != nil {
|
||||
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
subURL := fmt.Sprintf("%s/domain/%s/subdomains?children_only=false", base, query)
|
||||
subReq, err := http.NewRequestWithContext(ctx, http.MethodGet, subURL, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subReq.Header.Set("APIKEY", s.APIKey)
|
||||
|
||||
subResp, err := client.Do(ctx, subReq)
|
||||
if err != nil {
|
||||
return nil // non-fatal
|
||||
}
|
||||
|
||||
subData, err := io.ReadAll(io.LimitReader(subResp.Body, 512*1024))
|
||||
_ = subResp.Body.Close()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var subResult securityTrailsSubdomains
|
||||
if err := json.Unmarshal(subData, &subResult); err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Build FQDNs and limit to 20.
|
||||
var fqdns []string
|
||||
for _, sub := range subResult.Subdomains {
|
||||
fqdns = append(fqdns, sub+"."+query)
|
||||
if len(fqdns) >= 20 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Probe config endpoints on each subdomain.
|
||||
probeClient := &http.Client{Timeout: 5 * time.Second}
|
||||
for _, fqdn := range fqdns {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.probeSubdomain(ctx, probeClient, fqdn, out)
|
||||
}
|
||||
|
||||
// Phase 2: Check DNS history for key patterns in TXT records.
|
||||
if s.Limiters != nil {
|
||||
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
dnsURL := fmt.Sprintf("%s/domain/%s", base, query)
|
||||
dnsReq, err := http.NewRequestWithContext(ctx, http.MethodGet, dnsURL, nil)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
dnsReq.Header.Set("APIKEY", s.APIKey)
|
||||
|
||||
dnsResp, err := client.Do(ctx, dnsReq)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
dnsData, err := io.ReadAll(io.LimitReader(dnsResp.Body, 512*1024))
|
||||
_ = dnsResp.Body.Close()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if ciLogKeyPattern.Match(dnsData) {
|
||||
out <- recon.Finding{
|
||||
ProviderName: query,
|
||||
Source: dnsURL,
|
||||
SourceType: "recon:securitytrails",
|
||||
Confidence: "medium",
|
||||
DetectedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeSubdomain checks well-known config endpoints for key patterns.
|
||||
func (s *SecurityTrailsSource) probeSubdomain(ctx context.Context, probeClient *http.Client, subdomain string, out chan<- recon.Finding) {
|
||||
for _, ep := range configProbeEndpoints {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var probeURL string
|
||||
if s.ProbeBaseURL != "" {
|
||||
probeURL = s.ProbeBaseURL + "/" + subdomain + ep
|
||||
} else {
|
||||
probeURL = "https://" + subdomain + ep
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, probeURL, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
resp, err := probeClient.Do(req)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 64*1024))
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusOK && ciLogKeyPattern.Match(body) {
|
||||
out <- recon.Finding{
|
||||
ProviderName: subdomain,
|
||||
Source: probeURL,
|
||||
SourceType: "recon:securitytrails",
|
||||
Confidence: "high",
|
||||
DetectedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
180
pkg/recon/sources/securitytrails_test.go
Normal file
180
pkg/recon/sources/securitytrails_test.go
Normal file
@@ -0,0 +1,180 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
func TestSecurityTrails_Name(t *testing.T) {
|
||||
s := &SecurityTrailsSource{}
|
||||
if s.Name() != "securitytrails" {
|
||||
t.Fatalf("expected securitytrails, got %s", s.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecurityTrails_Enabled(t *testing.T) {
|
||||
s := &SecurityTrailsSource{}
|
||||
if s.Enabled(recon.Config{}) {
|
||||
t.Fatal("SecurityTrailsSource should be disabled without API key")
|
||||
}
|
||||
|
||||
s.APIKey = "test-key"
|
||||
if !s.Enabled(recon.Config{}) {
|
||||
t.Fatal("SecurityTrailsSource should be enabled with API key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecurityTrails_Sweep(t *testing.T) {
|
||||
// API server mocks SecurityTrails endpoints.
|
||||
apiMux := http.NewServeMux()
|
||||
|
||||
// Subdomain enumeration.
|
||||
apiMux.HandleFunc("/domain/example.com/subdomains", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("APIKEY") != "test-key" {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"subdomains":["api","staging"]}`))
|
||||
})
|
||||
|
||||
// DNS history.
|
||||
apiMux.HandleFunc("/domain/example.com", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("APIKEY") != "test-key" {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"current_dns":{"txt":{"values":[{"value":"token = sk-proj-ABCDEF1234567890abcdef"}]}}}`))
|
||||
})
|
||||
|
||||
apiSrv := httptest.NewServer(apiMux)
|
||||
defer apiSrv.Close()
|
||||
|
||||
// Probe server.
|
||||
probeMux := http.NewServeMux()
|
||||
probeMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasSuffix(r.URL.Path, "/.env") {
|
||||
_, _ = w.Write([]byte(`SECRET_KEY = "sk-proj-ABCDEF1234567890abcdef"`))
|
||||
return
|
||||
}
|
||||
http.NotFound(w, r)
|
||||
})
|
||||
probeSrv := httptest.NewServer(probeMux)
|
||||
defer probeSrv.Close()
|
||||
|
||||
s := &SecurityTrailsSource{
|
||||
APIKey: "test-key",
|
||||
BaseURL: apiSrv.URL,
|
||||
Client: NewClient(),
|
||||
ProbeBaseURL: probeSrv.URL,
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 20)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "example.com", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) == 0 {
|
||||
t.Fatal("expected at least one finding from SecurityTrails")
|
||||
}
|
||||
|
||||
// Check that we got both probe findings and DNS history findings.
|
||||
var probeFound, dnsFound bool
|
||||
for _, f := range findings {
|
||||
if f.SourceType != "recon:securitytrails" {
|
||||
t.Fatalf("expected recon:securitytrails, got %s", f.SourceType)
|
||||
}
|
||||
if strings.Contains(f.Source, "/.env") {
|
||||
probeFound = true
|
||||
}
|
||||
if strings.Contains(f.Source, "/domain/example.com") && !strings.Contains(f.Source, "subdomains") {
|
||||
dnsFound = true
|
||||
}
|
||||
}
|
||||
if !probeFound {
|
||||
t.Fatal("expected probe finding from SecurityTrails")
|
||||
}
|
||||
if !dnsFound {
|
||||
t.Fatal("expected DNS history finding from SecurityTrails")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecurityTrails_Sweep_SkipsKeywords(t *testing.T) {
|
||||
s := &SecurityTrailsSource{
|
||||
APIKey: "test-key",
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "sk-proj-", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("expected no findings for keyword query, got %d", len(findings))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecurityTrails_Sweep_NoSubdomains(t *testing.T) {
|
||||
apiMux := http.NewServeMux()
|
||||
apiMux.HandleFunc("/domain/empty.example.com/subdomains", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"subdomains":[]}`))
|
||||
})
|
||||
apiMux.HandleFunc("/domain/empty.example.com", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"current_dns":{}}`))
|
||||
})
|
||||
|
||||
apiSrv := httptest.NewServer(apiMux)
|
||||
defer apiSrv.Close()
|
||||
|
||||
s := &SecurityTrailsSource{
|
||||
APIKey: "test-key",
|
||||
BaseURL: apiSrv.URL,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "empty.example.com", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("expected no findings, got %d", len(findings))
|
||||
}
|
||||
}
|
||||
158
pkg/recon/sources/swaggerhub.go
Normal file
158
pkg/recon/sources/swaggerhub.go
Normal file
@@ -0,0 +1,158 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
// SwaggerHubSource searches published API definitions on SwaggerHub for
|
||||
// embedded API keys in example values, server URLs, and security scheme
|
||||
// defaults. The SwaggerHub specs API is publicly accessible.
|
||||
type SwaggerHubSource struct {
|
||||
BaseURL string
|
||||
Registry *providers.Registry
|
||||
Limiters *recon.LimiterRegistry
|
||||
Client *Client
|
||||
}
|
||||
|
||||
var _ recon.ReconSource = (*SwaggerHubSource)(nil)
|
||||
|
||||
func (s *SwaggerHubSource) Name() string { return "swaggerhub" }
|
||||
func (s *SwaggerHubSource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
|
||||
func (s *SwaggerHubSource) Burst() int { return 3 }
|
||||
func (s *SwaggerHubSource) RespectsRobots() bool { return false }
|
||||
func (s *SwaggerHubSource) Enabled(_ recon.Config) bool { return true }
|
||||
|
||||
// swaggerHubSearchResult represents a single API from the search response.
|
||||
type swaggerHubSearchResult struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
Properties []struct {
|
||||
Type string `json:"type"`
|
||||
URL string `json:"url"`
|
||||
} `json:"properties"`
|
||||
}
|
||||
|
||||
// swaggerHubSearchResponse is the top-level search response from SwaggerHub.
|
||||
type swaggerHubSearchResponse struct {
|
||||
APIs []swaggerHubSearchResult `json:"apis"`
|
||||
}
|
||||
|
||||
func (s *SwaggerHubSource) Sweep(ctx context.Context, query string, out chan<- recon.Finding) error {
|
||||
base := s.BaseURL
|
||||
if base == "" {
|
||||
base = "https://app.swaggerhub.com/apiproxy/specs"
|
||||
}
|
||||
client := s.Client
|
||||
if client == nil {
|
||||
client = NewClient()
|
||||
}
|
||||
|
||||
queries := BuildQueries(s.Registry, "swaggerhub")
|
||||
if len(queries) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, q := range queries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.Limiters != nil {
|
||||
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Search public API specs.
|
||||
searchURL := fmt.Sprintf(
|
||||
"%s?specType=ANY&visibility=PUBLIC&query=%s&limit=10&page=1",
|
||||
base, url.QueryEscape(q),
|
||||
)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
resp, err := client.Do(ctx, req)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var sr swaggerHubSearchResponse
|
||||
if err := json.Unmarshal(data, &sr); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Fetch each spec and scan for key patterns.
|
||||
for _, api := range sr.APIs {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
specURL := api.URL
|
||||
if specURL == "" {
|
||||
// Fall back to the first property URL with type "Swagger" or "X-URL".
|
||||
for _, p := range api.Properties {
|
||||
if p.URL != "" {
|
||||
specURL = p.URL
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if specURL == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if s.Limiters != nil {
|
||||
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
specReq, err := http.NewRequestWithContext(ctx, http.MethodGet, specURL, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
specResp, err := client.Do(ctx, specReq)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
specData, err := io.ReadAll(io.LimitReader(specResp.Body, 512*1024))
|
||||
_ = specResp.Body.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if ciLogKeyPattern.Match(specData) {
|
||||
out <- recon.Finding{
|
||||
ProviderName: q,
|
||||
Source: specURL,
|
||||
SourceType: "recon:swaggerhub",
|
||||
Confidence: "medium",
|
||||
DetectedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
182
pkg/recon/sources/swaggerhub_test.go
Normal file
182
pkg/recon/sources/swaggerhub_test.go
Normal file
@@ -0,0 +1,182 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
func TestSwaggerHub_Name(t *testing.T) {
|
||||
s := &SwaggerHubSource{}
|
||||
if s.Name() != "swaggerhub" {
|
||||
t.Fatalf("expected swaggerhub, got %s", s.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSwaggerHub_Enabled(t *testing.T) {
|
||||
s := &SwaggerHubSource{}
|
||||
if !s.Enabled(recon.Config{}) {
|
||||
t.Fatal("SwaggerHubSource should always be enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSwaggerHub_Sweep(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// Search endpoint returns one API with a spec URL.
|
||||
mux.HandleFunc("/specs", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"apis": [
|
||||
{
|
||||
"name": "Payment Gateway",
|
||||
"url": "",
|
||||
"properties": [
|
||||
{"type": "Swagger", "url": "SPEC_URL_PLACEHOLDER"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}`))
|
||||
})
|
||||
|
||||
// Spec endpoint returns OpenAPI JSON with an embedded key.
|
||||
mux.HandleFunc("/spec/payment-gateway", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"openapi": "3.0.0",
|
||||
"info": {"title": "Payment API"},
|
||||
"paths": {
|
||||
"/charge": {
|
||||
"post": {
|
||||
"parameters": [
|
||||
{
|
||||
"name": "Authorization",
|
||||
"in": "header",
|
||||
"example": "api_key = 'sk-proj-ABCDEF1234567890abcdef'"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}`))
|
||||
})
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
// Patch the spec URL placeholder with the test server URL.
|
||||
origHandler := mux
|
||||
_ = origHandler // keep for reference
|
||||
|
||||
// Re-create with the actual server URL known.
|
||||
mux2 := http.NewServeMux()
|
||||
mux2.HandleFunc("/specs", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"apis": [
|
||||
{
|
||||
"name": "Payment Gateway",
|
||||
"url": "` + srv.URL + `/spec/payment-gateway",
|
||||
"properties": []
|
||||
}
|
||||
]
|
||||
}`))
|
||||
})
|
||||
mux2.HandleFunc("/spec/payment-gateway", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"openapi": "3.0.0",
|
||||
"paths": {
|
||||
"/charge": {
|
||||
"post": {
|
||||
"parameters": [
|
||||
{
|
||||
"name": "Authorization",
|
||||
"in": "header",
|
||||
"example": "api_key = 'sk-proj-ABCDEF1234567890abcdef'"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}`))
|
||||
})
|
||||
|
||||
// Replace the handler on the existing server.
|
||||
srv.Config.Handler = mux2
|
||||
|
||||
reg := providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
|
||||
s := &SwaggerHubSource{
|
||||
BaseURL: srv.URL + "/specs",
|
||||
Registry: reg,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) == 0 {
|
||||
t.Fatal("expected at least one finding from SwaggerHub")
|
||||
}
|
||||
if findings[0].SourceType != "recon:swaggerhub" {
|
||||
t.Fatalf("expected recon:swaggerhub, got %s", findings[0].SourceType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSwaggerHub_Sweep_NoAPIs(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/specs", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"apis": []}`))
|
||||
})
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
reg := providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
|
||||
s := &SwaggerHubSource{
|
||||
BaseURL: srv.URL + "/specs",
|
||||
Registry: reg,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("expected no findings, got %d", len(findings))
|
||||
}
|
||||
}
|
||||
152
pkg/recon/sources/urlhaus.go
Normal file
152
pkg/recon/sources/urlhaus.go
Normal file
@@ -0,0 +1,152 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
// URLhausSource searches the abuse.ch URLhaus API for malicious URLs that
|
||||
// contain API key patterns. Threat actors often embed stolen API keys in
|
||||
// malware C2 URLs, phishing pages, and credential-harvesting infrastructure.
|
||||
// URLhaus is free and unauthenticated — no API key required.
|
||||
type URLhausSource struct {
|
||||
BaseURL string
|
||||
Registry *providers.Registry
|
||||
Limiters *recon.LimiterRegistry
|
||||
Client *Client
|
||||
}
|
||||
|
||||
var _ recon.ReconSource = (*URLhausSource)(nil)
|
||||
|
||||
func (s *URLhausSource) Name() string { return "urlhaus" }
|
||||
func (s *URLhausSource) RateLimit() rate.Limit { return rate.Every(3 * time.Second) }
|
||||
func (s *URLhausSource) Burst() int { return 2 }
|
||||
func (s *URLhausSource) RespectsRobots() bool { return false }
|
||||
func (s *URLhausSource) Enabled(_ recon.Config) bool { return true }
|
||||
|
||||
// urlhausResponse represents the URLhaus API response for tag/payload lookups.
|
||||
type urlhausResponse struct {
|
||||
QueryStatus string `json:"query_status"`
|
||||
URLs []urlhausEntry `json:"urls"`
|
||||
}
|
||||
|
||||
// urlhausEntry is a single URL record from URLhaus.
|
||||
type urlhausEntry struct {
|
||||
URL string `json:"url"`
|
||||
URLStatus string `json:"url_status"`
|
||||
Tags []string `json:"tags"`
|
||||
Reporter string `json:"reporter"`
|
||||
}
|
||||
|
||||
func (s *URLhausSource) Sweep(ctx context.Context, query string, out chan<- recon.Finding) error {
|
||||
base := s.BaseURL
|
||||
if base == "" {
|
||||
base = "https://urlhaus-api.abuse.ch/v1"
|
||||
}
|
||||
client := s.Client
|
||||
if client == nil {
|
||||
client = NewClient()
|
||||
}
|
||||
|
||||
queries := BuildQueries(s.Registry, "urlhaus")
|
||||
if len(queries) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, q := range queries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.Limiters != nil {
|
||||
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Try tag lookup first.
|
||||
tagURL := fmt.Sprintf("%s/tag/%s/", base, url.PathEscape(q))
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, tagURL, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
|
||||
resp, err := client.Do(ctx, req)
|
||||
if err != nil {
|
||||
// Fallback to payload endpoint on tag lookup failure.
|
||||
resp, err = s.payloadFallback(ctx, client, base, q)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var result urlhausResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// If tag lookup returned no results, try payload fallback.
|
||||
if result.QueryStatus != "ok" || len(result.URLs) == 0 {
|
||||
resp, err = s.payloadFallback(ctx, client, base, q)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
data, err = io.ReadAll(io.LimitReader(resp.Body, 512*1024))
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
for _, entry := range result.URLs {
|
||||
// Stringify the record and check for key patterns.
|
||||
record := fmt.Sprintf("url=%s status=%s tags=%v reporter=%s",
|
||||
entry.URL, entry.URLStatus, entry.Tags, entry.Reporter)
|
||||
if ciLogKeyPattern.MatchString(record) || ciLogKeyPattern.MatchString(entry.URL) {
|
||||
out <- recon.Finding{
|
||||
ProviderName: q,
|
||||
Source: entry.URL,
|
||||
SourceType: "recon:urlhaus",
|
||||
Confidence: "medium",
|
||||
DetectedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// payloadFallback tries the URLhaus payload endpoint as a secondary search method.
|
||||
func (s *URLhausSource) payloadFallback(ctx context.Context, client *Client, base, tag string) (*http.Response, error) {
|
||||
payloadURL := fmt.Sprintf("%s/payload/", base)
|
||||
body := fmt.Sprintf("md5_hash=&sha256_hash=&tag=%s", url.QueryEscape(tag))
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, payloadURL, strings.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
return client.Do(ctx, req)
|
||||
}
|
||||
119
pkg/recon/sources/urlhaus_test.go
Normal file
119
pkg/recon/sources/urlhaus_test.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
func TestURLhaus_Name(t *testing.T) {
|
||||
s := &URLhausSource{}
|
||||
if s.Name() != "urlhaus" {
|
||||
t.Fatalf("expected urlhaus, got %s", s.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestURLhaus_Enabled(t *testing.T) {
|
||||
s := &URLhausSource{}
|
||||
if !s.Enabled(recon.Config{}) {
|
||||
t.Fatal("URLhausSource should always be enabled (credentialless)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestURLhaus_Sweep(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/tag/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"query_status": "ok",
|
||||
"urls": [{
|
||||
"url": "https://evil.example.com/exfil?token=sk-proj-ABCDEF1234567890abcdef",
|
||||
"url_status": "online",
|
||||
"tags": ["malware", "api_key"],
|
||||
"reporter": "abuse_ch"
|
||||
}]
|
||||
}`))
|
||||
})
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
reg := providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
|
||||
s := &URLhausSource{
|
||||
BaseURL: srv.URL,
|
||||
Registry: reg,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) == 0 {
|
||||
t.Fatal("expected at least one finding from URLhaus")
|
||||
}
|
||||
if findings[0].SourceType != "recon:urlhaus" {
|
||||
t.Fatalf("expected recon:urlhaus, got %s", findings[0].SourceType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestURLhaus_Sweep_Empty(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/tag/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"query_status": "no_results", "urls": []}`))
|
||||
})
|
||||
mux.HandleFunc("/payload/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"query_status": "no_results", "urls": []}`))
|
||||
})
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
reg := providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
|
||||
s := &URLhausSource{
|
||||
BaseURL: srv.URL,
|
||||
Registry: reg,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("expected no findings, got %d", len(findings))
|
||||
}
|
||||
}
|
||||
116
pkg/recon/sources/virustotal.go
Normal file
116
pkg/recon/sources/virustotal.go
Normal file
@@ -0,0 +1,116 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
// VirusTotalSource searches the VirusTotal Intelligence API for files and URLs
|
||||
// containing API key patterns. Malware samples frequently contain hard-coded
|
||||
// API keys used by threat actors to exfiltrate data or proxy requests.
|
||||
type VirusTotalSource struct {
|
||||
APIKey string
|
||||
BaseURL string
|
||||
Registry *providers.Registry
|
||||
Limiters *recon.LimiterRegistry
|
||||
Client *Client
|
||||
}
|
||||
|
||||
var _ recon.ReconSource = (*VirusTotalSource)(nil)
|
||||
|
||||
func (s *VirusTotalSource) Name() string { return "virustotal" }
|
||||
func (s *VirusTotalSource) RateLimit() rate.Limit { return rate.Every(15 * time.Second) }
|
||||
func (s *VirusTotalSource) Burst() int { return 2 }
|
||||
func (s *VirusTotalSource) RespectsRobots() bool { return false }
|
||||
func (s *VirusTotalSource) Enabled(_ recon.Config) bool {
|
||||
return s.APIKey != ""
|
||||
}
|
||||
|
||||
// vtSearchResponse represents the top-level VT intelligence search response.
|
||||
type vtSearchResponse struct {
|
||||
Data []vtSearchItem `json:"data"`
|
||||
}
|
||||
|
||||
// vtSearchItem is a single item in the VT search results.
|
||||
type vtSearchItem struct {
|
||||
ID string `json:"id"`
|
||||
Attributes json.RawMessage `json:"attributes"`
|
||||
}
|
||||
|
||||
func (s *VirusTotalSource) Sweep(ctx context.Context, query string, out chan<- recon.Finding) error {
|
||||
base := s.BaseURL
|
||||
if base == "" {
|
||||
base = "https://www.virustotal.com/api/v3"
|
||||
}
|
||||
client := s.Client
|
||||
if client == nil {
|
||||
client = NewClient()
|
||||
}
|
||||
|
||||
queries := BuildQueries(s.Registry, "virustotal")
|
||||
if len(queries) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, q := range queries {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if s.Limiters != nil {
|
||||
if err := s.Limiters.Wait(ctx, s.Name(), s.RateLimit(), s.Burst(), false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
searchURL := fmt.Sprintf(
|
||||
"%s/intelligence/search?query=%s&limit=10",
|
||||
base, url.QueryEscape(q),
|
||||
)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("x-apikey", s.APIKey)
|
||||
|
||||
resp, err := client.Do(ctx, req)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var result vtSearchResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, item := range result.Data {
|
||||
attrs := string(item.Attributes)
|
||||
if ciLogKeyPattern.MatchString(attrs) {
|
||||
out <- recon.Finding{
|
||||
ProviderName: q,
|
||||
Source: fmt.Sprintf("https://www.virustotal.com/gui/file/%s", item.ID),
|
||||
SourceType: "recon:virustotal",
|
||||
Confidence: "medium",
|
||||
DetectedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
126
pkg/recon/sources/virustotal_test.go
Normal file
126
pkg/recon/sources/virustotal_test.go
Normal file
@@ -0,0 +1,126 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/salvacybersec/keyhunter/pkg/providers"
|
||||
"github.com/salvacybersec/keyhunter/pkg/recon"
|
||||
)
|
||||
|
||||
func TestVirusTotal_Name(t *testing.T) {
|
||||
s := &VirusTotalSource{}
|
||||
if s.Name() != "virustotal" {
|
||||
t.Fatalf("expected virustotal, got %s", s.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirusTotal_Enabled(t *testing.T) {
|
||||
s := &VirusTotalSource{}
|
||||
if s.Enabled(recon.Config{}) {
|
||||
t.Fatal("VirusTotalSource should be disabled without API key")
|
||||
}
|
||||
s.APIKey = "test-key"
|
||||
if !s.Enabled(recon.Config{}) {
|
||||
t.Fatal("VirusTotalSource should be enabled with API key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirusTotal_Sweep(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/intelligence/search", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("x-apikey") != "test-key" {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"data": [{
|
||||
"id": "abc123def456",
|
||||
"attributes": {
|
||||
"meaningful_name": "malware.exe",
|
||||
"tags": ["trojan"],
|
||||
"api_key": "sk-proj-ABCDEF1234567890abcdef"
|
||||
}
|
||||
}]
|
||||
}`))
|
||||
})
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
reg := providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
|
||||
s := &VirusTotalSource{
|
||||
APIKey: "test-key",
|
||||
BaseURL: srv.URL,
|
||||
Registry: reg,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) == 0 {
|
||||
t.Fatal("expected at least one finding from VirusTotal")
|
||||
}
|
||||
if findings[0].SourceType != "recon:virustotal" {
|
||||
t.Fatalf("expected recon:virustotal, got %s", findings[0].SourceType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirusTotal_Sweep_Empty(t *testing.T) {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/intelligence/search", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data": []}`))
|
||||
})
|
||||
|
||||
srv := httptest.NewServer(mux)
|
||||
defer srv.Close()
|
||||
|
||||
reg := providers.NewRegistryFromProviders([]providers.Provider{
|
||||
{Name: "openai", Keywords: []string{"sk-proj-"}},
|
||||
})
|
||||
|
||||
s := &VirusTotalSource{
|
||||
APIKey: "test-key",
|
||||
BaseURL: srv.URL,
|
||||
Registry: reg,
|
||||
Client: NewClient(),
|
||||
}
|
||||
|
||||
out := make(chan recon.Finding, 10)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := s.Sweep(ctx, "", out)
|
||||
close(out)
|
||||
if err != nil {
|
||||
t.Fatalf("Sweep error: %v", err)
|
||||
}
|
||||
|
||||
var findings []recon.Finding
|
||||
for f := range out {
|
||||
findings = append(findings, f)
|
||||
}
|
||||
if len(findings) != 0 {
|
||||
t.Fatalf("expected no findings, got %d", len(findings))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user