a195ef33a0
- SecurityTrailsSource enumerates subdomains via API, probes config endpoints - Credential-gated via SECURITYTRAILS_API_KEY env var - RegisterAll extended to 70 sources (67 Phase 10-15 + 3 Phase 16) - cmd/recon.go wires SecurityTrails API key from env/viper
298 lines
9.0 KiB
Go
298 lines
9.0 KiB
Go
package sources
|
|
|
|
import (
|
|
"github.com/salvacybersec/keyhunter/pkg/providers"
|
|
"github.com/salvacybersec/keyhunter/pkg/recon"
|
|
)
|
|
|
|
// SourcesConfig carries per-source credentials and shared dependencies read
|
|
// from viper/env by cmd/recon.go and handed to RegisterAll.
|
|
//
|
|
// Fields are populated from environment variables (GITHUB_TOKEN, GITLAB_TOKEN,
|
|
// ...) or viper config keys (recon.github.token, ...). Empty values are
|
|
// permitted: the corresponding source is still registered on the engine, but
|
|
// its Enabled() reports false so SweepAll skips it cleanly.
|
|
type SourcesConfig struct {
|
|
// GitHub / Gist share the same token.
|
|
GitHubToken string
|
|
// GitLab personal access token.
|
|
GitLabToken string
|
|
// Bitbucket Cloud app password or OAuth token + required workspace slug.
|
|
BitbucketToken string
|
|
BitbucketWorkspace string
|
|
// Codeberg (Gitea) token — optional, raises rate limit when present.
|
|
CodebergToken string
|
|
// HuggingFace Hub token — optional, raises rate limit when present.
|
|
HuggingFaceToken string
|
|
// Kaggle Basic-auth username + API key.
|
|
KaggleUser string
|
|
KaggleKey string
|
|
|
|
// Google Custom Search API key and search engine ID (CX).
|
|
GoogleAPIKey string
|
|
GoogleCX string
|
|
// Bing Web Search API subscription key.
|
|
BingAPIKey string
|
|
// Yandex XML Search user and API key.
|
|
YandexUser string
|
|
YandexAPIKey string
|
|
// Brave Search API subscription token.
|
|
BraveAPIKey string
|
|
|
|
// Phase 12: IoT scanner API keys.
|
|
ShodanAPIKey string
|
|
CensysAPIId string
|
|
CensysAPISecret string
|
|
ZoomEyeAPIKey string
|
|
FOFAEmail string
|
|
FOFAAPIKey string
|
|
NetlasAPIKey string
|
|
BinaryEdgeAPIKey string
|
|
|
|
// Phase 14: CI/CD source tokens.
|
|
CircleCIToken string
|
|
|
|
// Phase 16: DNS/threat intel source tokens.
|
|
SecurityTrailsAPIKey string
|
|
|
|
// Registry drives query generation for every source via BuildQueries.
|
|
Registry *providers.Registry
|
|
// Limiters is the shared per-source rate-limiter registry.
|
|
Limiters *recon.LimiterRegistry
|
|
}
|
|
|
|
// RegisterAll registers every Phase 10 code-hosting, Phase 11 search engine /
|
|
// paste site, Phase 12 IoT scanner / cloud storage, Phase 13 package
|
|
// registry / container / IaC, Phase 14 CI/CD log / web archive / frontend
|
|
// leak, Phase 15 forum / collaboration tool / log aggregator, and Phase 16
|
|
// mobile / DNS / threat intel source on engine (70 sources total).
|
|
//
|
|
// All sources are registered unconditionally so that cmd/recon.go can surface
|
|
// the full catalog via `keyhunter recon list` regardless of which credentials
|
|
// are configured. Sources without required credentials return Enabled()==false
|
|
// so SweepAll skips them without erroring.
|
|
//
|
|
// A nil engine is treated as a no-op (not an error) -- callers in broken init
|
|
// paths shouldn't panic.
|
|
func RegisterAll(engine *recon.Engine, cfg SourcesConfig) {
|
|
if engine == nil {
|
|
return
|
|
}
|
|
reg := cfg.Registry
|
|
lim := cfg.Limiters
|
|
|
|
// API sources with constructors.
|
|
engine.Register(NewGitHubSource(cfg.GitHubToken, reg, lim))
|
|
engine.Register(NewKaggleSource(cfg.KaggleUser, cfg.KaggleKey, reg, lim))
|
|
engine.Register(NewHuggingFaceSource(HuggingFaceConfig{
|
|
Token: cfg.HuggingFaceToken,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
}))
|
|
|
|
// API sources exposed as struct literals (no New* constructor in Wave 2).
|
|
engine.Register(&GitLabSource{
|
|
Token: cfg.GitLabToken,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&BitbucketSource{
|
|
Token: cfg.BitbucketToken,
|
|
Workspace: cfg.BitbucketWorkspace,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&GistSource{
|
|
Token: cfg.GitHubToken,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&CodebergSource{
|
|
Token: cfg.CodebergToken,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
|
|
// Scraping sources (credentialless).
|
|
engine.Register(&ReplitSource{
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&CodeSandboxSource{
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&SandboxesSource{
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
|
|
// Phase 11: Search engine dorking sources.
|
|
engine.Register(&GoogleDorkSource{
|
|
APIKey: cfg.GoogleAPIKey,
|
|
CX: cfg.GoogleCX,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&BingDorkSource{
|
|
APIKey: cfg.BingAPIKey,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&DuckDuckGoSource{
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&YandexSource{
|
|
User: cfg.YandexUser,
|
|
APIKey: cfg.YandexAPIKey,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&BraveSource{
|
|
APIKey: cfg.BraveAPIKey,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
|
|
// Phase 11: Paste site sources.
|
|
engine.Register(&PastebinSource{
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&GistPasteSource{
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&PasteSitesSource{
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
|
|
// Phase 12: IoT scanner sources.
|
|
engine.Register(&ShodanSource{
|
|
APIKey: cfg.ShodanAPIKey,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&CensysSource{
|
|
APIId: cfg.CensysAPIId,
|
|
APISecret: cfg.CensysAPISecret,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&ZoomEyeSource{
|
|
APIKey: cfg.ZoomEyeAPIKey,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&FOFASource{
|
|
Email: cfg.FOFAEmail,
|
|
APIKey: cfg.FOFAAPIKey,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&NetlasSource{
|
|
APIKey: cfg.NetlasAPIKey,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&BinaryEdgeSource{
|
|
APIKey: cfg.BinaryEdgeAPIKey,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
|
|
// Phase 12: Cloud storage sources (credentialless).
|
|
engine.Register(&S3Scanner{
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&GCSScanner{
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&AzureBlobScanner{
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&DOSpacesScanner{
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
|
|
// Phase 13: Package registry sources (credentialless).
|
|
engine.Register(&NpmSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&PyPISource{Registry: reg, Limiters: lim})
|
|
engine.Register(&CratesIOSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&RubyGemsSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&MavenSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&NuGetSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&GoProxySource{Registry: reg, Limiters: lim})
|
|
engine.Register(&PackagistSource{Registry: reg, Limiters: lim})
|
|
|
|
// Phase 13: Container and IaC sources (credentialless).
|
|
engine.Register(&DockerHubSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&KubernetesSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&TerraformSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&HelmSource{Registry: reg, Limiters: lim})
|
|
|
|
// Phase 14: Frontend leak sources (credentialless).
|
|
engine.Register(&SourceMapSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&WebpackSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&EnvLeakSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&SwaggerSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&DeployPreviewSource{Registry: reg, Limiters: lim})
|
|
|
|
// Phase 14: CI/CD log sources.
|
|
engine.Register(&TravisCISource{Registry: reg, Limiters: lim})
|
|
engine.Register(&GitHubActionsSource{
|
|
Token: cfg.GitHubToken,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&CircleCISource{
|
|
Token: cfg.CircleCIToken,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
engine.Register(&JenkinsSource{Registry: reg, Limiters: lim})
|
|
|
|
// Phase 14: Web archive sources (credentialless).
|
|
engine.Register(&WaybackMachineSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&CommonCrawlSource{Registry: reg, Limiters: lim})
|
|
|
|
// Phase 14: JS bundle analysis (credentialless).
|
|
engine.Register(&JSBundleSource{Registry: reg, Limiters: lim})
|
|
|
|
// Phase 15: Forum and discussion sources (credentialless).
|
|
engine.Register(&StackOverflowSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&RedditSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&HackerNewsSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&DiscordSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&SlackSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&DevToSource{Registry: reg, Limiters: lim})
|
|
|
|
// Phase 15: Collaboration tool sources (credentialless).
|
|
engine.Register(&TrelloSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&NotionSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&ConfluenceSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&GoogleDocsSource{Registry: reg, Limiters: lim})
|
|
|
|
// Phase 15: Log aggregator sources (credentialless — target exposed instances).
|
|
engine.Register(&ElasticsearchSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&KibanaSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&SplunkSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&GrafanaSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&SentrySource{Registry: reg, Limiters: lim})
|
|
|
|
// Phase 16: Mobile, DNS, and threat intel sources.
|
|
engine.Register(&APKMirrorSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&CrtShSource{Registry: reg, Limiters: lim})
|
|
engine.Register(&SecurityTrailsSource{
|
|
APIKey: cfg.SecurityTrailsAPIKey,
|
|
Registry: reg,
|
|
Limiters: lim,
|
|
})
|
|
}
|