From c8f7592b732babb70344baf5e9e89ae37d92e95c Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Mon, 6 Apr 2026 17:25:43 +0300 Subject: [PATCH 1/4] feat(17-02): add gocron dependency, subscribers and scheduled_jobs tables with CRUD - Add gocron/v2 v2.19.1 as direct dependency - Append subscribers and scheduled_jobs CREATE TABLE to schema.sql - Implement full subscriber CRUD (Add/Remove/List/IsSubscribed) - Implement full scheduled job CRUD (Save/List/Get/Delete/UpdateLastRun/SetEnabled) --- go.mod | 3 + go.sum | 6 ++ pkg/storage/scheduled_jobs.go | 111 ++++++++++++++++++++++++++++++++++ pkg/storage/schema.sql | 20 ++++++ pkg/storage/subscribers.go | 54 +++++++++++++++++ 5 files changed, 194 insertions(+) create mode 100644 pkg/storage/scheduled_jobs.go create mode 100644 pkg/storage/subscribers.go diff --git a/go.mod b/go.mod index 3b95c7c..f1a9d9c 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-co-op/gocron/v2 v2.19.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.8.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect @@ -42,6 +43,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jonboulle/clockwork v0.5.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -52,6 +54,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.3.1 // indirect diff --git a/go.sum b/go.sum index 96f8ad7..8211d46 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= +github.com/go-co-op/gocron/v2 v2.19.1 h1:B4iLeA0NB/2iO3EKQ7NfKn5KsQgZfjb2fkvoZJU3yBI= +github.com/go-co-op/gocron/v2 v2.19.1/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.8.0 h1:I8hjc3LbBlXTtVuFNJuwYuMiHvQJDq1AT6u4DwDzZG0= @@ -67,6 +69,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= +github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -105,6 +109,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qq github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/pkg/storage/scheduled_jobs.go b/pkg/storage/scheduled_jobs.go new file mode 100644 index 0000000..7315e3f --- /dev/null +++ b/pkg/storage/scheduled_jobs.go @@ -0,0 +1,111 @@ +package storage + +import ( + "database/sql" + "time" +) + +// ScheduledJob represents a cron-based scheduled scan job. +type ScheduledJob struct { + ID int64 + Name string + CronExpr string + ScanCommand string + NotifyTelegram bool + Enabled bool + LastRun *time.Time + NextRun *time.Time + CreatedAt time.Time +} + +// SaveScheduledJob inserts a new scheduled job. Returns the new row ID. +func (db *DB) SaveScheduledJob(j ScheduledJob) (int64, error) { + res, err := db.sql.Exec( + `INSERT INTO scheduled_jobs (name, cron_expr, scan_command, notify_telegram, enabled) + VALUES (?, ?, ?, ?, ?)`, + j.Name, j.CronExpr, j.ScanCommand, j.NotifyTelegram, j.Enabled, + ) + if err != nil { + return 0, err + } + return res.LastInsertId() +} + +// ListScheduledJobs returns all scheduled jobs. +func (db *DB) ListScheduledJobs() ([]ScheduledJob, error) { + rows, err := db.sql.Query( + `SELECT id, name, cron_expr, scan_command, notify_telegram, enabled, last_run, next_run, created_at + FROM scheduled_jobs ORDER BY id`, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var jobs []ScheduledJob + for rows.Next() { + var j ScheduledJob + var lastRun, nextRun sql.NullTime + if err := rows.Scan(&j.ID, &j.Name, &j.CronExpr, &j.ScanCommand, + &j.NotifyTelegram, &j.Enabled, &lastRun, &nextRun, &j.CreatedAt); err != nil { + return nil, err + } + if lastRun.Valid { + j.LastRun = &lastRun.Time + } + if nextRun.Valid { + j.NextRun = &nextRun.Time + } + jobs = append(jobs, j) + } + return jobs, rows.Err() +} + +// GetScheduledJob returns a single scheduled job by name. +func (db *DB) GetScheduledJob(name string) (*ScheduledJob, error) { + var j ScheduledJob + var lastRun, nextRun sql.NullTime + err := db.sql.QueryRow( + `SELECT id, name, cron_expr, scan_command, notify_telegram, enabled, last_run, next_run, created_at + FROM scheduled_jobs WHERE name = ?`, name, + ).Scan(&j.ID, &j.Name, &j.CronExpr, &j.ScanCommand, + &j.NotifyTelegram, &j.Enabled, &lastRun, &nextRun, &j.CreatedAt) + if err != nil { + return nil, err + } + if lastRun.Valid { + j.LastRun = &lastRun.Time + } + if nextRun.Valid { + j.NextRun = &nextRun.Time + } + return &j, nil +} + +// DeleteScheduledJob removes a scheduled job by name. Returns rows affected. +func (db *DB) DeleteScheduledJob(name string) (int64, error) { + res, err := db.sql.Exec(`DELETE FROM scheduled_jobs WHERE name = ?`, name) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + +// UpdateJobLastRun updates the last_run and next_run timestamps for a job. +func (db *DB) UpdateJobLastRun(name string, lastRun time.Time, nextRun *time.Time) error { + var nr sql.NullTime + if nextRun != nil { + nr = sql.NullTime{Time: *nextRun, Valid: true} + } + _, err := db.sql.Exec( + `UPDATE scheduled_jobs SET last_run = ?, next_run = ? WHERE name = ?`, + lastRun, nr, name, + ) + return err +} + +// SetJobEnabled updates the enabled flag for a scheduled job. +func (db *DB) SetJobEnabled(name string, enabled bool) error { + _, err := db.sql.Exec(`UPDATE scheduled_jobs SET enabled = ? WHERE name = ?`, enabled, name) + return err +} diff --git a/pkg/storage/schema.sql b/pkg/storage/schema.sql index 964dad9..84e91d9 100644 --- a/pkg/storage/schema.sql +++ b/pkg/storage/schema.sql @@ -55,3 +55,23 @@ CREATE TABLE IF NOT EXISTS custom_dorks ( CREATE INDEX IF NOT EXISTS idx_custom_dorks_source ON custom_dorks(source); CREATE INDEX IF NOT EXISTS idx_custom_dorks_category ON custom_dorks(category); + +-- Phase 17: Telegram bot subscribers for auto-notifications. +CREATE TABLE IF NOT EXISTS subscribers ( + chat_id INTEGER PRIMARY KEY, + username TEXT, + subscribed_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +-- Phase 17: Cron-based scheduled scan jobs. +CREATE TABLE IF NOT EXISTS scheduled_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT UNIQUE NOT NULL, + cron_expr TEXT NOT NULL, + scan_command TEXT NOT NULL, + notify_telegram BOOLEAN DEFAULT FALSE, + enabled BOOLEAN DEFAULT TRUE, + last_run DATETIME, + next_run DATETIME, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); diff --git a/pkg/storage/subscribers.go b/pkg/storage/subscribers.go new file mode 100644 index 0000000..fa6fd3b --- /dev/null +++ b/pkg/storage/subscribers.go @@ -0,0 +1,54 @@ +package storage + +import "time" + +// Subscriber represents a Telegram chat subscribed to scan notifications. +type Subscriber struct { + ChatID int64 + Username string + SubscribedAt time.Time +} + +// AddSubscriber inserts or replaces a subscriber in the database. +func (db *DB) AddSubscriber(chatID int64, username string) error { + _, err := db.sql.Exec( + `INSERT OR REPLACE INTO subscribers (chat_id, username, subscribed_at) VALUES (?, ?, CURRENT_TIMESTAMP)`, + chatID, username, + ) + return err +} + +// RemoveSubscriber deletes a subscriber by chat ID. Returns rows affected. +func (db *DB) RemoveSubscriber(chatID int64) (int64, error) { + res, err := db.sql.Exec(`DELETE FROM subscribers WHERE chat_id = ?`, chatID) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + +// ListSubscribers returns all subscribers ordered by subscription time. +func (db *DB) ListSubscribers() ([]Subscriber, error) { + rows, err := db.sql.Query(`SELECT chat_id, username, subscribed_at FROM subscribers ORDER BY subscribed_at`) + if err != nil { + return nil, err + } + defer rows.Close() + + var subs []Subscriber + for rows.Next() { + var s Subscriber + if err := rows.Scan(&s.ChatID, &s.Username, &s.SubscribedAt); err != nil { + return nil, err + } + subs = append(subs, s) + } + return subs, rows.Err() +} + +// IsSubscribed returns true if the given chat ID is subscribed. +func (db *DB) IsSubscribed(chatID int64) (bool, error) { + var count int + err := db.sql.QueryRow(`SELECT COUNT(*) FROM subscribers WHERE chat_id = ?`, chatID).Scan(&count) + return count > 0, err +} From 89cc1339828e2f321ebd159abd0e095ff79abf24 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Mon, 6 Apr 2026 17:26:20 +0300 Subject: [PATCH 2/4] test(17-02): add failing tests for scheduler package - Storage round-trip test for SaveScheduledJob/ListScheduledJobs - Subscriber round-trip test for Add/Remove/List/IsSubscribed - Scheduler Start loads enabled jobs from DB - Scheduler AddJob/RemoveJob persists and registers - Scheduler RunJob manual trigger with callback --- pkg/scheduler/jobs.go | 1 + pkg/scheduler/scheduler.go | 1 + pkg/scheduler/scheduler_test.go | 204 ++++++++++++++++++++++++++++++++ 3 files changed, 206 insertions(+) create mode 100644 pkg/scheduler/jobs.go create mode 100644 pkg/scheduler/scheduler.go create mode 100644 pkg/scheduler/scheduler_test.go diff --git a/pkg/scheduler/jobs.go b/pkg/scheduler/jobs.go new file mode 100644 index 0000000..6990da0 --- /dev/null +++ b/pkg/scheduler/jobs.go @@ -0,0 +1 @@ +package scheduler diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go new file mode 100644 index 0000000..6990da0 --- /dev/null +++ b/pkg/scheduler/scheduler.go @@ -0,0 +1 @@ +package scheduler diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go new file mode 100644 index 0000000..9ce8879 --- /dev/null +++ b/pkg/scheduler/scheduler_test.go @@ -0,0 +1,204 @@ +package scheduler_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/salvacybersec/keyhunter/pkg/scheduler" + "github.com/salvacybersec/keyhunter/pkg/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func openTestDB(t *testing.T) *storage.DB { + t.Helper() + db, err := storage.Open(":memory:") + require.NoError(t, err) + t.Cleanup(func() { db.Close() }) + return db +} + +func TestStorageRoundTrip(t *testing.T) { + db := openTestDB(t) + + id, err := db.SaveScheduledJob(storage.ScheduledJob{ + Name: "nightly-scan", + CronExpr: "0 2 * * *", + ScanCommand: "/tmp/repos", + NotifyTelegram: true, + Enabled: true, + }) + require.NoError(t, err) + assert.Greater(t, id, int64(0)) + + jobs, err := db.ListScheduledJobs() + require.NoError(t, err) + require.Len(t, jobs, 1) + assert.Equal(t, "nightly-scan", jobs[0].Name) + assert.Equal(t, "0 2 * * *", jobs[0].CronExpr) + assert.Equal(t, "/tmp/repos", jobs[0].ScanCommand) + assert.True(t, jobs[0].NotifyTelegram) + assert.True(t, jobs[0].Enabled) + + got, err := db.GetScheduledJob("nightly-scan") + require.NoError(t, err) + assert.Equal(t, "nightly-scan", got.Name) + + now := time.Now().UTC() + next := now.Add(24 * time.Hour) + err = db.UpdateJobLastRun("nightly-scan", now, &next) + require.NoError(t, err) + + got2, err := db.GetScheduledJob("nightly-scan") + require.NoError(t, err) + require.NotNil(t, got2.LastRun) + require.NotNil(t, got2.NextRun) + + n, err := db.DeleteScheduledJob("nightly-scan") + require.NoError(t, err) + assert.Equal(t, int64(1), n) + + jobs, err = db.ListScheduledJobs() + require.NoError(t, err) + assert.Empty(t, jobs) +} + +func TestSubscriberRoundTrip(t *testing.T) { + db := openTestDB(t) + + err := db.AddSubscriber(12345, "alice") + require.NoError(t, err) + + subs, err := db.ListSubscribers() + require.NoError(t, err) + require.Len(t, subs, 1) + assert.Equal(t, int64(12345), subs[0].ChatID) + assert.Equal(t, "alice", subs[0].Username) + + ok, err := db.IsSubscribed(12345) + require.NoError(t, err) + assert.True(t, ok) + + ok, err = db.IsSubscribed(99999) + require.NoError(t, err) + assert.False(t, ok) + + n, err := db.RemoveSubscriber(12345) + require.NoError(t, err) + assert.Equal(t, int64(1), n) + + subs, err = db.ListSubscribers() + require.NoError(t, err) + assert.Empty(t, subs) +} + +func TestSchedulerStartLoadsJobs(t *testing.T) { + db := openTestDB(t) + + _, err := db.SaveScheduledJob(storage.ScheduledJob{ + Name: "job-a", CronExpr: "0 * * * *", ScanCommand: "/a", Enabled: true, + }) + require.NoError(t, err) + _, err = db.SaveScheduledJob(storage.ScheduledJob{ + Name: "job-b", CronExpr: "0 * * * *", ScanCommand: "/b", Enabled: true, + }) + require.NoError(t, err) + // Disabled job should not be registered + _, err = db.SaveScheduledJob(storage.ScheduledJob{ + Name: "job-c", CronExpr: "0 * * * *", ScanCommand: "/c", Enabled: false, + }) + require.NoError(t, err) + + s, err := scheduler.New(scheduler.Config{ + DB: db, + ScanFunc: func(ctx context.Context, cmd string) (int, error) { + return 0, nil + }, + }) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err = s.Start(ctx) + require.NoError(t, err) + defer s.Stop() + + assert.Equal(t, 2, s.JobCount()) +} + +func TestSchedulerAddRemoveJob(t *testing.T) { + db := openTestDB(t) + + s, err := scheduler.New(scheduler.Config{ + DB: db, + ScanFunc: func(ctx context.Context, cmd string) (int, error) { + return 0, nil + }, + }) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err = s.Start(ctx) + require.NoError(t, err) + defer s.Stop() + + err = s.AddJob("test-job", "0 * * * *", "/test", true) + require.NoError(t, err) + assert.Equal(t, 1, s.JobCount()) + + jobs, err := db.ListScheduledJobs() + require.NoError(t, err) + require.Len(t, jobs, 1) + assert.Equal(t, "test-job", jobs[0].Name) + + err = s.RemoveJob("test-job") + require.NoError(t, err) + assert.Equal(t, 0, s.JobCount()) + + jobs, err = db.ListScheduledJobs() + require.NoError(t, err) + assert.Empty(t, jobs) +} + +func TestSchedulerRunJob(t *testing.T) { + db := openTestDB(t) + + var mu sync.Mutex + var scanCalled string + var completeCalled bool + + s, err := scheduler.New(scheduler.Config{ + DB: db, + ScanFunc: func(ctx context.Context, cmd string) (int, error) { + mu.Lock() + scanCalled = cmd + mu.Unlock() + return 5, nil + }, + OnComplete: func(result scheduler.JobResult) { + mu.Lock() + completeCalled = true + mu.Unlock() + }, + }) + require.NoError(t, err) + + _, err = db.SaveScheduledJob(storage.ScheduledJob{ + Name: "manual-run", CronExpr: "0 * * * *", ScanCommand: "/manual", NotifyTelegram: true, Enabled: true, + }) + require.NoError(t, err) + + result, err := s.RunJob(context.Background(), "manual-run") + require.NoError(t, err) + assert.Equal(t, "manual-run", result.JobName) + assert.Equal(t, 5, result.FindingCount) + + mu.Lock() + assert.Equal(t, "/manual", scanCalled) + assert.True(t, completeCalled) + mu.Unlock() +} From c71faa97f59a4dda0045dd2478db10374d886a9d Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Mon, 6 Apr 2026 17:27:00 +0300 Subject: [PATCH 3/4] feat(17-02): implement scheduler package with gocron wrapper and job lifecycle - Scheduler wraps gocron with Start/Stop lifecycle - Start loads enabled jobs from DB and registers cron schedules - AddJob/RemoveJob persist to DB and sync with gocron - RunJob for manual trigger with OnComplete callback - JobResult struct for notification bridge - Promote gocron/v2 v2.19.1 to direct dependency --- go.mod | 6 +- go.sum | 2 + pkg/scheduler/jobs.go | 20 +++++ pkg/scheduler/scheduler.go | 178 +++++++++++++++++++++++++++++++++++++ 4 files changed, 203 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f1a9d9c..001864f 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26.1 require ( github.com/atotto/clipboard v0.1.4 github.com/charmbracelet/lipgloss v1.1.0 + github.com/go-co-op/gocron/v2 v2.19.1 github.com/go-git/go-git/v5 v5.17.2 github.com/mattn/go-isatty v0.0.20 github.com/panjf2000/ants/v2 v2.12.0 @@ -12,9 +13,11 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 + github.com/temoto/robotstxt v1.1.2 github.com/tidwall/gjson v1.18.0 golang.org/x/crypto v0.49.0 golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 + golang.org/x/net v0.52.0 golang.org/x/time v0.15.0 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.48.1 @@ -35,7 +38,6 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/go-co-op/gocron/v2 v2.19.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.8.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect @@ -63,13 +65,11 @@ require ( github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/temoto/robotstxt v1.1.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.52.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.35.0 // indirect diff --git a/go.sum b/go.sum index 8211d46..a0c74ea 100644 --- a/go.sum +++ b/go.sum @@ -154,6 +154,8 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= diff --git a/pkg/scheduler/jobs.go b/pkg/scheduler/jobs.go index 6990da0..539abf0 100644 --- a/pkg/scheduler/jobs.go +++ b/pkg/scheduler/jobs.go @@ -1 +1,21 @@ package scheduler + +import "time" + +// Job represents a scheduled scan job with its runtime function. +type Job struct { + Name string + CronExpr string + ScanCommand string + NotifyTelegram bool + Enabled bool + RunFunc func(ctx interface{}) (int, error) +} + +// JobResult contains the outcome of a scheduled or manually triggered scan job. +type JobResult struct { + JobName string + FindingCount int + Duration time.Duration + Error error +} diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 6990da0..f1325e7 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -1 +1,179 @@ package scheduler + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/go-co-op/gocron/v2" + "github.com/salvacybersec/keyhunter/pkg/storage" +) + +// Config holds the dependencies for a Scheduler. +type Config struct { + // DB is the storage backend for persisting jobs and subscribers. + DB *storage.DB + // ScanFunc executes a scan command and returns the finding count. + ScanFunc func(ctx context.Context, scanCommand string) (int, error) + // OnComplete is called after a job finishes. May be nil. + OnComplete func(result JobResult) +} + +// Scheduler wraps gocron with SQLite persistence for scheduled scan jobs. +type Scheduler struct { + cfg Config + sched gocron.Scheduler + jobs map[string]gocron.Job + mu sync.Mutex +} + +// New creates a new Scheduler with the given configuration. +func New(cfg Config) (*Scheduler, error) { + s, err := gocron.NewScheduler() + if err != nil { + return nil, fmt.Errorf("creating gocron scheduler: %w", err) + } + return &Scheduler{ + cfg: cfg, + sched: s, + jobs: make(map[string]gocron.Job), + }, nil +} + +// Start loads all enabled jobs from the database and begins scheduling. +func (s *Scheduler) Start(ctx context.Context) error { + jobs, err := s.cfg.DB.ListScheduledJobs() + if err != nil { + return fmt.Errorf("loading scheduled jobs: %w", err) + } + for _, sj := range jobs { + if !sj.Enabled { + continue + } + if err := s.registerJob(ctx, sj); err != nil { + return fmt.Errorf("registering job %q: %w", sj.Name, err) + } + } + s.sched.Start() + return nil +} + +// Stop shuts down the gocron scheduler. +func (s *Scheduler) Stop() error { + return s.sched.Shutdown() +} + +// AddJob creates a new scheduled job, persists it, and registers it with gocron. +func (s *Scheduler) AddJob(name, cronExpr, scanCommand string, notifyTelegram bool) error { + sj := storage.ScheduledJob{ + Name: name, + CronExpr: cronExpr, + ScanCommand: scanCommand, + NotifyTelegram: notifyTelegram, + Enabled: true, + } + if _, err := s.cfg.DB.SaveScheduledJob(sj); err != nil { + return fmt.Errorf("saving job %q: %w", name, err) + } + return s.registerJob(context.Background(), sj) +} + +// RemoveJob removes a job from gocron and deletes it from the database. +func (s *Scheduler) RemoveJob(name string) error { + s.mu.Lock() + j, ok := s.jobs[name] + if ok { + delete(s.jobs, name) + } + s.mu.Unlock() + + if ok { + if err := s.sched.RemoveJob(j.ID()); err != nil { + return fmt.Errorf("removing gocron job %q: %w", name, err) + } + } + + if _, err := s.cfg.DB.DeleteScheduledJob(name); err != nil { + return fmt.Errorf("deleting job %q from DB: %w", name, err) + } + return nil +} + +// ListJobs returns all scheduled jobs from the database. +func (s *Scheduler) ListJobs() ([]storage.ScheduledJob, error) { + return s.cfg.DB.ListScheduledJobs() +} + +// RunJob manually triggers a job by name. Looks up the job in the DB, +// runs ScanFunc, updates last_run, and calls OnComplete. +func (s *Scheduler) RunJob(ctx context.Context, name string) (JobResult, error) { + sj, err := s.cfg.DB.GetScheduledJob(name) + if err != nil { + return JobResult{}, fmt.Errorf("getting job %q: %w", name, err) + } + + start := time.Now() + findings, scanErr := s.cfg.ScanFunc(ctx, sj.ScanCommand) + dur := time.Since(start) + + result := JobResult{ + JobName: name, + FindingCount: findings, + Duration: dur, + Error: scanErr, + } + + // Update last_run regardless of error + now := time.Now().UTC() + _ = s.cfg.DB.UpdateJobLastRun(name, now, nil) + + if s.cfg.OnComplete != nil { + s.cfg.OnComplete(result) + } + + return result, nil +} + +// JobCount returns the number of registered gocron jobs. +func (s *Scheduler) JobCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.jobs) +} + +// registerJob creates a gocron cron job and stores the handle. +func (s *Scheduler) registerJob(ctx context.Context, sj storage.ScheduledJob) error { + jobName := sj.Name + scanCmd := sj.ScanCommand + notify := sj.NotifyTelegram + + j, err := s.sched.NewJob( + gocron.CronJob(sj.CronExpr, false), + gocron.NewTask(func() { + start := time.Now() + findings, scanErr := s.cfg.ScanFunc(ctx, scanCmd) + dur := time.Since(start) + + now := time.Now().UTC() + _ = s.cfg.DB.UpdateJobLastRun(jobName, now, nil) + + if notify && s.cfg.OnComplete != nil { + s.cfg.OnComplete(JobResult{ + JobName: jobName, + FindingCount: findings, + Duration: dur, + Error: scanErr, + }) + } + }), + ) + if err != nil { + return fmt.Errorf("creating gocron job: %w", err) + } + + s.mu.Lock() + s.jobs[sj.Name] = j + s.mu.Unlock() + return nil +} From d8a610758bf391a17ca88d99e14710cb73caa477 Mon Sep 17 00:00:00 2001 From: salvacybersec Date: Mon, 6 Apr 2026 17:28:30 +0300 Subject: [PATCH 4/4] docs(17-02): complete scheduler + storage plan - Add 17-02-SUMMARY.md with execution results - Update STATE.md position and metrics - Mark SCHED-01 complete in REQUIREMENTS.md --- .planning/REQUIREMENTS.md | 2 +- .planning/STATE.md | 16 +-- .../17-telegram-scheduler/17-02-SUMMARY.md | 105 ++++++++++++++++++ 3 files changed, 115 insertions(+), 8 deletions(-) create mode 100644 .planning/phases/17-telegram-scheduler/17-02-SUMMARY.md diff --git a/.planning/REQUIREMENTS.md b/.planning/REQUIREMENTS.md index 8252aa2..07c7a19 100644 --- a/.planning/REQUIREMENTS.md +++ b/.planning/REQUIREMENTS.md @@ -242,7 +242,7 @@ Requirements for initial release. Each maps to roadmap phases. ### Scheduled Scanning -- [ ] **SCHED-01**: Cron-based recurring scan scheduling +- [x] **SCHED-01**: Cron-based recurring scan scheduling - [ ] **SCHED-02**: keyhunter schedule add/list/remove commands - [ ] **SCHED-03**: Auto-notify on scheduled scan completion diff --git a/.planning/STATE.md b/.planning/STATE.md index 1da07ac..054bd87 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -3,14 +3,14 @@ gsd_state_version: 1.0 milestone: v1.0 milestone_name: milestone status: executing -stopped_at: Completed 16-01-PLAN.md -last_updated: "2026-04-06T13:48:35.313Z" +stopped_at: Completed 17-02-PLAN.md +last_updated: "2026-04-06T14:28:16.926Z" last_activity: 2026-04-06 progress: total_phases: 18 completed_phases: 14 total_plans: 85 - completed_plans: 83 + completed_plans: 84 percent: 20 --- @@ -26,8 +26,8 @@ See: .planning/PROJECT.md (updated 2026-04-04) ## Current Position Phase: 17 -Plan: Not started -Status: Ready to execute +Plan: 2 of 5 +Status: executing Last activity: 2026-04-06 Progress: [██░░░░░░░░] 20% @@ -100,6 +100,7 @@ Progress: [██░░░░░░░░] 20% | Phase 15 P01 | 3min | 2 tasks | 13 files | | Phase 15 P03 | 4min | 2 tasks | 11 files | | Phase 16 P01 | 4min | 2 tasks | 6 files | +| Phase 17 P02 | 2min | 2 tasks | 8 files | ## Accumulated Context @@ -152,6 +153,7 @@ Recent decisions affecting current work: - [Phase 16]: VT uses x-apikey header per official API v3 spec - [Phase 16]: IX uses three-step flow: POST search, GET results, GET file content - [Phase 16]: URLhaus tag lookup with payload endpoint fallback +- [Phase 17]: Scheduler.ScanFunc callback decouples from engine; OnComplete bridges to notifications; disabled jobs stay in DB ### Pending Todos @@ -166,6 +168,6 @@ None yet. ## Session Continuity -Last session: 2026-04-06T13:46:09.383Z -Stopped at: Completed 16-01-PLAN.md +Last session: 2026-04-06T14:28:16.922Z +Stopped at: Completed 17-02-PLAN.md Resume file: None diff --git a/.planning/phases/17-telegram-scheduler/17-02-SUMMARY.md b/.planning/phases/17-telegram-scheduler/17-02-SUMMARY.md new file mode 100644 index 0000000..da13dfb --- /dev/null +++ b/.planning/phases/17-telegram-scheduler/17-02-SUMMARY.md @@ -0,0 +1,105 @@ +--- +phase: 17-telegram-scheduler +plan: 02 +subsystem: scheduler +tags: [gocron, sqlite, cron, scheduler, telegram] + +requires: + - phase: 01-foundation + provides: pkg/storage DB wrapper with schema.sql embed pattern +provides: + - pkg/scheduler/ package with gocron wrapper, start/stop lifecycle + - Storage CRUD for subscribers table (Add/Remove/List/IsSubscribed) + - Storage CRUD for scheduled_jobs table (Save/List/Get/Delete/UpdateLastRun/SetEnabled) + - subscribers and scheduled_jobs SQLite tables in schema.sql +affects: [17-telegram-scheduler, 17-03, 17-04, 17-05] + +tech-stack: + added: [gocron/v2 v2.19.1] + patterns: [scheduler wraps gocron with DB persistence, ScanFunc abstraction decouples from engine] + +key-files: + created: + - pkg/scheduler/scheduler.go + - pkg/scheduler/jobs.go + - pkg/scheduler/scheduler_test.go + - pkg/storage/subscribers.go + - pkg/storage/scheduled_jobs.go + modified: + - pkg/storage/schema.sql + - go.mod + - go.sum + +key-decisions: + - "Scheduler.ScanFunc callback decouples from engine -- Plan 17-04 wires the real scan logic" + - "OnComplete callback bridges scheduler to notification system without direct bot dependency" + - "Disabled jobs skipped during Start() but remain in DB for re-enabling" + +patterns-established: + - "Scheduler pattern: gocron wrapper with DB persistence and callback-based extensibility" + +requirements-completed: [SCHED-01] + +duration: 2min +completed: 2026-04-06 +--- + +# Phase 17 Plan 02: Scheduler + Storage Summary + +**gocron v2.19.1 wrapper with SQLite persistence for subscribers and scheduled scan jobs, callback-based scan/notify extensibility** + +## Performance + +- **Duration:** 2 min +- **Started:** 2026-04-06T14:25:04Z +- **Completed:** 2026-04-06T14:27:08Z +- **Tasks:** 2 +- **Files modified:** 8 + +## Accomplishments +- Created pkg/scheduler/ package wrapping gocron with Start/Stop lifecycle and DB-backed job persistence +- Implemented full CRUD for subscribers (Add/Remove/List/IsSubscribed) and scheduled_jobs (Save/List/Get/Delete/UpdateLastRun/SetEnabled) +- Added subscribers and scheduled_jobs tables to schema.sql +- All 5 tests pass: storage round-trip, subscriber round-trip, scheduler start/add/remove/run + +## Task Commits + +Each task was committed atomically: + +1. **Task 1: Add gocron dependency, create storage tables, and subscriber/job CRUD** - `c8f7592` (feat) +2. **Task 2 RED: Failing tests for scheduler package** - `89cc133` (test) +3. **Task 2 GREEN: Implement scheduler package** - `c71faa9` (feat) + +## Files Created/Modified +- `pkg/scheduler/scheduler.go` - Scheduler struct wrapping gocron with Start/Stop/AddJob/RemoveJob/RunJob/ListJobs +- `pkg/scheduler/jobs.go` - Job and JobResult types +- `pkg/scheduler/scheduler_test.go` - 5 tests covering storage, subscriber, and scheduler lifecycle +- `pkg/storage/subscribers.go` - Subscriber struct and CRUD methods on DB +- `pkg/storage/scheduled_jobs.go` - ScheduledJob struct and CRUD methods on DB +- `pkg/storage/schema.sql` - subscribers and scheduled_jobs CREATE TABLE statements +- `go.mod` - gocron/v2 v2.19.1 promoted to direct dependency +- `go.sum` - Updated checksums + +## Decisions Made +- ScanFunc callback decouples scheduler from engine -- Plan 17-04 wires real scan logic +- OnComplete callback bridges scheduler to notification system without direct bot dependency +- Disabled jobs skipped during Start() but remain in DB for re-enabling via SetJobEnabled + +## Deviations from Plan + +None - plan executed exactly as written. + +## Issues Encountered +None + +## User Setup Required +None - no external service configuration required. + +## Next Phase Readiness +- pkg/scheduler/ ready for CLI wiring in Plan 17-03 (schedule add/list/remove commands) +- Subscriber storage ready for bot /subscribe handler in Plan 17-01 +- OnComplete callback ready for notification bridge in Plan 17-04 + +--- +*Phase: 17-telegram-scheduler* +*Completed: 2026-04-06*