diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/digest.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/digest.go new file mode 100644 index 00000000..b4aed201 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/digest.go @@ -0,0 +1,101 @@ +// ©AngelaMos | 2026 +// digest.go + +package main + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/export" + "github.com/CarterPerez-dev/nadezhda/internal/rank" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const ( + defaultDigestTop = 20 + formatMarkdown = "md" + formatJSON = "json" + outFilePerm = 0o644 +) + +var ( + digestTop int + digestSince string + digestFormat string + digestOut string +) + +var digestCmd = &cobra.Command{ + Use: "digest", + Short: "Render a ranked digest of story clusters to Markdown or JSON", + RunE: runDigest, +} + +func init() { + digestCmd.Flags().IntVar(&digestTop, "top", defaultDigestTop, "show the top N ranked clusters") + digestCmd.Flags().StringVar(&digestSince, "since", "", "only clusters active within this window (e.g. 24h, 168h)") + digestCmd.Flags().StringVar(&digestFormat, "format", formatMarkdown, "output format: md or json") + digestCmd.Flags().StringVar(&digestOut, "out", "", "write to this file instead of stdout") + rootCmd.AddCommand(digestCmd) +} + +func runDigest(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + if digestFormat != formatMarkdown && digestFormat != formatJSON { + return fmt.Errorf("invalid --format %q: want md or json", digestFormat) + } + + var since int64 + now := time.Now() + if digestSince != "" { + d, err := time.ParseDuration(digestSince) + if err != nil { + return fmt.Errorf("invalid --since %q: %w", digestSince, err) + } + since = now.Add(-d).Unix() + } + + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + + clusters, err := st.DigestClusters(since) + if err != nil { + return err + } + scored := rank.Rank(clusters, cfg.Rank, cfg.Watchlist, now) + + shown := len(scored) + if digestTop > 0 && digestTop < shown { + shown = digestTop + } + + var rendered string + if digestFormat == formatJSON { + rendered, err = export.JSON(scored, digestTop) + if err != nil { + return err + } + } else { + rendered = export.Markdown(scored, digestTop) + } + + if digestOut != "" { + if err := os.WriteFile(digestOut, []byte(rendered), outFilePerm); err != nil { + return fmt.Errorf("write digest to %s: %w", digestOut, err) + } + fmt.Fprintf(cmd.OutOrStdout(), "wrote %d clusters to %s\n", shown, digestOut) + return nil + } + fmt.Fprint(cmd.OutOrStdout(), rendered) + return nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go index c1a4b829..e12b52fe 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go @@ -21,7 +21,6 @@ func init() { short string milestone string }{ - {"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"}, {"tui", "Browse aggregated news in an interactive terminal UI", "milestone M5"}, {"ideate", "Generate content angles from ranked clusters via an AI provider", "milestone M6"}, {"watch", "Run as a daemon, re-ingesting on an interval", "milestone M7"}, diff --git a/PROJECTS/intermediate/security-news-scraper/internal/export/export.go b/PROJECTS/intermediate/security-news-scraper/internal/export/export.go new file mode 100644 index 00000000..38e0a148 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/export/export.go @@ -0,0 +1,172 @@ +// ©AngelaMos | 2026 +// export.go + +package export + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/rank" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const ( + dateLayout = "2006-01-02" + noDate = "----------" + scoreRounder = 10000.0 + digestTitle = "# Nadezhda Digest" +) + +type CVEEntry struct { + ID string `json:"id"` + CVSS *float64 `json:"cvss,omitempty"` + EPSS *float64 `json:"epss,omitempty"` + KEV bool `json:"kev"` +} + +type Entry struct { + Rank int `json:"rank"` + Score float64 `json:"score"` + Headline string `json:"headline"` + URL string `json:"url"` + Outlets []string `json:"outlets"` + FirstSeen string `json:"first_seen"` + LastSeen string `json:"last_seen"` + CVEs []CVEEntry `json:"cves,omitempty"` +} + +type Digest struct { + Count int `json:"count"` + Entries []Entry `json:"entries"` +} + +func Build(scored []rank.Scored, top int) Digest { + limit := len(scored) + if top > 0 && top < limit { + limit = top + } + entries := make([]Entry, 0, limit) + for i := 0; i < limit; i++ { + s := scored[i] + h := headline(s.Cluster.Articles) + entries = append(entries, Entry{ + Rank: i + 1, + Score: roundScore(s.Score), + Headline: h.Title, + URL: h.CanonicalURL, + Outlets: outlets(s.Cluster.Articles), + FirstSeen: formatDate(s.Cluster.FirstSeen), + LastSeen: formatDate(s.Cluster.LastSeen), + CVEs: cveEntries(s.Cluster.CVEs), + }) + } + return Digest{Count: len(entries), Entries: entries} +} + +func JSON(scored []rank.Scored, top int) (string, error) { + b, err := json.MarshalIndent(Build(scored, top), "", " ") + if err != nil { + return "", fmt.Errorf("export json: %w", err) + } + return string(b), nil +} + +func Markdown(scored []rank.Scored, top int) string { + d := Build(scored, top) + var b strings.Builder + b.WriteString(digestTitle + "\n\n") + for _, e := range d.Entries { + fmt.Fprintf(&b, "## %d. %s\n", e.Rank, oneLine(e.Headline)) + fmt.Fprintf(&b, "score %.2f | %d outlet(s): %s\n", e.Score, len(e.Outlets), strings.Join(e.Outlets, ", ")) + for _, c := range e.CVEs { + fmt.Fprintf(&b, "- %s\n", cveLine(c)) + } + fmt.Fprintf(&b, "%s\n\n", e.URL) + } + return b.String() +} + +func headline(articles []store.DigestArticle) store.DigestArticle { + var best store.DigestArticle + for i, a := range articles { + if i == 0 || a.PublishedAt > best.PublishedAt || + (a.PublishedAt == best.PublishedAt && a.ID < best.ID) { + best = a + } + } + return best +} + +func outlets(articles []store.DigestArticle) []string { + seen := make(map[string]struct{}, len(articles)) + var out []string + for _, a := range articles { + if _, ok := seen[a.SourceName]; ok { + continue + } + seen[a.SourceName] = struct{}{} + out = append(out, a.SourceName) + } + sort.Strings(out) + return out +} + +func cveEntries(cves []store.DigestCVE) []CVEEntry { + out := make([]CVEEntry, 0, len(cves)) + for _, v := range cves { + out = append(out, CVEEntry{ID: v.ID, CVSS: v.CVSSScore, EPSS: v.EPSS, KEV: v.IsKEV}) + } + sort.SliceStable(out, func(a, b int) bool { + if out[a].KEV != out[b].KEV { + return out[a].KEV + } + if cvssOf(out[a]) != cvssOf(out[b]) { + return cvssOf(out[a]) > cvssOf(out[b]) + } + return out[a].ID < out[b].ID + }) + return out +} + +func cvssOf(c CVEEntry) float64 { + if c.CVSS == nil { + return -1 + } + return *c.CVSS +} + +func cveLine(c CVEEntry) string { + var parts []string + if c.KEV { + parts = append(parts, "KEV") + } + if c.CVSS != nil { + parts = append(parts, fmt.Sprintf("CVSS %.1f", *c.CVSS)) + } + if c.EPSS != nil { + parts = append(parts, fmt.Sprintf("EPSS %.2f", *c.EPSS)) + } + if len(parts) == 0 { + return c.ID + } + return fmt.Sprintf("%s (%s)", c.ID, strings.Join(parts, ", ")) +} + +func oneLine(s string) string { + return strings.Join(strings.Fields(s), " ") +} + +func roundScore(v float64) float64 { + return float64(int64(v*scoreRounder+0.5)) / scoreRounder +} + +func formatDate(unix int64) string { + if unix == 0 { + return noDate + } + return time.Unix(unix, 0).UTC().Format(dateLayout) +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/export/export_test.go b/PROJECTS/intermediate/security-news-scraper/internal/export/export_test.go new file mode 100644 index 00000000..88c499fc --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/export/export_test.go @@ -0,0 +1,104 @@ +// ©AngelaMos | 2026 +// export_test.go + +package export + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/CarterPerez-dev/nadezhda/internal/rank" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +func f(v float64) *float64 { return &v } + +func sample() []rank.Scored { + return []rank.Scored{ + { + Score: 0.9412, + Cluster: store.DigestCluster{ + Size: 3, FirstSeen: 1751600000, LastSeen: 1751700000, + Articles: []store.DigestArticle{ + {ID: 1, SourceName: "thehackernews", Title: "Old framing", CanonicalURL: "https://thn/1", PublishedAt: 100}, + {ID: 2, SourceName: "bleepingcomputer", Title: "Freshest framing", CanonicalURL: "https://bc/2", PublishedAt: 300}, + {ID: 3, SourceName: "bleepingcomputer", Title: "Mid framing", CanonicalURL: "https://bc/3", PublishedAt: 200}, + }, + CVEs: []store.DigestCVE{ + {ID: "CVE-2026-2", CVSSScore: f(7.5), IsKEV: false}, + {ID: "CVE-2026-1", CVSSScore: f(9.8), EPSS: f(0.91), IsKEV: true}, + }, + }, + }, + { + Score: 0.10, + Cluster: store.DigestCluster{ + Size: 1, FirstSeen: 1751000000, LastSeen: 1751000000, + Articles: []store.DigestArticle{{ID: 9, SourceName: "krebs", Title: "Quiet story", CanonicalURL: "https://k/9", PublishedAt: 50}}, + }, + }, + } +} + +func TestBuildHeadlineAndOutlets(t *testing.T) { + d := Build(sample(), 0) + if d.Count != 2 { + t.Fatalf("count = %d, want 2", d.Count) + } + e := d.Entries[0] + if e.Rank != 1 || e.Score != 0.9412 { + t.Errorf("rank/score = %d/%v", e.Rank, e.Score) + } + if e.Headline != "Freshest framing" { + t.Errorf("headline = %q, want the freshest member", e.Headline) + } + if e.URL != "https://bc/2" { + t.Errorf("url = %q, want the freshest member's", e.URL) + } + if len(e.Outlets) != 2 || e.Outlets[0] != "bleepingcomputer" || e.Outlets[1] != "thehackernews" { + t.Errorf("outlets = %v, want distinct + sorted", e.Outlets) + } + if e.CVEs[0].ID != "CVE-2026-1" { + t.Errorf("first cve = %q, want the KEV one first", e.CVEs[0].ID) + } +} + +func TestBuildTopLimit(t *testing.T) { + d := Build(sample(), 1) + if d.Count != 1 { + t.Errorf("top=1 should yield 1 entry, got %d", d.Count) + } +} + +func TestJSONValidAndStable(t *testing.T) { + out, err := JSON(sample(), 0) + if err != nil { + t.Fatal(err) + } + var back Digest + if err := json.Unmarshal([]byte(out), &back); err != nil { + t.Fatalf("output is not valid JSON: %v", err) + } + if back.Entries[0].Headline != "Freshest framing" { + t.Errorf("round-trip headline = %q", back.Entries[0].Headline) + } + if !back.Entries[0].CVEs[0].KEV { + t.Error("first cve should be the KEV one") + } +} + +func TestMarkdownContainsSignals(t *testing.T) { + md := Markdown(sample(), 0) + for _, want := range []string{ + "# Nadezhda Digest", + "## 1. Freshest framing", + "2 outlet(s): bleepingcomputer, thehackernews", + "CVE-2026-1 (KEV, CVSS 9.8, EPSS 0.91)", + "https://bc/2", + } { + if !strings.Contains(md, want) { + t.Errorf("markdown missing %q\n---\n%s", want, md) + } + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/rank/rank.go b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank.go new file mode 100644 index 00000000..04f2bf33 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank.go @@ -0,0 +1,159 @@ +// ©AngelaMos | 2026 +// rank.go + +package rank + +import ( + "math" + "sort" + "strings" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const ( + cvssMax = 10.0 + minVelocityWindowHours = 1.0 + secondsPerHour = 3600.0 +) + +type Signals struct { + AgeHours float64 + MaxCVSS float64 + KEV bool + MaxEPSS float64 + ClusterSize int + ClusterAgeHours float64 + SourceWeight float64 + KeywordMatch bool +} + +func Score(s Signals, cfg config.Rank) float64 { + w := cfg.Weights + return w.Recency*recency(s.AgeHours, cfg.HalfLifeHours) + + w.CVSS*clamp01(s.MaxCVSS/cvssMax) + + w.KEV*boolScore(s.KEV) + + w.EPSS*clamp01(s.MaxEPSS) + + w.Velocity*velocity(s.ClusterSize, s.ClusterAgeHours, cfg.VelocityNorm) + + w.Source*clamp01(s.SourceWeight) + + w.Keyword*boolScore(s.KeywordMatch) +} + +func recency(ageHours float64, halfLifeHours int) float64 { + if ageHours < 0 { + ageHours = 0 + } + if halfLifeHours < 1 { + halfLifeHours = 1 + } + return math.Exp(-math.Ln2 * ageHours / float64(halfLifeHours)) +} + +func velocity(size int, ageHours, norm float64) float64 { + if size <= 1 || norm <= 0 { + return 0 + } + if ageHours < minVelocityWindowHours { + ageHours = minVelocityWindowHours + } + return clamp01((float64(size) / ageHours) / norm) +} + +func boolScore(b bool) float64 { + if b { + return 1 + } + return 0 +} + +func clamp01(v float64) float64 { + if v < 0 { + return 0 + } + if v > 1 { + return 1 + } + return v +} + +type Scored struct { + Cluster store.DigestCluster + Score float64 +} + +func Rank(clusters []store.DigestCluster, cfg config.Rank, watchlist []string, now time.Time) []Scored { + out := make([]Scored, len(clusters)) + for i, c := range clusters { + out[i] = Scored{Cluster: c, Score: Score(signalsFor(c, watchlist, now), cfg)} + } + sort.SliceStable(out, func(a, b int) bool { + if out[a].Score != out[b].Score { + return out[a].Score > out[b].Score + } + return out[a].Cluster.LastSeen > out[b].Cluster.LastSeen + }) + return out +} + +func signalsFor(c store.DigestCluster, watchlist []string, now time.Time) Signals { + s := Signals{ + AgeHours: hoursSince(c.LastSeen, now), + ClusterSize: c.Size, + ClusterAgeHours: float64(c.LastSeen-c.FirstSeen) / secondsPerHour, + SourceWeight: maxSourceWeight(c.Articles), + KeywordMatch: matchesWatchlist(c, watchlist), + } + for _, v := range c.CVEs { + if v.CVSSScore != nil && *v.CVSSScore > s.MaxCVSS { + s.MaxCVSS = *v.CVSSScore + } + if v.EPSS != nil && *v.EPSS > s.MaxEPSS { + s.MaxEPSS = *v.EPSS + } + if v.IsKEV { + s.KEV = true + } + } + return s +} + +func hoursSince(unix int64, now time.Time) float64 { + return float64(now.Unix()-unix) / secondsPerHour +} + +func maxSourceWeight(articles []store.DigestArticle) float64 { + var max float64 + for _, a := range articles { + if a.SourceWeight > max { + max = a.SourceWeight + } + } + return max +} + +func matchesWatchlist(c store.DigestCluster, watchlist []string) bool { + if len(watchlist) == 0 { + return false + } + var sb strings.Builder + for _, a := range c.Articles { + sb.WriteString(strings.ToLower(a.Title)) + sb.WriteByte(' ') + } + for _, v := range c.CVEs { + sb.WriteString(strings.ToLower(v.ID)) + sb.WriteByte(' ') + } + hay := sb.String() + for _, term := range watchlist { + if term == "" { + continue + } + if strings.Contains(hay, strings.ToLower(term)) { + return true + } + } + return false +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/rank/rank_test.go b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank_test.go new file mode 100644 index 00000000..1748bc13 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank_test.go @@ -0,0 +1,106 @@ +// ©AngelaMos | 2026 +// rank_test.go + +package rank + +import ( + "math" + "testing" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +func f(v float64) *float64 { return &v } + +func TestScoreGoldenOrder(t *testing.T) { + cfg := config.Default().Rank + + a := Signals{ + AgeHours: 2, MaxCVSS: 9.8, KEV: true, MaxEPSS: 0.97, + ClusterSize: 5, ClusterAgeHours: 6, SourceWeight: 1.0, KeywordMatch: true, + } + b := Signals{ + AgeHours: 120, MaxCVSS: 4.3, KEV: false, MaxEPSS: 0.02, + ClusterSize: 1, ClusterAgeHours: 0, SourceWeight: 0.6, KeywordMatch: false, + } + + sa, sb := Score(a, cfg), Score(b, cfg) + if !(sa > sb) { + t.Fatalf("expected A (%.4f) to rank strictly above B (%.4f)", sa, sb) + } + if sa < 0.95 { + t.Errorf("A score = %.4f, want ~0.98", sa) + } + if sb > 0.20 { + t.Errorf("B score = %.4f, want ~0.14", sb) + } +} + +func TestRecencyDecay(t *testing.T) { + if got := recency(0, 48); math.Abs(got-1.0) > 1e-9 { + t.Errorf("recency(0) = %v, want 1.0", got) + } + if got := recency(48, 48); math.Abs(got-0.5) > 1e-9 { + t.Errorf("recency(one half-life) = %v, want 0.5", got) + } + if got := recency(-5, 48); got != 1.0 { + t.Errorf("negative age should clamp to 1.0, got %v", got) + } +} + +func TestVelocity(t *testing.T) { + if v := velocity(1, 0, 0.5); v != 0 { + t.Errorf("size-1 velocity = %v, want 0", v) + } + if v := velocity(6, 12, 0.5); v != 1.0 { + t.Errorf("size-6 in 12h should saturate, got %v", v) + } + if v := velocity(3, 0, 0.5); v != 1.0 { + t.Errorf("burst (age 1e-9 { + t.Errorf("slow cluster velocity = %v, want 0.04", v) + } +} + +func TestRankOrdersByScore(t *testing.T) { + now := time.Unix(1_000_000, 0) + hot := store.DigestCluster{ + ClusterID: 1, Size: 4, FirstSeen: now.Unix() - 3600, LastSeen: now.Unix() - 60, + Articles: []store.DigestArticle{{SourceName: "a", SourceWeight: 1.0, PublishedAt: now.Unix() - 60}}, + CVEs: []store.DigestCVE{{ID: "CVE-2026-1", CVSSScore: f(9.9), EPSS: f(0.95), IsKEV: true}}, + } + cold := store.DigestCluster{ + ClusterID: 2, Size: 1, FirstSeen: now.Unix() - 500*3600, LastSeen: now.Unix() - 500*3600, + Articles: []store.DigestArticle{{SourceName: "b", SourceWeight: 0.5, PublishedAt: now.Unix() - 500*3600}}, + } + + scored := Rank([]store.DigestCluster{cold, hot}, config.Default().Rank, nil, now) + if scored[0].Cluster.ClusterID != 1 { + t.Errorf("hot KEV cluster should rank first, got cluster %d", scored[0].Cluster.ClusterID) + } + if !(scored[0].Score > scored[1].Score) { + t.Errorf("scores not descending: %.4f then %.4f", scored[0].Score, scored[1].Score) + } +} + +func TestKeywordMatch(t *testing.T) { + c := store.DigestCluster{ + Articles: []store.DigestArticle{{Title: "Fortinet FortiOS flaw exploited"}}, + CVEs: []store.DigestCVE{{ID: "CVE-2026-1"}}, + } + if !matchesWatchlist(c, []string{"fortinet"}) { + t.Error("case-insensitive watchlist term should match title") + } + if !matchesWatchlist(c, []string{"CVE-2026-1"}) { + t.Error("watchlist should match against CVE ids") + } + if matchesWatchlist(c, []string{"cisco"}) { + t.Error("non-matching term should not match") + } + if matchesWatchlist(c, nil) { + t.Error("empty watchlist should not match") + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/digest_test.go b/PROJECTS/intermediate/security-news-scraper/internal/store/digest_test.go new file mode 100644 index 00000000..b6230eba --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/digest_test.go @@ -0,0 +1,110 @@ +// ©AngelaMos | 2026 +// digest_test.go + +package store + +import "testing" + +func TestDigestClustersAggregates(t *testing.T) { + s := openTemp(t) + + src1, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1.0, Enabled: true}) + src2, _ := s.UpsertSource(SourceInput{Name: "b", URL: "https://b/f", Type: "rss", Weight: 0.8, Enabled: true}) + + a1, _ := s.InsertArticle(Article{SourceID: src1, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "Story one", PublishedAt: 1000}) + a2, _ := s.InsertArticle(Article{SourceID: src2, CanonicalURL: "https://b/1", ContentHash: "c2", TitleHash: "t2", Title: "Story two", PublishedAt: 2000}) + + if err := s.UpsertCVEStub("CVE-2026-1"); err != nil { + t.Fatal(err) + } + score := 9.8 + if err := s.UpdateCVEEnrichment(CVE{ID: "CVE-2026-1", CVSSScore: &score, IsKEV: true, EnrichedAt: 1, EnrichStatus: EnrichStatusOK}); err != nil { + t.Fatal(err) + } + if err := s.LinkArticleCVE(a1, "CVE-2026-1"); err != nil { + t.Fatal(err) + } + if err := s.LinkArticleCVE(a2, "CVE-2026-1"); err != nil { + t.Fatal(err) + } + + if err := s.ReplaceClusters([]ClusterRow{ + {Key: "1", Members: []int64{a1, a2}, FirstSeen: 1000, LastSeen: 2000}, + }); err != nil { + t.Fatal(err) + } + + clusters, err := s.DigestClusters(0) + if err != nil { + t.Fatal(err) + } + if len(clusters) != 1 { + t.Fatalf("clusters = %d, want 1", len(clusters)) + } + c := clusters[0] + if len(c.Articles) != 2 { + t.Errorf("articles = %d, want 2", len(c.Articles)) + } + if len(c.CVEs) != 1 { + t.Fatalf("cves = %d, want 1 (deduped across both articles)", len(c.CVEs)) + } + if !c.CVEs[0].IsKEV || c.CVEs[0].CVSSScore == nil || *c.CVEs[0].CVSSScore != 9.8 { + t.Errorf("cve signals not aggregated: %+v", c.CVEs[0]) + } +} + +func TestDigestClustersAttachToCorrectCluster(t *testing.T) { + s := openTemp(t) + src, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1, Enabled: true}) + + a1, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "one"}) + a2, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/2", ContentHash: "c2", TitleHash: "t2", Title: "two"}) + + for _, id := range []string{"CVE-2026-1", "CVE-2026-2"} { + if err := s.UpsertCVEStub(id); err != nil { + t.Fatal(err) + } + } + _ = s.LinkArticleCVE(a1, "CVE-2026-1") + _ = s.LinkArticleCVE(a2, "CVE-2026-2") + + if err := s.ReplaceClusters([]ClusterRow{ + {Key: "1", Members: []int64{a1}, FirstSeen: 1000, LastSeen: 1000}, + {Key: "2", Members: []int64{a2}, FirstSeen: 2000, LastSeen: 2000}, + }); err != nil { + t.Fatal(err) + } + + clusters, err := s.DigestClusters(0) + if err != nil { + t.Fatal(err) + } + if len(clusters) != 2 { + t.Fatalf("clusters = %d, want 2", len(clusters)) + } + byArticle := map[int64]DigestCluster{} + for _, c := range clusters { + if len(c.Articles) != 1 || len(c.CVEs) != 1 { + t.Fatalf("cluster %d: articles=%d cves=%d, want 1/1", c.ClusterID, len(c.Articles), len(c.CVEs)) + } + byArticle[c.Articles[0].ID] = c + } + if byArticle[a1].CVEs[0].ID != "CVE-2026-1" || byArticle[a2].CVEs[0].ID != "CVE-2026-2" { + t.Error("CVEs attached to the wrong cluster") + } +} + +func TestDigestClustersSinceFilter(t *testing.T) { + s := openTemp(t) + src, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1, Enabled: true}) + a1, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "old"}) + if err := s.ReplaceClusters([]ClusterRow{{Key: "1", Members: []int64{a1}, FirstSeen: 500, LastSeen: 500}}); err != nil { + t.Fatal(err) + } + if got, _ := s.DigestClusters(1000); len(got) != 0 { + t.Errorf("cluster with last_seen 500 should be excluded by since=1000, got %d", len(got)) + } + if got, _ := s.DigestClusters(0); len(got) != 1 { + t.Errorf("since=0 should include it, got %d", len(got)) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go index b65e681e..e28bb698 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go @@ -295,6 +295,122 @@ func (s *Store) ArticlesForCVE(id string) ([]ArticleSummary, error) { return scanArticleSummaries(rows) } +type DigestArticle struct { + ID int64 + SourceName string + SourceWeight float64 + Title string + CanonicalURL string + PublishedAt int64 +} + +type DigestCVE struct { + ID string + CVSSScore *float64 + EPSS *float64 + IsKEV bool +} + +type DigestCluster struct { + ClusterID int64 + Key string + Size int + FirstSeen int64 + LastSeen int64 + Articles []DigestArticle + CVEs []DigestCVE +} + +func (s *Store) DigestClusters(since int64) ([]DigestCluster, error) { + byID, order, err := s.digestClusterRows(since) + if err != nil { + return nil, err + } + if err := s.digestAttachArticles(byID); err != nil { + return nil, err + } + if err := s.digestAttachCVEs(byID); err != nil { + return nil, err + } + out := make([]DigestCluster, 0, len(order)) + for _, id := range order { + out = append(out, *byID[id]) + } + return out, nil +} + +func (s *Store) digestClusterRows(since int64) (map[int64]*DigestCluster, []int64, error) { + rows, err := s.db.Query(` + SELECT id, cluster_key, size, first_seen, last_seen + FROM clusters WHERE last_seen >= ? ORDER BY id`, since) + if err != nil { + return nil, nil, fmt.Errorf("digest clusters: %w", err) + } + defer rows.Close() + byID := make(map[int64]*DigestCluster) + var order []int64 + for rows.Next() { + var dc DigestCluster + if err := rows.Scan(&dc.ClusterID, &dc.Key, &dc.Size, &dc.FirstSeen, &dc.LastSeen); err != nil { + return nil, nil, fmt.Errorf("digest clusters: scan: %w", err) + } + clone := dc + byID[dc.ClusterID] = &clone + order = append(order, dc.ClusterID) + } + return byID, order, rows.Err() +} + +func (s *Store) digestAttachArticles(byID map[int64]*DigestCluster) error { + rows, err := s.db.Query(` + SELECT cm.cluster_id, a.id, s.name, s.weight, a.title, a.canonical_url, a.published_at + FROM cluster_members cm + JOIN articles a ON a.id = cm.article_id + JOIN sources s ON s.id = a.source_id + ORDER BY cm.cluster_id, a.id`) + if err != nil { + return fmt.Errorf("digest articles: %w", err) + } + defer rows.Close() + for rows.Next() { + var clusterID int64 + var a DigestArticle + if err := rows.Scan(&clusterID, &a.ID, &a.SourceName, &a.SourceWeight, &a.Title, &a.CanonicalURL, &a.PublishedAt); err != nil { + return fmt.Errorf("digest articles: scan: %w", err) + } + if dc, ok := byID[clusterID]; ok { + dc.Articles = append(dc.Articles, a) + } + } + return rows.Err() +} + +func (s *Store) digestAttachCVEs(byID map[int64]*DigestCluster) error { + rows, err := s.db.Query(` + SELECT DISTINCT cm.cluster_id, c.id, c.cvss_score, c.epss, c.is_kev + FROM cluster_members cm + JOIN article_cves ac ON ac.article_id = cm.article_id + JOIN cves c ON c.id = ac.cve_id + ORDER BY cm.cluster_id, c.id`) + if err != nil { + return fmt.Errorf("digest cves: %w", err) + } + defer rows.Close() + for rows.Next() { + var clusterID int64 + var v DigestCVE + var isKEV int + if err := rows.Scan(&clusterID, &v.ID, &v.CVSSScore, &v.EPSS, &isKEV); err != nil { + return fmt.Errorf("digest cves: scan: %w", err) + } + v.IsKEV = isKEV != 0 + if dc, ok := byID[clusterID]; ok { + dc.CVEs = append(dc.CVEs, v) + } + } + return rows.Err() +} + func (s *Store) UpsertCVEStub(id string) error { _, err := s.db.Exec(`INSERT INTO cves (id) VALUES (?) ON CONFLICT(id) DO NOTHING`, id) if err != nil {