feat(nadezhda): M4 ranking + digest + Markdown/JSON export
Turn the aggregated store into a ranked, exportable digest of story clusters. - internal/rank: pure deterministic Score(Signals, cfg.Rank) — recency (exp half-life decay), cvss/10, kev, epss, velocity (cluster size/age), source weight, watchlist keyword. Every signal clamped to [0,1]; all weights + half_life + velocity_norm from config, no literals. recency/velocity are total (guard half_life<1 and norm<=0). Rank() scores each cluster and stable-sorts desc, tiebreak freshest. Research 06's two worked examples are the golden-order test (A ~0.99 strictly above B ~0.13). - internal/export: per-cluster (story) digest. Headline = freshest member, distinct sorted outlets, CVEs sorted KEV-first then CVSS. Markdown + deterministic JSON. - store.DigestClusters: 3-query per-cluster aggregation (clusters by since, member articles, union of CVE signals) with explicit ORDER BY; per-cluster CVE dedup via SELECT DISTINCT. - digest command: --top/--since/--format md|json/--out. Off the stub list. - Proven live: scrape -> digest --format md and json render correctly, per-cluster (FortiBleed shows 3 outlets). Suite offline + -race. One read-only audit agent; 0 Crit/High/Med, golden-order math verified; Low/Nit fixed in-phase (top<=0 count, pure-fn guards, ORDER BY, headline sanitize, 2-cluster attach test). Digest ranks per-cluster (Carter's choice), not per-article.
This commit is contained in:
CarterPerez-dev
parent
5865bb6149
commit
8713712bf9
|
|
@ -0,0 +1,101 @@
|
|||
// ©AngelaMos | 2026
|
||||
// digest.go
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/export"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/rank"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultDigestTop = 20
|
||||
formatMarkdown = "md"
|
||||
formatJSON = "json"
|
||||
outFilePerm = 0o644
|
||||
)
|
||||
|
||||
var (
|
||||
digestTop int
|
||||
digestSince string
|
||||
digestFormat string
|
||||
digestOut string
|
||||
)
|
||||
|
||||
var digestCmd = &cobra.Command{
|
||||
Use: "digest",
|
||||
Short: "Render a ranked digest of story clusters to Markdown or JSON",
|
||||
RunE: runDigest,
|
||||
}
|
||||
|
||||
func init() {
|
||||
digestCmd.Flags().IntVar(&digestTop, "top", defaultDigestTop, "show the top N ranked clusters")
|
||||
digestCmd.Flags().StringVar(&digestSince, "since", "", "only clusters active within this window (e.g. 24h, 168h)")
|
||||
digestCmd.Flags().StringVar(&digestFormat, "format", formatMarkdown, "output format: md or json")
|
||||
digestCmd.Flags().StringVar(&digestOut, "out", "", "write to this file instead of stdout")
|
||||
rootCmd.AddCommand(digestCmd)
|
||||
}
|
||||
|
||||
func runDigest(cmd *cobra.Command, args []string) error {
|
||||
cfg, err := loadConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if digestFormat != formatMarkdown && digestFormat != formatJSON {
|
||||
return fmt.Errorf("invalid --format %q: want md or json", digestFormat)
|
||||
}
|
||||
|
||||
var since int64
|
||||
now := time.Now()
|
||||
if digestSince != "" {
|
||||
d, err := time.ParseDuration(digestSince)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid --since %q: %w", digestSince, err)
|
||||
}
|
||||
since = now.Add(-d).Unix()
|
||||
}
|
||||
|
||||
st, err := store.Open(cfg.DBPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer st.Close()
|
||||
|
||||
clusters, err := st.DigestClusters(since)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
scored := rank.Rank(clusters, cfg.Rank, cfg.Watchlist, now)
|
||||
|
||||
shown := len(scored)
|
||||
if digestTop > 0 && digestTop < shown {
|
||||
shown = digestTop
|
||||
}
|
||||
|
||||
var rendered string
|
||||
if digestFormat == formatJSON {
|
||||
rendered, err = export.JSON(scored, digestTop)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
rendered = export.Markdown(scored, digestTop)
|
||||
}
|
||||
|
||||
if digestOut != "" {
|
||||
if err := os.WriteFile(digestOut, []byte(rendered), outFilePerm); err != nil {
|
||||
return fmt.Errorf("write digest to %s: %w", digestOut, err)
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "wrote %d clusters to %s\n", shown, digestOut)
|
||||
return nil
|
||||
}
|
||||
fmt.Fprint(cmd.OutOrStdout(), rendered)
|
||||
return nil
|
||||
}
|
||||
|
|
@ -21,7 +21,6 @@ func init() {
|
|||
short string
|
||||
milestone string
|
||||
}{
|
||||
{"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"},
|
||||
{"tui", "Browse aggregated news in an interactive terminal UI", "milestone M5"},
|
||||
{"ideate", "Generate content angles from ranked clusters via an AI provider", "milestone M6"},
|
||||
{"watch", "Run as a daemon, re-ingesting on an interval", "milestone M7"},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,172 @@
|
|||
// ©AngelaMos | 2026
|
||||
// export.go
|
||||
|
||||
package export
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/rank"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
dateLayout = "2006-01-02"
|
||||
noDate = "----------"
|
||||
scoreRounder = 10000.0
|
||||
digestTitle = "# Nadezhda Digest"
|
||||
)
|
||||
|
||||
type CVEEntry struct {
|
||||
ID string `json:"id"`
|
||||
CVSS *float64 `json:"cvss,omitempty"`
|
||||
EPSS *float64 `json:"epss,omitempty"`
|
||||
KEV bool `json:"kev"`
|
||||
}
|
||||
|
||||
type Entry struct {
|
||||
Rank int `json:"rank"`
|
||||
Score float64 `json:"score"`
|
||||
Headline string `json:"headline"`
|
||||
URL string `json:"url"`
|
||||
Outlets []string `json:"outlets"`
|
||||
FirstSeen string `json:"first_seen"`
|
||||
LastSeen string `json:"last_seen"`
|
||||
CVEs []CVEEntry `json:"cves,omitempty"`
|
||||
}
|
||||
|
||||
type Digest struct {
|
||||
Count int `json:"count"`
|
||||
Entries []Entry `json:"entries"`
|
||||
}
|
||||
|
||||
func Build(scored []rank.Scored, top int) Digest {
|
||||
limit := len(scored)
|
||||
if top > 0 && top < limit {
|
||||
limit = top
|
||||
}
|
||||
entries := make([]Entry, 0, limit)
|
||||
for i := 0; i < limit; i++ {
|
||||
s := scored[i]
|
||||
h := headline(s.Cluster.Articles)
|
||||
entries = append(entries, Entry{
|
||||
Rank: i + 1,
|
||||
Score: roundScore(s.Score),
|
||||
Headline: h.Title,
|
||||
URL: h.CanonicalURL,
|
||||
Outlets: outlets(s.Cluster.Articles),
|
||||
FirstSeen: formatDate(s.Cluster.FirstSeen),
|
||||
LastSeen: formatDate(s.Cluster.LastSeen),
|
||||
CVEs: cveEntries(s.Cluster.CVEs),
|
||||
})
|
||||
}
|
||||
return Digest{Count: len(entries), Entries: entries}
|
||||
}
|
||||
|
||||
func JSON(scored []rank.Scored, top int) (string, error) {
|
||||
b, err := json.MarshalIndent(Build(scored, top), "", " ")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("export json: %w", err)
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
func Markdown(scored []rank.Scored, top int) string {
|
||||
d := Build(scored, top)
|
||||
var b strings.Builder
|
||||
b.WriteString(digestTitle + "\n\n")
|
||||
for _, e := range d.Entries {
|
||||
fmt.Fprintf(&b, "## %d. %s\n", e.Rank, oneLine(e.Headline))
|
||||
fmt.Fprintf(&b, "score %.2f | %d outlet(s): %s\n", e.Score, len(e.Outlets), strings.Join(e.Outlets, ", "))
|
||||
for _, c := range e.CVEs {
|
||||
fmt.Fprintf(&b, "- %s\n", cveLine(c))
|
||||
}
|
||||
fmt.Fprintf(&b, "%s\n\n", e.URL)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func headline(articles []store.DigestArticle) store.DigestArticle {
|
||||
var best store.DigestArticle
|
||||
for i, a := range articles {
|
||||
if i == 0 || a.PublishedAt > best.PublishedAt ||
|
||||
(a.PublishedAt == best.PublishedAt && a.ID < best.ID) {
|
||||
best = a
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
func outlets(articles []store.DigestArticle) []string {
|
||||
seen := make(map[string]struct{}, len(articles))
|
||||
var out []string
|
||||
for _, a := range articles {
|
||||
if _, ok := seen[a.SourceName]; ok {
|
||||
continue
|
||||
}
|
||||
seen[a.SourceName] = struct{}{}
|
||||
out = append(out, a.SourceName)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func cveEntries(cves []store.DigestCVE) []CVEEntry {
|
||||
out := make([]CVEEntry, 0, len(cves))
|
||||
for _, v := range cves {
|
||||
out = append(out, CVEEntry{ID: v.ID, CVSS: v.CVSSScore, EPSS: v.EPSS, KEV: v.IsKEV})
|
||||
}
|
||||
sort.SliceStable(out, func(a, b int) bool {
|
||||
if out[a].KEV != out[b].KEV {
|
||||
return out[a].KEV
|
||||
}
|
||||
if cvssOf(out[a]) != cvssOf(out[b]) {
|
||||
return cvssOf(out[a]) > cvssOf(out[b])
|
||||
}
|
||||
return out[a].ID < out[b].ID
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func cvssOf(c CVEEntry) float64 {
|
||||
if c.CVSS == nil {
|
||||
return -1
|
||||
}
|
||||
return *c.CVSS
|
||||
}
|
||||
|
||||
func cveLine(c CVEEntry) string {
|
||||
var parts []string
|
||||
if c.KEV {
|
||||
parts = append(parts, "KEV")
|
||||
}
|
||||
if c.CVSS != nil {
|
||||
parts = append(parts, fmt.Sprintf("CVSS %.1f", *c.CVSS))
|
||||
}
|
||||
if c.EPSS != nil {
|
||||
parts = append(parts, fmt.Sprintf("EPSS %.2f", *c.EPSS))
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return c.ID
|
||||
}
|
||||
return fmt.Sprintf("%s (%s)", c.ID, strings.Join(parts, ", "))
|
||||
}
|
||||
|
||||
func oneLine(s string) string {
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
|
||||
func roundScore(v float64) float64 {
|
||||
return float64(int64(v*scoreRounder+0.5)) / scoreRounder
|
||||
}
|
||||
|
||||
func formatDate(unix int64) string {
|
||||
if unix == 0 {
|
||||
return noDate
|
||||
}
|
||||
return time.Unix(unix, 0).UTC().Format(dateLayout)
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
// ©AngelaMos | 2026
|
||||
// export_test.go
|
||||
|
||||
package export
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/rank"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/store"
|
||||
)
|
||||
|
||||
func f(v float64) *float64 { return &v }
|
||||
|
||||
func sample() []rank.Scored {
|
||||
return []rank.Scored{
|
||||
{
|
||||
Score: 0.9412,
|
||||
Cluster: store.DigestCluster{
|
||||
Size: 3, FirstSeen: 1751600000, LastSeen: 1751700000,
|
||||
Articles: []store.DigestArticle{
|
||||
{ID: 1, SourceName: "thehackernews", Title: "Old framing", CanonicalURL: "https://thn/1", PublishedAt: 100},
|
||||
{ID: 2, SourceName: "bleepingcomputer", Title: "Freshest framing", CanonicalURL: "https://bc/2", PublishedAt: 300},
|
||||
{ID: 3, SourceName: "bleepingcomputer", Title: "Mid framing", CanonicalURL: "https://bc/3", PublishedAt: 200},
|
||||
},
|
||||
CVEs: []store.DigestCVE{
|
||||
{ID: "CVE-2026-2", CVSSScore: f(7.5), IsKEV: false},
|
||||
{ID: "CVE-2026-1", CVSSScore: f(9.8), EPSS: f(0.91), IsKEV: true},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Score: 0.10,
|
||||
Cluster: store.DigestCluster{
|
||||
Size: 1, FirstSeen: 1751000000, LastSeen: 1751000000,
|
||||
Articles: []store.DigestArticle{{ID: 9, SourceName: "krebs", Title: "Quiet story", CanonicalURL: "https://k/9", PublishedAt: 50}},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHeadlineAndOutlets(t *testing.T) {
|
||||
d := Build(sample(), 0)
|
||||
if d.Count != 2 {
|
||||
t.Fatalf("count = %d, want 2", d.Count)
|
||||
}
|
||||
e := d.Entries[0]
|
||||
if e.Rank != 1 || e.Score != 0.9412 {
|
||||
t.Errorf("rank/score = %d/%v", e.Rank, e.Score)
|
||||
}
|
||||
if e.Headline != "Freshest framing" {
|
||||
t.Errorf("headline = %q, want the freshest member", e.Headline)
|
||||
}
|
||||
if e.URL != "https://bc/2" {
|
||||
t.Errorf("url = %q, want the freshest member's", e.URL)
|
||||
}
|
||||
if len(e.Outlets) != 2 || e.Outlets[0] != "bleepingcomputer" || e.Outlets[1] != "thehackernews" {
|
||||
t.Errorf("outlets = %v, want distinct + sorted", e.Outlets)
|
||||
}
|
||||
if e.CVEs[0].ID != "CVE-2026-1" {
|
||||
t.Errorf("first cve = %q, want the KEV one first", e.CVEs[0].ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildTopLimit(t *testing.T) {
|
||||
d := Build(sample(), 1)
|
||||
if d.Count != 1 {
|
||||
t.Errorf("top=1 should yield 1 entry, got %d", d.Count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONValidAndStable(t *testing.T) {
|
||||
out, err := JSON(sample(), 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var back Digest
|
||||
if err := json.Unmarshal([]byte(out), &back); err != nil {
|
||||
t.Fatalf("output is not valid JSON: %v", err)
|
||||
}
|
||||
if back.Entries[0].Headline != "Freshest framing" {
|
||||
t.Errorf("round-trip headline = %q", back.Entries[0].Headline)
|
||||
}
|
||||
if !back.Entries[0].CVEs[0].KEV {
|
||||
t.Error("first cve should be the KEV one")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkdownContainsSignals(t *testing.T) {
|
||||
md := Markdown(sample(), 0)
|
||||
for _, want := range []string{
|
||||
"# Nadezhda Digest",
|
||||
"## 1. Freshest framing",
|
||||
"2 outlet(s): bleepingcomputer, thehackernews",
|
||||
"CVE-2026-1 (KEV, CVSS 9.8, EPSS 0.91)",
|
||||
"https://bc/2",
|
||||
} {
|
||||
if !strings.Contains(md, want) {
|
||||
t.Errorf("markdown missing %q\n---\n%s", want, md)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,159 @@
|
|||
// ©AngelaMos | 2026
|
||||
// rank.go
|
||||
|
||||
package rank
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/config"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
cvssMax = 10.0
|
||||
minVelocityWindowHours = 1.0
|
||||
secondsPerHour = 3600.0
|
||||
)
|
||||
|
||||
type Signals struct {
|
||||
AgeHours float64
|
||||
MaxCVSS float64
|
||||
KEV bool
|
||||
MaxEPSS float64
|
||||
ClusterSize int
|
||||
ClusterAgeHours float64
|
||||
SourceWeight float64
|
||||
KeywordMatch bool
|
||||
}
|
||||
|
||||
func Score(s Signals, cfg config.Rank) float64 {
|
||||
w := cfg.Weights
|
||||
return w.Recency*recency(s.AgeHours, cfg.HalfLifeHours) +
|
||||
w.CVSS*clamp01(s.MaxCVSS/cvssMax) +
|
||||
w.KEV*boolScore(s.KEV) +
|
||||
w.EPSS*clamp01(s.MaxEPSS) +
|
||||
w.Velocity*velocity(s.ClusterSize, s.ClusterAgeHours, cfg.VelocityNorm) +
|
||||
w.Source*clamp01(s.SourceWeight) +
|
||||
w.Keyword*boolScore(s.KeywordMatch)
|
||||
}
|
||||
|
||||
func recency(ageHours float64, halfLifeHours int) float64 {
|
||||
if ageHours < 0 {
|
||||
ageHours = 0
|
||||
}
|
||||
if halfLifeHours < 1 {
|
||||
halfLifeHours = 1
|
||||
}
|
||||
return math.Exp(-math.Ln2 * ageHours / float64(halfLifeHours))
|
||||
}
|
||||
|
||||
func velocity(size int, ageHours, norm float64) float64 {
|
||||
if size <= 1 || norm <= 0 {
|
||||
return 0
|
||||
}
|
||||
if ageHours < minVelocityWindowHours {
|
||||
ageHours = minVelocityWindowHours
|
||||
}
|
||||
return clamp01((float64(size) / ageHours) / norm)
|
||||
}
|
||||
|
||||
func boolScore(b bool) float64 {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func clamp01(v float64) float64 {
|
||||
if v < 0 {
|
||||
return 0
|
||||
}
|
||||
if v > 1 {
|
||||
return 1
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
type Scored struct {
|
||||
Cluster store.DigestCluster
|
||||
Score float64
|
||||
}
|
||||
|
||||
func Rank(clusters []store.DigestCluster, cfg config.Rank, watchlist []string, now time.Time) []Scored {
|
||||
out := make([]Scored, len(clusters))
|
||||
for i, c := range clusters {
|
||||
out[i] = Scored{Cluster: c, Score: Score(signalsFor(c, watchlist, now), cfg)}
|
||||
}
|
||||
sort.SliceStable(out, func(a, b int) bool {
|
||||
if out[a].Score != out[b].Score {
|
||||
return out[a].Score > out[b].Score
|
||||
}
|
||||
return out[a].Cluster.LastSeen > out[b].Cluster.LastSeen
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func signalsFor(c store.DigestCluster, watchlist []string, now time.Time) Signals {
|
||||
s := Signals{
|
||||
AgeHours: hoursSince(c.LastSeen, now),
|
||||
ClusterSize: c.Size,
|
||||
ClusterAgeHours: float64(c.LastSeen-c.FirstSeen) / secondsPerHour,
|
||||
SourceWeight: maxSourceWeight(c.Articles),
|
||||
KeywordMatch: matchesWatchlist(c, watchlist),
|
||||
}
|
||||
for _, v := range c.CVEs {
|
||||
if v.CVSSScore != nil && *v.CVSSScore > s.MaxCVSS {
|
||||
s.MaxCVSS = *v.CVSSScore
|
||||
}
|
||||
if v.EPSS != nil && *v.EPSS > s.MaxEPSS {
|
||||
s.MaxEPSS = *v.EPSS
|
||||
}
|
||||
if v.IsKEV {
|
||||
s.KEV = true
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func hoursSince(unix int64, now time.Time) float64 {
|
||||
return float64(now.Unix()-unix) / secondsPerHour
|
||||
}
|
||||
|
||||
func maxSourceWeight(articles []store.DigestArticle) float64 {
|
||||
var max float64
|
||||
for _, a := range articles {
|
||||
if a.SourceWeight > max {
|
||||
max = a.SourceWeight
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
func matchesWatchlist(c store.DigestCluster, watchlist []string) bool {
|
||||
if len(watchlist) == 0 {
|
||||
return false
|
||||
}
|
||||
var sb strings.Builder
|
||||
for _, a := range c.Articles {
|
||||
sb.WriteString(strings.ToLower(a.Title))
|
||||
sb.WriteByte(' ')
|
||||
}
|
||||
for _, v := range c.CVEs {
|
||||
sb.WriteString(strings.ToLower(v.ID))
|
||||
sb.WriteByte(' ')
|
||||
}
|
||||
hay := sb.String()
|
||||
for _, term := range watchlist {
|
||||
if term == "" {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(hay, strings.ToLower(term)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
// ©AngelaMos | 2026
|
||||
// rank_test.go
|
||||
|
||||
package rank
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/config"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/store"
|
||||
)
|
||||
|
||||
func f(v float64) *float64 { return &v }
|
||||
|
||||
func TestScoreGoldenOrder(t *testing.T) {
|
||||
cfg := config.Default().Rank
|
||||
|
||||
a := Signals{
|
||||
AgeHours: 2, MaxCVSS: 9.8, KEV: true, MaxEPSS: 0.97,
|
||||
ClusterSize: 5, ClusterAgeHours: 6, SourceWeight: 1.0, KeywordMatch: true,
|
||||
}
|
||||
b := Signals{
|
||||
AgeHours: 120, MaxCVSS: 4.3, KEV: false, MaxEPSS: 0.02,
|
||||
ClusterSize: 1, ClusterAgeHours: 0, SourceWeight: 0.6, KeywordMatch: false,
|
||||
}
|
||||
|
||||
sa, sb := Score(a, cfg), Score(b, cfg)
|
||||
if !(sa > sb) {
|
||||
t.Fatalf("expected A (%.4f) to rank strictly above B (%.4f)", sa, sb)
|
||||
}
|
||||
if sa < 0.95 {
|
||||
t.Errorf("A score = %.4f, want ~0.98", sa)
|
||||
}
|
||||
if sb > 0.20 {
|
||||
t.Errorf("B score = %.4f, want ~0.14", sb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecencyDecay(t *testing.T) {
|
||||
if got := recency(0, 48); math.Abs(got-1.0) > 1e-9 {
|
||||
t.Errorf("recency(0) = %v, want 1.0", got)
|
||||
}
|
||||
if got := recency(48, 48); math.Abs(got-0.5) > 1e-9 {
|
||||
t.Errorf("recency(one half-life) = %v, want 0.5", got)
|
||||
}
|
||||
if got := recency(-5, 48); got != 1.0 {
|
||||
t.Errorf("negative age should clamp to 1.0, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVelocity(t *testing.T) {
|
||||
if v := velocity(1, 0, 0.5); v != 0 {
|
||||
t.Errorf("size-1 velocity = %v, want 0", v)
|
||||
}
|
||||
if v := velocity(6, 12, 0.5); v != 1.0 {
|
||||
t.Errorf("size-6 in 12h should saturate, got %v", v)
|
||||
}
|
||||
if v := velocity(3, 0, 0.5); v != 1.0 {
|
||||
t.Errorf("burst (age<floor) should saturate, got %v", v)
|
||||
}
|
||||
if v := velocity(2, 100, 0.5); math.Abs(v-0.04) > 1e-9 {
|
||||
t.Errorf("slow cluster velocity = %v, want 0.04", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRankOrdersByScore(t *testing.T) {
|
||||
now := time.Unix(1_000_000, 0)
|
||||
hot := store.DigestCluster{
|
||||
ClusterID: 1, Size: 4, FirstSeen: now.Unix() - 3600, LastSeen: now.Unix() - 60,
|
||||
Articles: []store.DigestArticle{{SourceName: "a", SourceWeight: 1.0, PublishedAt: now.Unix() - 60}},
|
||||
CVEs: []store.DigestCVE{{ID: "CVE-2026-1", CVSSScore: f(9.9), EPSS: f(0.95), IsKEV: true}},
|
||||
}
|
||||
cold := store.DigestCluster{
|
||||
ClusterID: 2, Size: 1, FirstSeen: now.Unix() - 500*3600, LastSeen: now.Unix() - 500*3600,
|
||||
Articles: []store.DigestArticle{{SourceName: "b", SourceWeight: 0.5, PublishedAt: now.Unix() - 500*3600}},
|
||||
}
|
||||
|
||||
scored := Rank([]store.DigestCluster{cold, hot}, config.Default().Rank, nil, now)
|
||||
if scored[0].Cluster.ClusterID != 1 {
|
||||
t.Errorf("hot KEV cluster should rank first, got cluster %d", scored[0].Cluster.ClusterID)
|
||||
}
|
||||
if !(scored[0].Score > scored[1].Score) {
|
||||
t.Errorf("scores not descending: %.4f then %.4f", scored[0].Score, scored[1].Score)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeywordMatch(t *testing.T) {
|
||||
c := store.DigestCluster{
|
||||
Articles: []store.DigestArticle{{Title: "Fortinet FortiOS flaw exploited"}},
|
||||
CVEs: []store.DigestCVE{{ID: "CVE-2026-1"}},
|
||||
}
|
||||
if !matchesWatchlist(c, []string{"fortinet"}) {
|
||||
t.Error("case-insensitive watchlist term should match title")
|
||||
}
|
||||
if !matchesWatchlist(c, []string{"CVE-2026-1"}) {
|
||||
t.Error("watchlist should match against CVE ids")
|
||||
}
|
||||
if matchesWatchlist(c, []string{"cisco"}) {
|
||||
t.Error("non-matching term should not match")
|
||||
}
|
||||
if matchesWatchlist(c, nil) {
|
||||
t.Error("empty watchlist should not match")
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,110 @@
|
|||
// ©AngelaMos | 2026
|
||||
// digest_test.go
|
||||
|
||||
package store
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestDigestClustersAggregates(t *testing.T) {
|
||||
s := openTemp(t)
|
||||
|
||||
src1, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1.0, Enabled: true})
|
||||
src2, _ := s.UpsertSource(SourceInput{Name: "b", URL: "https://b/f", Type: "rss", Weight: 0.8, Enabled: true})
|
||||
|
||||
a1, _ := s.InsertArticle(Article{SourceID: src1, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "Story one", PublishedAt: 1000})
|
||||
a2, _ := s.InsertArticle(Article{SourceID: src2, CanonicalURL: "https://b/1", ContentHash: "c2", TitleHash: "t2", Title: "Story two", PublishedAt: 2000})
|
||||
|
||||
if err := s.UpsertCVEStub("CVE-2026-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
score := 9.8
|
||||
if err := s.UpdateCVEEnrichment(CVE{ID: "CVE-2026-1", CVSSScore: &score, IsKEV: true, EnrichedAt: 1, EnrichStatus: EnrichStatusOK}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.LinkArticleCVE(a1, "CVE-2026-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.LinkArticleCVE(a2, "CVE-2026-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := s.ReplaceClusters([]ClusterRow{
|
||||
{Key: "1", Members: []int64{a1, a2}, FirstSeen: 1000, LastSeen: 2000},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
clusters, err := s.DigestClusters(0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(clusters) != 1 {
|
||||
t.Fatalf("clusters = %d, want 1", len(clusters))
|
||||
}
|
||||
c := clusters[0]
|
||||
if len(c.Articles) != 2 {
|
||||
t.Errorf("articles = %d, want 2", len(c.Articles))
|
||||
}
|
||||
if len(c.CVEs) != 1 {
|
||||
t.Fatalf("cves = %d, want 1 (deduped across both articles)", len(c.CVEs))
|
||||
}
|
||||
if !c.CVEs[0].IsKEV || c.CVEs[0].CVSSScore == nil || *c.CVEs[0].CVSSScore != 9.8 {
|
||||
t.Errorf("cve signals not aggregated: %+v", c.CVEs[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDigestClustersAttachToCorrectCluster(t *testing.T) {
|
||||
s := openTemp(t)
|
||||
src, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1, Enabled: true})
|
||||
|
||||
a1, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "one"})
|
||||
a2, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/2", ContentHash: "c2", TitleHash: "t2", Title: "two"})
|
||||
|
||||
for _, id := range []string{"CVE-2026-1", "CVE-2026-2"} {
|
||||
if err := s.UpsertCVEStub(id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_ = s.LinkArticleCVE(a1, "CVE-2026-1")
|
||||
_ = s.LinkArticleCVE(a2, "CVE-2026-2")
|
||||
|
||||
if err := s.ReplaceClusters([]ClusterRow{
|
||||
{Key: "1", Members: []int64{a1}, FirstSeen: 1000, LastSeen: 1000},
|
||||
{Key: "2", Members: []int64{a2}, FirstSeen: 2000, LastSeen: 2000},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
clusters, err := s.DigestClusters(0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(clusters) != 2 {
|
||||
t.Fatalf("clusters = %d, want 2", len(clusters))
|
||||
}
|
||||
byArticle := map[int64]DigestCluster{}
|
||||
for _, c := range clusters {
|
||||
if len(c.Articles) != 1 || len(c.CVEs) != 1 {
|
||||
t.Fatalf("cluster %d: articles=%d cves=%d, want 1/1", c.ClusterID, len(c.Articles), len(c.CVEs))
|
||||
}
|
||||
byArticle[c.Articles[0].ID] = c
|
||||
}
|
||||
if byArticle[a1].CVEs[0].ID != "CVE-2026-1" || byArticle[a2].CVEs[0].ID != "CVE-2026-2" {
|
||||
t.Error("CVEs attached to the wrong cluster")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDigestClustersSinceFilter(t *testing.T) {
|
||||
s := openTemp(t)
|
||||
src, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1, Enabled: true})
|
||||
a1, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "old"})
|
||||
if err := s.ReplaceClusters([]ClusterRow{{Key: "1", Members: []int64{a1}, FirstSeen: 500, LastSeen: 500}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := s.DigestClusters(1000); len(got) != 0 {
|
||||
t.Errorf("cluster with last_seen 500 should be excluded by since=1000, got %d", len(got))
|
||||
}
|
||||
if got, _ := s.DigestClusters(0); len(got) != 1 {
|
||||
t.Errorf("since=0 should include it, got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
|
@ -295,6 +295,122 @@ func (s *Store) ArticlesForCVE(id string) ([]ArticleSummary, error) {
|
|||
return scanArticleSummaries(rows)
|
||||
}
|
||||
|
||||
type DigestArticle struct {
|
||||
ID int64
|
||||
SourceName string
|
||||
SourceWeight float64
|
||||
Title string
|
||||
CanonicalURL string
|
||||
PublishedAt int64
|
||||
}
|
||||
|
||||
type DigestCVE struct {
|
||||
ID string
|
||||
CVSSScore *float64
|
||||
EPSS *float64
|
||||
IsKEV bool
|
||||
}
|
||||
|
||||
type DigestCluster struct {
|
||||
ClusterID int64
|
||||
Key string
|
||||
Size int
|
||||
FirstSeen int64
|
||||
LastSeen int64
|
||||
Articles []DigestArticle
|
||||
CVEs []DigestCVE
|
||||
}
|
||||
|
||||
func (s *Store) DigestClusters(since int64) ([]DigestCluster, error) {
|
||||
byID, order, err := s.digestClusterRows(since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.digestAttachArticles(byID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.digestAttachCVEs(byID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]DigestCluster, 0, len(order))
|
||||
for _, id := range order {
|
||||
out = append(out, *byID[id])
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Store) digestClusterRows(since int64) (map[int64]*DigestCluster, []int64, error) {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT id, cluster_key, size, first_seen, last_seen
|
||||
FROM clusters WHERE last_seen >= ? ORDER BY id`, since)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("digest clusters: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
byID := make(map[int64]*DigestCluster)
|
||||
var order []int64
|
||||
for rows.Next() {
|
||||
var dc DigestCluster
|
||||
if err := rows.Scan(&dc.ClusterID, &dc.Key, &dc.Size, &dc.FirstSeen, &dc.LastSeen); err != nil {
|
||||
return nil, nil, fmt.Errorf("digest clusters: scan: %w", err)
|
||||
}
|
||||
clone := dc
|
||||
byID[dc.ClusterID] = &clone
|
||||
order = append(order, dc.ClusterID)
|
||||
}
|
||||
return byID, order, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) digestAttachArticles(byID map[int64]*DigestCluster) error {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT cm.cluster_id, a.id, s.name, s.weight, a.title, a.canonical_url, a.published_at
|
||||
FROM cluster_members cm
|
||||
JOIN articles a ON a.id = cm.article_id
|
||||
JOIN sources s ON s.id = a.source_id
|
||||
ORDER BY cm.cluster_id, a.id`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("digest articles: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var clusterID int64
|
||||
var a DigestArticle
|
||||
if err := rows.Scan(&clusterID, &a.ID, &a.SourceName, &a.SourceWeight, &a.Title, &a.CanonicalURL, &a.PublishedAt); err != nil {
|
||||
return fmt.Errorf("digest articles: scan: %w", err)
|
||||
}
|
||||
if dc, ok := byID[clusterID]; ok {
|
||||
dc.Articles = append(dc.Articles, a)
|
||||
}
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) digestAttachCVEs(byID map[int64]*DigestCluster) error {
|
||||
rows, err := s.db.Query(`
|
||||
SELECT DISTINCT cm.cluster_id, c.id, c.cvss_score, c.epss, c.is_kev
|
||||
FROM cluster_members cm
|
||||
JOIN article_cves ac ON ac.article_id = cm.article_id
|
||||
JOIN cves c ON c.id = ac.cve_id
|
||||
ORDER BY cm.cluster_id, c.id`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("digest cves: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var clusterID int64
|
||||
var v DigestCVE
|
||||
var isKEV int
|
||||
if err := rows.Scan(&clusterID, &v.ID, &v.CVSSScore, &v.EPSS, &isKEV); err != nil {
|
||||
return fmt.Errorf("digest cves: scan: %w", err)
|
||||
}
|
||||
v.IsKEV = isKEV != 0
|
||||
if dc, ok := byID[clusterID]; ok {
|
||||
dc.CVEs = append(dc.CVEs, v)
|
||||
}
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) UpsertCVEStub(id string) error {
|
||||
_, err := s.db.Exec(`INSERT INTO cves (id) VALUES (?) ON CONFLICT(id) DO NOTHING`, id)
|
||||
if err != nil {
|
||||
|
|
|
|||
Loading…
Reference in New Issue