From 8b8eaafa1f0440611dcbfc7a5ea8381f2a3e4054 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Sun, 5 Jul 2026 13:22:08 -0400 Subject: [PATCH 01/15] feat(nadezhda): M0 scaffold for security news + CVE aggregator Go engine that aggregates cybersecurity news and enriches CVEs. This first commit lays the M0 foundation: - config: fetch/enrich/cluster/rank/AI settings, YAML + validation, defaults as named constants, AI opt-in and off by default - source: Source registry with embedded default feeds (7 verified RSS sources), external-file override, dedup/URL validation - store: pure-Go SQLite (modernc), WAL + foreign_keys, forward-only embedded migrations with schema_migrations tracking and a loud newer-than-binary guard; ErrDuplicate sentinel for dedup - cmd: cobra skeleton, version + sources wired, remaining commands stubbed to their milestones go vet / gofmt / go test all clean. --- .../security-news-scraper/.gitignore | 24 ++ .../security-news-scraper/README.md | 35 +++ .../cmd/nadezhda/main.go | 12 + .../cmd/nadezhda/root.go | 38 ++++ .../cmd/nadezhda/sources.go | 60 +++++ .../cmd/nadezhda/stubs.go | 39 ++++ .../cmd/nadezhda/version.go | 25 +++ .../intermediate/security-news-scraper/go.mod | 23 ++ .../intermediate/security-news-scraper/go.sum | 64 ++++++ .../internal/config/config.go | 206 ++++++++++++++++++ .../internal/config/config_test.go | 94 ++++++++ .../internal/source/source.go | 92 ++++++++ .../internal/source/source_test.go | 103 +++++++++ .../internal/source/sources.yaml | 58 +++++ .../internal/store/migrate.go | 103 +++++++++ .../internal/store/migrations/0001_init.sql | 87 ++++++++ .../internal/store/store.go | 149 +++++++++++++ .../internal/store/store_test.go | 116 ++++++++++ .../internal/version/version.go | 9 + .../security-news-scraper/justfile | 33 +++ 20 files changed, 1370 insertions(+) create mode 100644 PROJECTS/intermediate/security-news-scraper/.gitignore create mode 100644 PROJECTS/intermediate/security-news-scraper/README.md create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/root.go create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/sources.go create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/version.go create mode 100644 PROJECTS/intermediate/security-news-scraper/go.mod create mode 100644 PROJECTS/intermediate/security-news-scraper/go.sum create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/config/config.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/config/config_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/source/source.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/source/source_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/source/sources.yaml create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/migrate.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0001_init.sql create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/store.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/store_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/version/version.go create mode 100644 PROJECTS/intermediate/security-news-scraper/justfile diff --git a/PROJECTS/intermediate/security-news-scraper/.gitignore b/PROJECTS/intermediate/security-news-scraper/.gitignore new file mode 100644 index 00000000..38335067 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/.gitignore @@ -0,0 +1,24 @@ +# ©AngelaMos | 2026 +# .gitignore + +# private dev + research material (never public) +docs/ + +# Go build artifacts +/nadezhda +/bin/ +*.test +*.out +*.prof + +# local runtime data (SQLite store + WAL sidecars) +*.db +*.db-wal +*.db-shm +*.sqlite +/data/ + +# env / secrets / local overrides +.env +*.local.yaml +config.local.yaml diff --git a/PROJECTS/intermediate/security-news-scraper/README.md b/PROJECTS/intermediate/security-news-scraper/README.md new file mode 100644 index 00000000..f7e62dff --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/README.md @@ -0,0 +1,35 @@ +# Nadezhda + +A concurrent security-news and CVE aggregation engine, written in Go. + +Nadezhda ingests cybersecurity news from reliable RSS feeds, enriches every +referenced CVE with authoritative exploit intelligence (NVD, CISA KEV, FIRST +EPSS), clusters the same story across outlets, ranks items by real-world +significance, and surfaces content angles. It ships as a single static binary +with a local SQLite store, a colorful terminal UI, and Markdown/JSON export. + +## Status + +Early development. The scaffold is in place: configuration, source registry, +SQLite store with forward-only migrations, and the command skeleton. + +``` +nadezhda version # print version +nadezhda sources # list configured feeds and persist them to the store +``` + +Ingestion, CVE enrichment, ranking, the TUI, and the AI ideation layer land in +subsequent milestones. + +## Build + +``` +just build # -> ./nadezhda +just test +``` + +Requires Go 1.25+. + +--- + +Full documentation lands in `learn/` as the project matures. diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go new file mode 100644 index 00000000..a4c7e183 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go @@ -0,0 +1,12 @@ +// ©AngelaMos | 2026 +// main.go + +package main + +import "os" + +func main() { + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/root.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/root.go new file mode 100644 index 00000000..50cd452d --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/root.go @@ -0,0 +1,38 @@ +// ©AngelaMos | 2026 +// root.go + +package main + +import ( + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/config" +) + +var ( + flagConfig string + flagDB string +) + +var rootCmd = &cobra.Command{ + Use: "nadezhda", + Short: "Security news and CVE aggregation engine", + Long: "Nadezhda aggregates cybersecurity news, enriches CVEs with NVD, CISA KEV, and EPSS intelligence, clusters stories across outlets, and ranks what matters.", + SilenceUsage: true, +} + +func init() { + rootCmd.PersistentFlags().StringVar(&flagConfig, "config", "", "path to config.yaml (optional)") + rootCmd.PersistentFlags().StringVar(&flagDB, "db", "", "override database path") +} + +func loadConfig() (config.Config, error) { + cfg, err := config.Load(flagConfig) + if err != nil { + return config.Config{}, err + } + if flagDB != "" { + cfg.DBPath = flagDB + } + return cfg, nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/sources.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/sources.go new file mode 100644 index 00000000..0c902d8f --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/sources.go @@ -0,0 +1,60 @@ +// ©AngelaMos | 2026 +// sources.go + +package main + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/source" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +var sourcesCmd = &cobra.Command{ + Use: "sources", + Short: "List configured sources and persist them to the store", + RunE: runSources, +} + +func init() { + rootCmd.AddCommand(sourcesCmd) +} + +func runSources(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + srcs, err := source.Load(cfg.SourcesPath) + if err != nil { + return err + } + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + + for _, s := range srcs { + if _, err := st.UpsertSource(store.SourceInput{ + Name: s.Name, Title: s.Title, URL: s.URL, Type: string(s.Type), + Weight: s.Weight, Tags: s.Tags, Enabled: s.Enabled, + }); err != nil { + return err + } + } + + fmt.Printf("%-18s %-8s %-7s %s\n", "NAME", "ENABLED", "WEIGHT", "URL") + for _, s := range srcs { + enabled := "no" + if s.Enabled { + enabled = "yes" + } + fmt.Printf("%-18s %-8s %-7.2f %s\n", s.Name, enabled, s.Weight, s.URL) + } + fmt.Printf("\n%d sources (%d enabled) persisted to %s\n", + len(srcs), len(source.Enabled(srcs)), cfg.DBPath) + return nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go new file mode 100644 index 00000000..2f9eb03e --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go @@ -0,0 +1,39 @@ +// ©AngelaMos | 2026 +// stubs.go + +package main + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +func notImplemented(milestone string) func(*cobra.Command, []string) error { + return func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("%s is not implemented yet (%s)", cmd.Name(), milestone) + } +} + +func init() { + stubs := []struct { + use string + short string + milestone string + }{ + {"scrape", "Ingest all enabled sources once", "milestone M1"}, + {"list", "List stored articles with filters", "milestone M3"}, + {"cve", "Show an enriched CVE and articles mentioning it", "milestone M3"}, + {"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"}, + {"tui", "Browse aggregated news in an interactive terminal UI", "milestone M5"}, + {"ideate", "Generate content angles from ranked clusters via an AI provider", "milestone M6"}, + {"watch", "Run as a daemon, re-ingesting on an interval", "milestone M7"}, + } + for _, s := range stubs { + rootCmd.AddCommand(&cobra.Command{ + Use: s.use, + Short: s.short, + RunE: notImplemented(s.milestone), + }) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/version.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/version.go new file mode 100644 index 00000000..7192b63e --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/version.go @@ -0,0 +1,25 @@ +// ©AngelaMos | 2026 +// version.go + +package main + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/version" +) + +var versionCmd = &cobra.Command{ + Use: "version", + Short: "Print version", + RunE: func(cmd *cobra.Command, args []string) error { + fmt.Printf("%s %s\n", version.Name, version.Version) + return nil + }, +} + +func init() { + rootCmd.AddCommand(versionCmd) +} diff --git a/PROJECTS/intermediate/security-news-scraper/go.mod b/PROJECTS/intermediate/security-news-scraper/go.mod new file mode 100644 index 00000000..0f7c5876 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/go.mod @@ -0,0 +1,23 @@ +module github.com/CarterPerez-dev/nadezhda + +go 1.25.0 + +require ( + github.com/spf13/cobra v1.10.2 + gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.53.0 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/spf13/pflag v1.0.9 // indirect + golang.org/x/sys v0.44.0 // indirect + modernc.org/libc v1.73.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/PROJECTS/intermediate/security-news-scraper/go.sum b/PROJECTS/intermediate/security-news-scraper/go.sum new file mode 100644 index 00000000..7c4811a2 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/go.sum @@ -0,0 +1,64 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= +modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= +modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc= +modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= +modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= +modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go new file mode 100644 index 00000000..cbe18b30 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go @@ -0,0 +1,206 @@ +// ©AngelaMos | 2026 +// config.go + +package config + +import ( + "fmt" + "os" + + "gopkg.in/yaml.v3" +) + +const ( + defaultDBPath = "nadezhda.db" + + defaultUserAgent = "nadezhda/0.1 (+https://github.com/CarterPerez-dev/nadezhda)" + defaultPerHostRate = 0.5 + defaultPerHostBurst = 1 + defaultTimeoutSeconds = 25 + defaultWorkers = 8 + defaultMaxRetries = 3 + + defaultCacheTTLHours = 24 + defaultNegativeTTLHours = 3 + + defaultTitleJaccard = 0.6 + defaultWindowHours = 72 + + defaultHalfLifeHours = 48 + defaultVelocityNorm = 0.5 + + defaultWeightRecency = 0.20 + defaultWeightCVSS = 0.15 + defaultWeightKEV = 0.25 + defaultWeightEPSS = 0.15 + defaultWeightVelocity = 0.15 + defaultWeightSource = 0.05 + defaultWeightKeyword = 0.05 + + defaultAIProvider = "qwen" + defaultQwenBaseURL = "http://localhost:11434/v1" + defaultQwenModel = "qwen2.5:7b" + defaultOpenAIURL = "https://api.openai.com/v1" + defaultOpenAIModel = "gpt-4o-mini" + defaultGeminiURL = "https://generativelanguage.googleapis.com/v1beta/openai/" + defaultGeminiModel = "gemini-2.5-flash" + defaultAnthropicURL = "https://api.anthropic.com/v1" + defaultClaudeModel = "claude-opus-4-8" +) + +type Fetch struct { + UserAgent string `yaml:"user_agent"` + PerHostRate float64 `yaml:"per_host_rate"` + PerHostBurst int `yaml:"per_host_burst"` + TimeoutSeconds int `yaml:"timeout_seconds"` + Workers int `yaml:"workers"` + MaxRetries int `yaml:"max_retries"` +} + +type Enrich struct { + CacheTTLHours int `yaml:"cache_ttl_hours"` + NegativeTTLHours int `yaml:"negative_ttl_hours"` +} + +type Cluster struct { + TitleJaccard float64 `yaml:"title_jaccard_threshold"` + WindowHours int `yaml:"window_hours"` +} + +type Weights struct { + Recency float64 `yaml:"recency"` + CVSS float64 `yaml:"cvss"` + KEV float64 `yaml:"kev"` + EPSS float64 `yaml:"epss"` + Velocity float64 `yaml:"velocity"` + Source float64 `yaml:"source"` + Keyword float64 `yaml:"keyword"` +} + +type Rank struct { + HalfLifeHours int `yaml:"half_life_hours"` + VelocityNorm float64 `yaml:"velocity_norm"` + Weights Weights `yaml:"weights"` +} + +type Provider struct { + BaseURL string `yaml:"base_url"` + Model string `yaml:"model"` +} + +type AI struct { + Enabled bool `yaml:"enabled"` + Provider string `yaml:"provider"` + Qwen Provider `yaml:"qwen"` + OpenAI Provider `yaml:"openai"` + Gemini Provider `yaml:"gemini"` + Anthropic Provider `yaml:"anthropic"` +} + +type Config struct { + DBPath string `yaml:"db_path"` + SourcesPath string `yaml:"sources_path"` + Watchlist []string `yaml:"watchlist"` + Fetch Fetch `yaml:"fetch"` + Enrich Enrich `yaml:"enrich"` + Cluster Cluster `yaml:"cluster"` + Rank Rank `yaml:"rank"` + AI AI `yaml:"ai"` +} + +func Default() Config { + return Config{ + DBPath: defaultDBPath, + Fetch: Fetch{ + UserAgent: defaultUserAgent, + PerHostRate: defaultPerHostRate, + PerHostBurst: defaultPerHostBurst, + TimeoutSeconds: defaultTimeoutSeconds, + Workers: defaultWorkers, + MaxRetries: defaultMaxRetries, + }, + Enrich: Enrich{ + CacheTTLHours: defaultCacheTTLHours, + NegativeTTLHours: defaultNegativeTTLHours, + }, + Cluster: Cluster{ + TitleJaccard: defaultTitleJaccard, + WindowHours: defaultWindowHours, + }, + Rank: Rank{ + HalfLifeHours: defaultHalfLifeHours, + VelocityNorm: defaultVelocityNorm, + Weights: Weights{ + Recency: defaultWeightRecency, + CVSS: defaultWeightCVSS, + KEV: defaultWeightKEV, + EPSS: defaultWeightEPSS, + Velocity: defaultWeightVelocity, + Source: defaultWeightSource, + Keyword: defaultWeightKeyword, + }, + }, + AI: AI{ + Enabled: false, + Provider: defaultAIProvider, + Qwen: Provider{BaseURL: defaultQwenBaseURL, Model: defaultQwenModel}, + OpenAI: Provider{BaseURL: defaultOpenAIURL, Model: defaultOpenAIModel}, + Gemini: Provider{BaseURL: defaultGeminiURL, Model: defaultGeminiModel}, + Anthropic: Provider{BaseURL: defaultAnthropicURL, Model: defaultClaudeModel}, + }, + } +} + +func Load(path string) (Config, error) { + cfg := Default() + if path == "" { + return cfg, nil + } + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return cfg, nil + } + return Config{}, fmt.Errorf("read config %s: %w", path, err) + } + if err := yaml.Unmarshal(raw, &cfg); err != nil { + return Config{}, fmt.Errorf("parse config %s: %w", path, err) + } + if err := cfg.validate(); err != nil { + return Config{}, err + } + return cfg, nil +} + +func (c Config) validate() error { + if c.DBPath == "" { + return fmt.Errorf("config: db_path must not be empty") + } + if c.Fetch.Workers < 1 { + return fmt.Errorf("config: fetch.workers must be >= 1, got %d", c.Fetch.Workers) + } + if c.Fetch.PerHostRate <= 0 { + return fmt.Errorf("config: fetch.per_host_rate must be > 0, got %v", c.Fetch.PerHostRate) + } + if c.Fetch.PerHostBurst < 1 { + return fmt.Errorf("config: fetch.per_host_burst must be >= 1, got %d", c.Fetch.PerHostBurst) + } + if c.Enrich.CacheTTLHours < 0 || c.Enrich.NegativeTTLHours < 0 { + return fmt.Errorf("config: enrich TTLs must be >= 0, got cache=%d negative=%d", c.Enrich.CacheTTLHours, c.Enrich.NegativeTTLHours) + } + if c.Cluster.TitleJaccard < 0 || c.Cluster.TitleJaccard > 1 { + return fmt.Errorf("config: cluster.title_jaccard_threshold must be in [0,1], got %v", c.Cluster.TitleJaccard) + } + if c.Rank.HalfLifeHours < 1 { + return fmt.Errorf("config: rank.half_life_hours must be >= 1, got %d", c.Rank.HalfLifeHours) + } + if c.Rank.VelocityNorm <= 0 { + return fmt.Errorf("config: rank.velocity_norm must be > 0, got %v", c.Rank.VelocityNorm) + } + switch c.AI.Provider { + case "qwen", "openai", "gemini", "anthropic": + default: + return fmt.Errorf("config: ai.provider must be one of qwen|openai|gemini|anthropic, got %q", c.AI.Provider) + } + return nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config_test.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config_test.go new file mode 100644 index 00000000..9a490a58 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config_test.go @@ -0,0 +1,94 @@ +// ©AngelaMos | 2026 +// config_test.go + +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDefaultHasSaneValues(t *testing.T) { + c := Default() + if c.DBPath != defaultDBPath { + t.Errorf("db_path = %q, want %q", c.DBPath, defaultDBPath) + } + if c.AI.Enabled { + t.Error("ai.enabled must default to false (opt-in)") + } + if c.AI.Provider != "qwen" { + t.Errorf("ai.provider = %q, want qwen", c.AI.Provider) + } + if c.AI.Qwen.Model != defaultQwenModel { + t.Errorf("qwen model = %q, want %q", c.AI.Qwen.Model, defaultQwenModel) + } + sum := c.Rank.Weights.Recency + c.Rank.Weights.CVSS + c.Rank.Weights.KEV + + c.Rank.Weights.EPSS + c.Rank.Weights.Velocity + c.Rank.Weights.Source + + c.Rank.Weights.Keyword + if sum < 0.99 || sum > 1.01 { + t.Errorf("rank weights sum = %v, want ~1.0", sum) + } +} + +func TestLoadMissingFileReturnsDefaults(t *testing.T) { + c, err := Load(filepath.Join(t.TempDir(), "nope.yaml")) + if err != nil { + t.Fatalf("Load missing file: %v", err) + } + if c.DBPath != defaultDBPath { + t.Errorf("missing file should yield defaults, got db_path %q", c.DBPath) + } +} + +func TestLoadOverridesDefaults(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + body := ` +db_path: /tmp/custom.db +watchlist: [fortinet, "cisco ios"] +fetch: + workers: 4 +ai: + enabled: true + provider: anthropic + anthropic: + model: claude-sonnet-4-6 +` + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + c, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if c.DBPath != "/tmp/custom.db" { + t.Errorf("db_path = %q, want /tmp/custom.db", c.DBPath) + } + if len(c.Watchlist) != 2 || c.Watchlist[0] != "fortinet" { + t.Errorf("watchlist = %v", c.Watchlist) + } + if c.Fetch.Workers != 4 { + t.Errorf("workers = %d, want 4", c.Fetch.Workers) + } + if !c.AI.Enabled || c.AI.Provider != "anthropic" { + t.Errorf("ai override failed: %+v", c.AI) + } + if c.AI.Anthropic.Model != "claude-sonnet-4-6" { + t.Errorf("anthropic model = %q, want claude-sonnet-4-6", c.AI.Anthropic.Model) + } + if c.AI.Qwen.Model != defaultQwenModel { + t.Errorf("unset qwen model should keep default, got %q", c.AI.Qwen.Model) + } +} + +func TestValidateRejectsBadProvider(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + if err := os.WriteFile(path, []byte("ai:\n provider: llama\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err == nil { + t.Error("expected error for invalid ai.provider") + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/source/source.go b/PROJECTS/intermediate/security-news-scraper/internal/source/source.go new file mode 100644 index 00000000..864dd3ea --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/source/source.go @@ -0,0 +1,92 @@ +// ©AngelaMos | 2026 +// source.go + +package source + +import ( + _ "embed" + "fmt" + "net/url" + "os" + + "gopkg.in/yaml.v3" +) + +//go:embed sources.yaml +var embedded []byte + +type Kind string + +const ( + KindRSS Kind = "rss" + KindAtom Kind = "atom" + KindHTML Kind = "html" +) + +type Source struct { + Name string `yaml:"name"` + Title string `yaml:"title"` + URL string `yaml:"url"` + Type Kind `yaml:"type"` + Extractor string `yaml:"extractor"` + Weight float64 `yaml:"weight"` + Tags []string `yaml:"tags"` + Enabled bool `yaml:"enabled"` +} + +func Defaults() ([]Source, error) { + return parse(embedded) +} + +func Load(path string) ([]Source, error) { + if path == "" { + return Defaults() + } + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return Defaults() + } + return nil, fmt.Errorf("read sources %s: %w", path, err) + } + return parse(raw) +} + +func parse(raw []byte) ([]Source, error) { + var out []Source + if err := yaml.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("parse sources: %w", err) + } + if len(out) == 0 { + return nil, fmt.Errorf("sources: no entries") + } + seen := make(map[string]struct{}, len(out)) + for i, s := range out { + if s.Name == "" { + return nil, fmt.Errorf("sources[%d]: name is required", i) + } + if _, dup := seen[s.Name]; dup { + return nil, fmt.Errorf("sources: duplicate name %q", s.Name) + } + seen[s.Name] = struct{}{} + if _, err := url.ParseRequestURI(s.URL); err != nil { + return nil, fmt.Errorf("sources[%s]: invalid url %q: %w", s.Name, s.URL, err) + } + switch s.Type { + case KindRSS, KindAtom, KindHTML: + default: + return nil, fmt.Errorf("sources[%s]: type must be rss|atom|html, got %q", s.Name, s.Type) + } + } + return out, nil +} + +func Enabled(all []Source) []Source { + out := make([]Source, 0, len(all)) + for _, s := range all { + if s.Enabled { + out = append(out, s) + } + } + return out +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/source/source_test.go b/PROJECTS/intermediate/security-news-scraper/internal/source/source_test.go new file mode 100644 index 00000000..54137e1d --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/source/source_test.go @@ -0,0 +1,103 @@ +// ©AngelaMos | 2026 +// source_test.go + +package source + +import ( + "os" + "path/filepath" + "testing" +) + +func TestEmbeddedDefaultsParse(t *testing.T) { + got, err := Defaults() + if err != nil { + t.Fatalf("Defaults: %v", err) + } + want := map[string]bool{ + "krebs": false, "thehackernews": false, "bleepingcomputer": false, + "securityweek": false, "darkreading": false, "theregister": false, + "cisa": false, + } + for _, s := range got { + if _, ok := want[s.Name]; !ok { + continue + } + want[s.Name] = true + if s.Type != KindRSS { + t.Errorf("%s: type = %q, want rss", s.Name, s.Type) + } + if s.Weight <= 0 || s.Weight > 1 { + t.Errorf("%s: weight %v out of (0,1]", s.Name, s.Weight) + } + if !s.Enabled { + t.Errorf("%s: expected enabled by default", s.Name) + } + } + for name, seen := range want { + if !seen { + t.Errorf("seed source %q missing from embedded defaults", name) + } + } +} + +func TestExternalOverride(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sources.yaml") + body := ` +- name: custom + title: Custom Feed + url: https://example.com/feed.xml + type: rss + weight: 0.5 + enabled: true +` + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + got, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(got) != 1 || got[0].Name != "custom" { + t.Errorf("external override = %+v", got) + } +} + +func TestRejectsBadURL(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + body := "- name: x\n url: not-a-url\n type: rss\n enabled: true\n" + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err == nil { + t.Error("expected error for invalid url") + } +} + +func TestRejectsDuplicateName(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "dup.yaml") + body := ` +- {name: a, url: "https://a.com/f", type: rss, enabled: true} +- {name: a, url: "https://b.com/f", type: rss, enabled: true} +` + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err == nil { + t.Error("expected error for duplicate source name") + } +} + +func TestEnabledFilter(t *testing.T) { + all := []Source{ + {Name: "on", Enabled: true}, + {Name: "off", Enabled: false}, + } + got := Enabled(all) + if len(got) != 1 || got[0].Name != "on" { + t.Errorf("Enabled filter = %+v", got) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/source/sources.yaml b/PROJECTS/intermediate/security-news-scraper/internal/source/sources.yaml new file mode 100644 index 00000000..98e5ccbf --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/source/sources.yaml @@ -0,0 +1,58 @@ +# ©AngelaMos | 2026 +# sources.yaml + +- name: krebs + title: Krebs on Security + url: https://krebsonsecurity.com/feed/ + type: rss + weight: 1.0 + tags: [news] + enabled: true + +- name: thehackernews + title: The Hacker News + url: https://feeds.feedburner.com/TheHackersNews + type: rss + weight: 0.8 + tags: [news] + enabled: true + +- name: bleepingcomputer + title: BleepingComputer + url: https://www.bleepingcomputer.com/feed/ + type: rss + weight: 0.9 + tags: [news] + enabled: true + +- name: securityweek + title: SecurityWeek + url: https://www.securityweek.com/feed/ + type: rss + weight: 0.8 + tags: [news] + enabled: true + +- name: darkreading + title: Dark Reading + url: https://www.darkreading.com/rss.xml + type: rss + weight: 0.8 + tags: [news] + enabled: true + +- name: theregister + title: The Register (Security) + url: https://www.theregister.com/security/headlines.atom + type: rss + weight: 0.8 + tags: [news] + enabled: true + +- name: cisa + title: CISA Advisories + url: https://www.cisa.gov/cybersecurity-advisories/all.xml + type: rss + weight: 1.0 + tags: [advisory, gov] + enabled: true diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/migrate.go b/PROJECTS/intermediate/security-news-scraper/internal/store/migrate.go new file mode 100644 index 00000000..db1f0ac6 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/migrate.go @@ -0,0 +1,103 @@ +// ©AngelaMos | 2026 +// migrate.go + +package store + +import ( + "database/sql" + "embed" + "fmt" + "io/fs" + "sort" + "strconv" + "strings" +) + +//go:embed migrations/*.sql +var migrationsFS embed.FS + +type migration struct { + version int + name string + sql string +} + +func loadMigrations() ([]migration, error) { + entries, err := fs.ReadDir(migrationsFS, "migrations") + if err != nil { + return nil, fmt.Errorf("read migrations dir: %w", err) + } + out := make([]migration, 0, len(entries)) + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".sql") { + continue + } + verStr, _, ok := strings.Cut(e.Name(), "_") + if !ok { + return nil, fmt.Errorf("migration %q: expected NNNN_name.sql", e.Name()) + } + ver, err := strconv.Atoi(verStr) + if err != nil { + return nil, fmt.Errorf("migration %q: bad version prefix: %w", e.Name(), err) + } + body, err := migrationsFS.ReadFile("migrations/" + e.Name()) + if err != nil { + return nil, fmt.Errorf("read migration %q: %w", e.Name(), err) + } + out = append(out, migration{version: ver, name: e.Name(), sql: string(body)}) + } + sort.Slice(out, func(i, j int) bool { return out[i].version < out[j].version }) + return out, nil +} + +func migrate(db *sql.DB) (int, error) { + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) + )`); err != nil { + return 0, fmt.Errorf("create schema_migrations: %w", err) + } + + var current int + if err := db.QueryRow(`SELECT COALESCE(MAX(version), 0) FROM schema_migrations`).Scan(¤t); err != nil { + return 0, fmt.Errorf("read current version: %w", err) + } + + migrations, err := loadMigrations() + if err != nil { + return current, err + } + + maxKnown := 0 + for _, m := range migrations { + if m.version > maxKnown { + maxKnown = m.version + } + } + if current > maxKnown { + return current, fmt.Errorf("store schema version %d is newer than this binary supports (max %d); refusing to run against a store written by a newer build", current, maxKnown) + } + + for _, m := range migrations { + if m.version <= current { + continue + } + tx, err := db.Begin() + if err != nil { + return current, fmt.Errorf("begin migration %d: %w", m.version, err) + } + if _, err := tx.Exec(m.sql); err != nil { + _ = tx.Rollback() + return current, fmt.Errorf("apply migration %q: %w", m.name, err) + } + if _, err := tx.Exec(`INSERT INTO schema_migrations(version) VALUES (?)`, m.version); err != nil { + _ = tx.Rollback() + return current, fmt.Errorf("record migration %d: %w", m.version, err) + } + if err := tx.Commit(); err != nil { + return current, fmt.Errorf("commit migration %d: %w", m.version, err) + } + current = m.version + } + return current, nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0001_init.sql b/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0001_init.sql new file mode 100644 index 00000000..c0dc5cf8 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0001_init.sql @@ -0,0 +1,87 @@ +-- ©AngelaMos | 2026 +-- 0001_init.sql + +CREATE TABLE sources ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + title TEXT NOT NULL DEFAULT '', + url TEXT NOT NULL, + type TEXT NOT NULL, + weight REAL NOT NULL DEFAULT 1.0, + tags TEXT NOT NULL DEFAULT '', + enabled INTEGER NOT NULL DEFAULT 1 +); + +CREATE TABLE fetch_state ( + source_id INTEGER PRIMARY KEY REFERENCES sources(id) ON DELETE CASCADE, + etag TEXT NOT NULL DEFAULT '', + last_modified TEXT NOT NULL DEFAULT '', + last_fetched INTEGER NOT NULL DEFAULT 0, + last_status INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE articles ( + id INTEGER PRIMARY KEY, + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + canonical_url TEXT NOT NULL UNIQUE, + content_hash TEXT NOT NULL UNIQUE, + title TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL DEFAULT '', + author TEXT NOT NULL DEFAULT '', + published_at INTEGER NOT NULL DEFAULT 0, + fetched_at INTEGER NOT NULL DEFAULT 0 +); + +CREATE INDEX idx_articles_published ON articles(published_at); +CREATE INDEX idx_articles_source ON articles(source_id); + +CREATE TABLE cves ( + id TEXT PRIMARY KEY, + description TEXT NOT NULL DEFAULT '', + cvss_score REAL, + cvss_version TEXT NOT NULL DEFAULT '', + cvss_severity TEXT NOT NULL DEFAULT '', + cvss_vector TEXT NOT NULL DEFAULT '', + cwe TEXT NOT NULL DEFAULT '', + is_kev INTEGER NOT NULL DEFAULT 0, + kev_date_added TEXT NOT NULL DEFAULT '', + kev_ransomware INTEGER NOT NULL DEFAULT 0, + epss REAL, + epss_percentile REAL, + nvd_published TEXT NOT NULL DEFAULT '', + nvd_modified TEXT NOT NULL DEFAULT '', + enriched_at INTEGER NOT NULL DEFAULT 0, + enrich_status TEXT NOT NULL DEFAULT '' +); + +CREATE TABLE article_cves ( + article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + cve_id TEXT NOT NULL REFERENCES cves(id) ON DELETE CASCADE, + PRIMARY KEY (article_id, cve_id) +); + +CREATE TABLE clusters ( + id INTEGER PRIMARY KEY, + cluster_key TEXT NOT NULL UNIQUE, + first_seen INTEGER NOT NULL DEFAULT 0, + last_seen INTEGER NOT NULL DEFAULT 0, + size INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE cluster_members ( + cluster_id INTEGER NOT NULL REFERENCES clusters(id) ON DELETE CASCADE, + article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + PRIMARY KEY (cluster_id, article_id) +); + +CREATE TABLE ai_notes ( + id INTEGER PRIMARY KEY, + cluster_id INTEGER NOT NULL REFERENCES clusters(id) ON DELETE CASCADE, + provider TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + why TEXT NOT NULL DEFAULT '', + angles_json TEXT NOT NULL DEFAULT '', + format TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL DEFAULT 0 +); diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go new file mode 100644 index 00000000..dc874de7 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go @@ -0,0 +1,149 @@ +// ©AngelaMos | 2026 +// store.go + +package store + +import ( + "database/sql" + "errors" + "fmt" + "strings" + + "modernc.org/sqlite" + sqlite3 "modernc.org/sqlite/lib" +) + +var ErrDuplicate = errors.New("store: article already exists") + +type Store struct { + db *sql.DB + version int +} + +type SourceInput struct { + Name string + Title string + URL string + Type string + Weight float64 + Tags []string + Enabled bool +} + +type SourceRow struct { + ID int64 + Name string + Title string + URL string + Type string + Weight float64 + Tags []string + Enabled bool +} + +type Article struct { + SourceID int64 + CanonicalURL string + ContentHash string + Title string + Summary string + Body string + Author string + PublishedAt int64 + FetchedAt int64 +} + +func Open(path string) (*Store, error) { + dsn := fmt.Sprintf("file:%s?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)", path) + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("open sqlite %s: %w", path, err) + } + if err := db.Ping(); err != nil { + _ = db.Close() + return nil, fmt.Errorf("ping sqlite %s: %w", path, err) + } + version, err := migrate(db) + if err != nil { + _ = db.Close() + return nil, err + } + return &Store{db: db, version: version}, nil +} + +func (s *Store) Close() error { return s.db.Close() } +func (s *Store) Version() int { return s.version } +func (s *Store) DB() *sql.DB { return s.db } + +func (s *Store) UpsertSource(in SourceInput) (int64, error) { + tags := strings.Join(in.Tags, ",") + var id int64 + err := s.db.QueryRow(` + INSERT INTO sources (name, title, url, type, weight, tags, enabled) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(name) DO UPDATE SET + title = excluded.title, url = excluded.url, type = excluded.type, + weight = excluded.weight, tags = excluded.tags, enabled = excluded.enabled + RETURNING id`, + in.Name, in.Title, in.URL, in.Type, in.Weight, tags, boolToInt(in.Enabled), + ).Scan(&id) + if err != nil { + return 0, fmt.Errorf("upsert source %q: %w", in.Name, err) + } + return id, nil +} + +func (s *Store) GetSourceByName(name string) (SourceRow, error) { + var r SourceRow + var tags string + var enabled int + err := s.db.QueryRow(` + SELECT id, name, title, url, type, weight, tags, enabled + FROM sources WHERE name = ?`, name, + ).Scan(&r.ID, &r.Name, &r.Title, &r.URL, &r.Type, &r.Weight, &tags, &enabled) + if err != nil { + return SourceRow{}, fmt.Errorf("get source %q: %w", name, err) + } + if tags != "" { + r.Tags = strings.Split(tags, ",") + } + r.Enabled = enabled != 0 + return r, nil +} + +func (s *Store) InsertArticle(a Article) (int64, error) { + res, err := s.db.Exec(` + INSERT INTO articles + (source_id, canonical_url, content_hash, title, summary, body, author, published_at, fetched_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + a.SourceID, a.CanonicalURL, a.ContentHash, a.Title, a.Summary, a.Body, + a.Author, a.PublishedAt, a.FetchedAt, + ) + if err != nil { + var se *sqlite.Error + if errors.As(err, &se) && se.Code() == sqlite3.SQLITE_CONSTRAINT_UNIQUE { + return 0, ErrDuplicate + } + return 0, fmt.Errorf("insert article %q: %w", a.CanonicalURL, err) + } + id, err := res.LastInsertId() + if err != nil { + return 0, fmt.Errorf("insert article %q: last insert id: %w", a.CanonicalURL, err) + } + return id, nil +} + +func (s *Store) CountArticles() (int, error) { + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM articles`).Scan(&n); err != nil { + return 0, fmt.Errorf("count articles: %w", err) + } + return n, nil +} + +func boolToInt(b bool) int { + if b { + return 1 + } + return 0 +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store_test.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store_test.go new file mode 100644 index 00000000..0bdec31c --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store_test.go @@ -0,0 +1,116 @@ +// ©AngelaMos | 2026 +// store_test.go + +package store + +import ( + "errors" + "path/filepath" + "testing" +) + +func openTemp(t *testing.T) *Store { + t.Helper() + s, err := Open(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + return s +} + +func TestMigrateAppliesLatest(t *testing.T) { + s := openTemp(t) + if s.Version() < 1 { + t.Fatalf("schema version = %d, want >= 1", s.Version()) + } + var n int + if err := s.DB().QueryRow( + `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='articles'`, + ).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Error("articles table not created by migration") + } +} + +func TestMigrateIsIdempotent(t *testing.T) { + path := filepath.Join(t.TempDir(), "idem.db") + s1, err := Open(path) + if err != nil { + t.Fatal(err) + } + v1 := s1.Version() + _ = s1.Close() + + s2, err := Open(path) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer s2.Close() + if s2.Version() != v1 { + t.Errorf("reopen version = %d, want %d", s2.Version(), v1) + } +} + +func TestSourceRoundTrip(t *testing.T) { + s := openTemp(t) + id, err := s.UpsertSource(SourceInput{ + Name: "krebs", Title: "Krebs", URL: "https://krebsonsecurity.com/feed/", + Type: "rss", Weight: 1.0, Tags: []string{"news"}, Enabled: true, + }) + if err != nil { + t.Fatalf("UpsertSource: %v", err) + } + row, err := s.GetSourceByName("krebs") + if err != nil { + t.Fatalf("GetSourceByName: %v", err) + } + if row.ID != id || row.Weight != 1.0 || !row.Enabled || len(row.Tags) != 1 { + t.Errorf("round trip mismatch: %+v", row) + } + + id2, err := s.UpsertSource(SourceInput{ + Name: "krebs", Title: "Krebs Updated", URL: "https://krebsonsecurity.com/feed/", + Type: "rss", Weight: 0.9, Tags: []string{"news"}, Enabled: true, + }) + if err != nil { + t.Fatalf("re-upsert: %v", err) + } + if id2 != id { + t.Errorf("upsert should keep id %d, got %d", id, id2) + } + row2, _ := s.GetSourceByName("krebs") + if row2.Title != "Krebs Updated" || row2.Weight != 0.9 { + t.Errorf("upsert did not update fields: %+v", row2) + } +} + +func TestArticleUniqueConstraint(t *testing.T) { + s := openTemp(t) + srcID, err := s.UpsertSource(SourceInput{ + Name: "krebs", URL: "https://krebsonsecurity.com/feed/", Type: "rss", Enabled: true, + }) + if err != nil { + t.Fatal(err) + } + a := Article{ + SourceID: srcID, CanonicalURL: "https://krebsonsecurity.com/post-1", + ContentHash: "hash-1", Title: "Post 1", PublishedAt: 100, FetchedAt: 200, + } + if _, err := s.InsertArticle(a); err != nil { + t.Fatalf("first insert: %v", err) + } + if _, err := s.InsertArticle(a); !errors.Is(err, ErrDuplicate) { + t.Errorf("duplicate insert: got %v, want ErrDuplicate", err) + } + + n, err := s.CountArticles() + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("article count = %d, want 1 (dup rejected)", n) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/version/version.go b/PROJECTS/intermediate/security-news-scraper/internal/version/version.go new file mode 100644 index 00000000..6d79e67b --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/version/version.go @@ -0,0 +1,9 @@ +// ©AngelaMos | 2026 +// version.go + +package version + +const ( + Name = "nadezhda" + Version = "0.1.0-dev" +) diff --git a/PROJECTS/intermediate/security-news-scraper/justfile b/PROJECTS/intermediate/security-news-scraper/justfile new file mode 100644 index 00000000..85d7f3c8 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/justfile @@ -0,0 +1,33 @@ +# ©AngelaMos | 2026 +# justfile + +set shell := ["bash", "-cu"] + +binary := "nadezhda" + +default: + @just --list + +build: + go build -o {{binary}} ./cmd/nadezhda + +run *args: + go run ./cmd/nadezhda {{args}} + +test: + go test ./... + +vet: + go vet ./... + +fmt: + gofmt -w . + +lint: vet + test -z "$(gofmt -l .)" + +tidy: + go mod tidy + +clean: + rm -f {{binary}} *.db *.db-wal *.db-shm From b41276004fb25ad984bc8b134e3f7771c29b2b56 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Sun, 5 Jul 2026 14:51:57 -0400 Subject: [PATCH 02/15] feat(nadezhda): M1 ingestion core (fetch, parse, normalize, dedup) Concurrent, rate-limited feed ingestion wired end to end: 'scrape' pulls live security news into the SQLite store, idempotent across re-runs. - internal/fetch: per-host token-bucket rate limiting, conditional GET (ETag/Last-Modified with 304 handling), retry+backoff that honors Retry-After and treats timeouts/cancellation as non-retryable, per-source deadline, honest User-Agent, 16MB body cap. robots.txt is enforced only via Client.Allowed() for the future HTML article-scrape path, not on subscribed feed fetches (user-directed retrieval; several publishers blanket-disallow generic bots yet serve a public feed). - internal/parse: gofeed RSS/Atom to a normalized Item; RFC1123Z/RFC822Z/ RFC3339 time fallbacks (CISA emits RFC822Z, not RFC1123Z as first researched). - internal/normalize: canonical URL (lowercase scheme+host, drop fragment, strip utm_*/gclid/fbclid/ref/mc_cid/mc_eid, drop trailing slash), title normalization, goquery HTML strip, sha256 content+title hashes. - internal/ingest: errgroup fan-out over sources, fail-soft per source, fetch -> parse -> normalize -> InsertArticle dedup via store.ErrDuplicate, plus fetch_state upsert for conditional GET. - store: migration 0002 adds articles.title_hash; GetFetchState/UpsertFetchState. - Trimmed testdata/feeds fixtures (3 items each) back golden parse tests; full suite is offline and passes under -race. Proven live: cold run 215 new across 7 sources, warm run 0 new (304s + dedup). Ctrl-C aborts scrape cleanly via signal-aware context. --- .../cmd/nadezhda/scrape.go | 127 ++++ .../cmd/nadezhda/stubs.go | 1 - .../intermediate/security-news-scraper/go.mod | 12 + .../intermediate/security-news-scraper/go.sum | 102 ++- .../internal/config/config.go | 61 +- .../internal/fetch/fetch.go | 235 +++++++ .../internal/fetch/fetch_test.go | 244 +++++++ .../internal/fetch/robots.go | 88 +++ .../internal/ingest/ingest.go | 162 +++++ .../internal/ingest/ingest_test.go | 199 ++++++ .../internal/normalize/normalize.go | 111 ++++ .../internal/normalize/normalize_test.go | 111 ++++ .../internal/parse/parse.go | 74 +++ .../internal/parse/parse_test.go | 105 +++ .../internal/store/fetch_state_test.go | 81 +++ .../migrations/0002_article_title_hash.sql | 6 + .../internal/store/store.go | 44 +- .../testdata/feeds/bleepingcomputer.xml | 62 ++ .../testdata/feeds/cisa.xml | 598 ++++++++++++++++++ .../testdata/feeds/darkreading.xml | 52 ++ .../testdata/feeds/krebs.xml | 245 +++++++ .../testdata/feeds/securityweek.xml | 90 +++ .../testdata/feeds/thehackernews.xml | 9 + .../testdata/feeds/theregister.xml | 48 ++ 24 files changed, 2839 insertions(+), 28 deletions(-) create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/fetch/robots.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/parse/parse.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/parse/parse_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/fetch_state_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0002_article_title_hash.sql create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/feeds/bleepingcomputer.xml create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/feeds/cisa.xml create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/feeds/darkreading.xml create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/feeds/krebs.xml create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/feeds/securityweek.xml create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/feeds/thehackernews.xml create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/feeds/theregister.xml diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go new file mode 100644 index 00000000..6d3c5587 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go @@ -0,0 +1,127 @@ +// ©AngelaMos | 2026 +// scrape.go + +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "time" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/fetch" + "github.com/CarterPerez-dev/nadezhda/internal/ingest" + "github.com/CarterPerez-dev/nadezhda/internal/source" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const ( + statusNotModified = "304" + statusError = "error" + statusOK = "ok" + dash = "-" +) + +var scrapeSource string + +var scrapeCmd = &cobra.Command{ + Use: "scrape", + Short: "Ingest all enabled sources once", + RunE: runScrape, +} + +func init() { + scrapeCmd.Flags().StringVar(&scrapeSource, "source", "", "ingest only this source by name") + rootCmd.AddCommand(scrapeCmd) +} + +func runScrape(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + srcs, err := source.Load(cfg.SourcesPath) + if err != nil { + return err + } + + targets, err := selectTargets(srcs, scrapeSource) + if err != nil { + return err + } + + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + + fc := fetch.New(fetch.Options{ + UserAgent: cfg.Fetch.UserAgent, + PerHostRate: cfg.Fetch.PerHostRate, + PerHostBurst: cfg.Fetch.PerHostBurst, + Timeout: time.Duration(cfg.Fetch.TimeoutSeconds) * time.Second, + MaxRetries: cfg.Fetch.MaxRetries, + }) + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + summary, err := ingest.Run(ctx, fc, st, cfg, targets, time.Now()) + if err != nil { + return err + } + + printSummary(cmd, summary) + return nil +} + +func selectTargets(srcs []source.Source, only string) ([]source.Source, error) { + if only != "" { + for _, s := range srcs { + if s.Name == only { + return []source.Source{s}, nil + } + } + return nil, fmt.Errorf("scrape: unknown source %q", only) + } + return source.Enabled(srcs), nil +} + +func printSummary(cmd *cobra.Command, summary ingest.Summary) { + out := cmd.OutOrStdout() + fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s\n", "SOURCE", "STATUS", "PARSED", "NEW", "DUP", "ERR") + for _, r := range summary.Results { + fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s\n", + r.Name, status(r), count(r, r.Parsed), count(r, r.New), count(r, r.Duplicates), count(r, r.ItemErrors)) + } + newArticles, duplicates, failed := summary.Totals() + fmt.Fprintf(out, "\n%d new, %d duplicate across %d sources (%d failed)\n", + newArticles, duplicates, len(summary.Results), failed) + for _, r := range summary.Results { + if r.Err != nil { + fmt.Fprintf(out, " %s: %v\n", r.Name, r.Err) + } + } +} + +func status(r ingest.SourceResult) string { + switch { + case r.Err != nil: + return statusError + case r.NotModified: + return statusNotModified + default: + return statusOK + } +} + +func count(r ingest.SourceResult, n int) string { + if r.Err != nil || r.NotModified { + return dash + } + return fmt.Sprintf("%d", n) +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go index 2f9eb03e..3b9f1431 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go @@ -21,7 +21,6 @@ func init() { short string milestone string }{ - {"scrape", "Ingest all enabled sources once", "milestone M1"}, {"list", "List stored articles with filters", "milestone M3"}, {"cve", "Show an enriched CVE and articles mentioning it", "milestone M3"}, {"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"}, diff --git a/PROJECTS/intermediate/security-news-scraper/go.mod b/PROJECTS/intermediate/security-news-scraper/go.mod index 0f7c5876..7a74c19b 100644 --- a/PROJECTS/intermediate/security-news-scraper/go.mod +++ b/PROJECTS/intermediate/security-news-scraper/go.mod @@ -3,20 +3,32 @@ module github.com/CarterPerez-dev/nadezhda go 1.25.0 require ( + github.com/PuerkitoBio/goquery v1.12.0 + github.com/mmcdole/gofeed v1.3.0 github.com/spf13/cobra v1.10.2 + github.com/temoto/robotstxt v1.1.2 + golang.org/x/sync v0.21.0 + golang.org/x/time v0.15.0 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.53.0 ) require ( + github.com/andybalholm/cascadia v1.3.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/spf13/pflag v1.0.9 // indirect + golang.org/x/net v0.52.0 // indirect golang.org/x/sys v0.44.0 // indirect + golang.org/x/text v0.35.0 // indirect modernc.org/libc v1.73.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/PROJECTS/intermediate/security-news-scraper/go.sum b/PROJECTS/intermediate/security-news-scraper/go.sum index 7c4811a2..d61e7395 100644 --- a/PROJECTS/intermediate/security-news-scraper/go.sum +++ b/PROJECTS/intermediate/security-news-scraper/go.sum @@ -1,6 +1,15 @@ +github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo= +github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -9,10 +18,23 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= +github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -20,16 +42,92 @@ github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= +github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go index cbe18b30..a8692f5f 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go @@ -13,12 +13,13 @@ import ( const ( defaultDBPath = "nadezhda.db" - defaultUserAgent = "nadezhda/0.1 (+https://github.com/CarterPerez-dev/nadezhda)" - defaultPerHostRate = 0.5 - defaultPerHostBurst = 1 - defaultTimeoutSeconds = 25 - defaultWorkers = 8 - defaultMaxRetries = 3 + defaultUserAgent = "nadezhda/0.1 (+https://github.com/CarterPerez-dev/nadezhda)" + defaultPerHostRate = 0.5 + defaultPerHostBurst = 1 + defaultTimeoutSeconds = 25 + defaultSourceTimeoutSeconds = 90 + defaultWorkers = 8 + defaultMaxRetries = 3 defaultCacheTTLHours = 24 defaultNegativeTTLHours = 3 @@ -26,6 +27,8 @@ const ( defaultTitleJaccard = 0.6 defaultWindowHours = 72 + trackingUTMPrefix = "utm_*" + defaultHalfLifeHours = 48 defaultVelocityNorm = 0.5 @@ -48,13 +51,18 @@ const ( defaultClaudeModel = "claude-opus-4-8" ) +var defaultTrackingParams = []string{ + trackingUTMPrefix, "gclid", "fbclid", "ref", "mc_cid", "mc_eid", +} + type Fetch struct { - UserAgent string `yaml:"user_agent"` - PerHostRate float64 `yaml:"per_host_rate"` - PerHostBurst int `yaml:"per_host_burst"` - TimeoutSeconds int `yaml:"timeout_seconds"` - Workers int `yaml:"workers"` - MaxRetries int `yaml:"max_retries"` + UserAgent string `yaml:"user_agent"` + PerHostRate float64 `yaml:"per_host_rate"` + PerHostBurst int `yaml:"per_host_burst"` + TimeoutSeconds int `yaml:"timeout_seconds"` + SourceTimeoutSeconds int `yaml:"source_timeout_seconds"` + Workers int `yaml:"workers"` + MaxRetries int `yaml:"max_retries"` } type Enrich struct { @@ -63,8 +71,9 @@ type Enrich struct { } type Cluster struct { - TitleJaccard float64 `yaml:"title_jaccard_threshold"` - WindowHours int `yaml:"window_hours"` + TitleJaccard float64 `yaml:"title_jaccard_threshold"` + WindowHours int `yaml:"window_hours"` + TrackingParams []string `yaml:"tracking_params"` } type Weights struct { @@ -112,20 +121,22 @@ func Default() Config { return Config{ DBPath: defaultDBPath, Fetch: Fetch{ - UserAgent: defaultUserAgent, - PerHostRate: defaultPerHostRate, - PerHostBurst: defaultPerHostBurst, - TimeoutSeconds: defaultTimeoutSeconds, - Workers: defaultWorkers, - MaxRetries: defaultMaxRetries, + UserAgent: defaultUserAgent, + PerHostRate: defaultPerHostRate, + PerHostBurst: defaultPerHostBurst, + TimeoutSeconds: defaultTimeoutSeconds, + SourceTimeoutSeconds: defaultSourceTimeoutSeconds, + Workers: defaultWorkers, + MaxRetries: defaultMaxRetries, }, Enrich: Enrich{ CacheTTLHours: defaultCacheTTLHours, NegativeTTLHours: defaultNegativeTTLHours, }, Cluster: Cluster{ - TitleJaccard: defaultTitleJaccard, - WindowHours: defaultWindowHours, + TitleJaccard: defaultTitleJaccard, + WindowHours: defaultWindowHours, + TrackingParams: defaultTrackingParams, }, Rank: Rank{ HalfLifeHours: defaultHalfLifeHours, @@ -185,6 +196,12 @@ func (c Config) validate() error { if c.Fetch.PerHostBurst < 1 { return fmt.Errorf("config: fetch.per_host_burst must be >= 1, got %d", c.Fetch.PerHostBurst) } + if c.Fetch.TimeoutSeconds < 1 { + return fmt.Errorf("config: fetch.timeout_seconds must be >= 1, got %d", c.Fetch.TimeoutSeconds) + } + if c.Fetch.SourceTimeoutSeconds < 1 { + return fmt.Errorf("config: fetch.source_timeout_seconds must be >= 1, got %d", c.Fetch.SourceTimeoutSeconds) + } if c.Enrich.CacheTTLHours < 0 || c.Enrich.NegativeTTLHours < 0 { return fmt.Errorf("config: enrich TTLs must be >= 0, got cache=%d negative=%d", c.Enrich.CacheTTLHours, c.Enrich.NegativeTTLHours) } diff --git a/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch.go b/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch.go new file mode 100644 index 00000000..42419123 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch.go @@ -0,0 +1,235 @@ +// ©AngelaMos | 2026 +// fetch.go + +package fetch + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "sync" + "time" + + "golang.org/x/time/rate" +) + +const ( + headerUserAgent = "User-Agent" + headerAccept = "Accept" + headerIfNoneMatch = "If-None-Match" + headerIfModifiedSince = "If-Modified-Since" + headerETag = "ETag" + headerLastModified = "Last-Modified" + headerRetryAfter = "Retry-After" + + acceptFeed = "application/rss+xml, application/atom+xml, application/xml;q=0.9, */*;q=0.8" + + defaultBackoffBase = 500 * time.Millisecond + maxBodyBytes = 16 << 20 + maxRetryAfter = 60 * time.Second +) + +type Options struct { + UserAgent string + PerHostRate float64 + PerHostBurst int + Timeout time.Duration + MaxRetries int +} + +type Request struct { + URL string + ETag string + LastModified string +} + +type Result struct { + Status int + Body []byte + ETag string + LastModified string + NotModified bool +} + +type Client struct { + http *http.Client + ua string + rate rate.Limit + burst int + maxRetries int + backoffBase time.Duration + + mu sync.Mutex + limiters map[string]*rate.Limiter + + robots *robotsCache +} + +func New(opts Options) *Client { + c := &Client{ + http: &http.Client{Timeout: opts.Timeout}, + ua: opts.UserAgent, + rate: rate.Limit(opts.PerHostRate), + burst: opts.PerHostBurst, + maxRetries: opts.MaxRetries, + backoffBase: defaultBackoffBase, + limiters: make(map[string]*rate.Limiter), + } + c.robots = newRobotsCache(c) + return c +} + +func (c *Client) limiterFor(host string) *rate.Limiter { + c.mu.Lock() + defer c.mu.Unlock() + l, ok := c.limiters[host] + if !ok { + l = rate.NewLimiter(c.rate, c.burst) + c.limiters[host] = l + } + return l +} + +func (c *Client) Fetch(ctx context.Context, req Request) (Result, error) { + u, err := url.Parse(req.URL) + if err != nil { + return Result{}, fmt.Errorf("fetch: parse url %q: %w", req.URL, err) + } + + var lastErr error + var override time.Duration + for attempt := 0; attempt <= c.maxRetries; attempt++ { + if attempt > 0 { + delay := c.backoff(attempt) + if override > 0 { + delay = override + } + if err := sleep(ctx, delay); err != nil { + return Result{}, err + } + } + if err := c.limiterFor(u.Host).Wait(ctx); err != nil { + return Result{}, err + } + res, retryAfter, retry, err := c.do(ctx, req) + if err == nil { + return res, nil + } + lastErr = err + override = retryAfter + if !retry { + return Result{}, err + } + } + return Result{}, fmt.Errorf("fetch %s: exhausted %d retries: %w", req.URL, c.maxRetries, lastErr) +} + +func (c *Client) Allowed(ctx context.Context, rawURL string) (bool, error) { + u, err := url.Parse(rawURL) + if err != nil { + return false, fmt.Errorf("fetch: parse url %q: %w", rawURL, err) + } + return c.robots.allowed(ctx, u) +} + +func (c *Client) do(ctx context.Context, req Request) (Result, time.Duration, bool, error) { + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, req.URL, nil) + if err != nil { + return Result{}, 0, false, fmt.Errorf("fetch %s: build request: %w", req.URL, err) + } + httpReq.Header.Set(headerUserAgent, c.ua) + httpReq.Header.Set(headerAccept, acceptFeed) + if req.ETag != "" { + httpReq.Header.Set(headerIfNoneMatch, req.ETag) + } + if req.LastModified != "" { + httpReq.Header.Set(headerIfModifiedSince, req.LastModified) + } + + resp, err := c.http.Do(httpReq) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return Result{}, 0, false, fmt.Errorf("fetch %s: %w", req.URL, err) + } + return Result{}, 0, true, fmt.Errorf("fetch %s: %w", req.URL, err) + } + defer resp.Body.Close() + + switch { + case resp.StatusCode == http.StatusNotModified: + return Result{ + Status: resp.StatusCode, + NotModified: true, + ETag: req.ETag, + LastModified: req.LastModified, + }, 0, false, nil + case resp.StatusCode == http.StatusOK: + body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes)) + if err != nil { + return Result{}, 0, true, fmt.Errorf("fetch %s: read body: %w", req.URL, err) + } + return Result{ + Status: resp.StatusCode, + Body: body, + ETag: resp.Header.Get(headerETag), + LastModified: resp.Header.Get(headerLastModified), + }, 0, false, nil + case resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= http.StatusInternalServerError: + drain(resp.Body) + return Result{}, retryAfter(resp), true, fmt.Errorf("fetch %s: server status %d", req.URL, resp.StatusCode) + default: + drain(resp.Body) + return Result{}, 0, false, fmt.Errorf("fetch %s: status %d", req.URL, resp.StatusCode) + } +} + +func drain(body io.Reader) { + _, _ = io.Copy(io.Discard, io.LimitReader(body, maxBodyBytes)) +} + +func retryAfter(resp *http.Response) time.Duration { + v := resp.Header.Get(headerRetryAfter) + if v == "" { + return 0 + } + if secs, err := strconv.Atoi(v); err == nil { + d := time.Duration(secs) * time.Second + if d > maxRetryAfter { + return maxRetryAfter + } + if d < 0 { + return 0 + } + return d + } + if t, err := http.ParseTime(v); err == nil { + d := time.Until(t) + if d <= 0 { + return 0 + } + if d > maxRetryAfter { + return maxRetryAfter + } + return d + } + return 0 +} + +func (c *Client) backoff(attempt int) time.Duration { + return c.backoffBase * time.Duration(1<<(attempt-1)) +} + +func sleep(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch_test.go b/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch_test.go new file mode 100644 index 00000000..a94beefa --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch_test.go @@ -0,0 +1,244 @@ +// ©AngelaMos | 2026 +// fetch_test.go + +package fetch + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +func testClient() *Client { + c := New(Options{ + UserAgent: "nadezhda-test/1.0", + PerHostRate: 1e6, + PerHostBurst: 1, + Timeout: 5 * time.Second, + MaxRetries: 3, + }) + c.backoffBase = time.Millisecond + return c +} + +func TestFetchOKCapturesValidators(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerETag, `"v1"`) + w.Header().Set(headerLastModified, "Wed, 01 Jul 2026 00:00:00 GMT") + _, _ = w.Write([]byte("")) + })) + defer srv.Close() + + res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"}) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if res.Status != http.StatusOK { + t.Errorf("status = %d, want 200", res.Status) + } + if string(res.Body) != "" { + t.Errorf("body = %q", res.Body) + } + if res.ETag != `"v1"` { + t.Errorf("etag = %q, want \"v1\"", res.ETag) + } + if res.LastModified == "" { + t.Error("last-modified not captured") + } +} + +func TestFetchConditionalGET(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get(headerIfNoneMatch) == `"v1"` { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set(headerETag, `"v1"`) + _, _ = w.Write([]byte("body")) + })) + defer srv.Close() + + c := testClient() + first, err := c.Fetch(context.Background(), Request{URL: srv.URL + "/feed"}) + if err != nil || first.Status != http.StatusOK { + t.Fatalf("first fetch: status=%d err=%v", first.Status, err) + } + + second, err := c.Fetch(context.Background(), Request{URL: srv.URL + "/feed", ETag: first.ETag}) + if err != nil { + t.Fatalf("second fetch: %v", err) + } + if !second.NotModified { + t.Error("expected NotModified on matching ETag") + } + if second.ETag != `"v1"` { + t.Errorf("304 should retain etag, got %q", second.ETag) + } +} + +func TestFetchRetriesServerError(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 { + w.WriteHeader(http.StatusInternalServerError) + return + } + _, _ = w.Write([]byte("ok")) + })) + defer srv.Close() + + res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"}) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if string(res.Body) != "ok" { + t.Errorf("body = %q, want ok", res.Body) + } + if calls.Load() != 2 { + t.Errorf("calls = %d, want 2 (one 500, one 200)", calls.Load()) + } +} + +func TestFetchDoesNotRetryClientError(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + _, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"}) + if err == nil { + t.Fatal("expected error on 404") + } + if calls.Load() != 1 { + t.Errorf("calls = %d, want 1 (no retry on 4xx)", calls.Load()) + } +} + +func TestFetch429IsRetried(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 { + w.WriteHeader(http.StatusTooManyRequests) + return + } + _, _ = w.Write([]byte("ok")) + })) + defer srv.Close() + + res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"}) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if string(res.Body) != "ok" || calls.Load() != 2 { + t.Errorf("body=%q calls=%d, want ok/2", res.Body, calls.Load()) + } +} + +func TestFetchTimeoutNotRetried(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + time.Sleep(100 * time.Millisecond) + })) + defer srv.Close() + + c := New(Options{ + UserAgent: "nadezhda-test/1.0", PerHostRate: 1e6, PerHostBurst: 1, + Timeout: 15 * time.Millisecond, MaxRetries: 3, + }) + c.backoffBase = time.Millisecond + + _, err := c.Fetch(context.Background(), Request{URL: srv.URL + "/feed"}) + if err == nil { + t.Fatal("expected timeout error") + } + if calls.Load() != 1 { + t.Errorf("calls = %d, want 1 (timeout must not be retried)", calls.Load()) + } +} + +func TestRetryAfterParsing(t *testing.T) { + cases := []struct { + header string + want time.Duration + }{ + {"", 0}, + {"5", 5 * time.Second}, + {"0", 0}, + {"-3", 0}, + {"9999", maxRetryAfter}, + {"garbage", 0}, + } + for _, tc := range cases { + resp := &http.Response{Header: http.Header{}} + if tc.header != "" { + resp.Header.Set(headerRetryAfter, tc.header) + } + if got := retryAfter(resp); got != tc.want { + t.Errorf("retryAfter(%q) = %s, want %s", tc.header, got, tc.want) + } + } +} + +func TestFeedFetchIgnoresRobots(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == robotsPath { + t.Error("Fetch must not request robots.txt on the feed path") + _, _ = w.Write([]byte("User-agent: *\nDisallow: /\n")) + return + } + _, _ = w.Write([]byte("feed body")) + })) + defer srv.Close() + + res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"}) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if string(res.Body) != "feed body" { + t.Errorf("body = %q, want feed body", res.Body) + } +} + +func TestAllowedRespectsDisallow(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == robotsPath { + _, _ = w.Write([]byte("User-agent: *\nDisallow: /article\n")) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + ok, err := testClient().Allowed(context.Background(), srv.URL+"/article/123") + if err != nil { + t.Fatalf("Allowed: %v", err) + } + if ok { + t.Error("expected disallowed for /article path") + } +} + +func TestAllowedPermitsOtherPaths(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == robotsPath { + _, _ = w.Write([]byte("User-agent: *\nDisallow: /admin\n")) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + ok, err := testClient().Allowed(context.Background(), srv.URL+"/article/123") + if err != nil { + t.Fatalf("Allowed: %v", err) + } + if !ok { + t.Error("expected allowed for non-admin path") + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/fetch/robots.go b/PROJECTS/intermediate/security-news-scraper/internal/fetch/robots.go new file mode 100644 index 00000000..f8aee1be --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/fetch/robots.go @@ -0,0 +1,88 @@ +// ©AngelaMos | 2026 +// robots.go + +package fetch + +import ( + "context" + "io" + "net/http" + "net/url" + "sync" + + "github.com/temoto/robotstxt" +) + +const ( + robotsPath = "/robots.txt" + maxRobotsBytes = 512 << 10 + robotsRootPath = "/" +) + +type robotsEntry struct { + once sync.Once + data *robotstxt.RobotsData +} + +type robotsCache struct { + client *Client + + mu sync.Mutex + entries map[string]*robotsEntry +} + +func newRobotsCache(client *Client) *robotsCache { + return &robotsCache{ + client: client, + entries: make(map[string]*robotsEntry), + } +} + +func (rc *robotsCache) allowed(ctx context.Context, u *url.URL) (bool, error) { + rc.mu.Lock() + e, ok := rc.entries[u.Host] + if !ok { + e = &robotsEntry{} + rc.entries[u.Host] = e + } + rc.mu.Unlock() + + e.once.Do(func() { + data, err := rc.load(ctx, u) + if err != nil { + data, _ = robotstxt.FromStatusAndBytes(http.StatusOK, nil) + } + e.data = data + }) + + path := u.EscapedPath() + if path == "" { + path = robotsRootPath + } + return e.data.FindGroup(rc.client.ua).Test(path), nil +} + +func (rc *robotsCache) load(ctx context.Context, u *url.URL) (*robotstxt.RobotsData, error) { + robotsURL := u.Scheme + "://" + u.Host + robotsPath + if err := rc.client.limiterFor(u.Host).Wait(ctx); err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, robotsURL, nil) + if err != nil { + return nil, err + } + req.Header.Set(headerUserAgent, rc.client.ua) + + resp, err := rc.client.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, maxRobotsBytes)) + if err != nil { + return nil, err + } + return robotstxt.FromStatusAndBytes(resp.StatusCode, body) +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go b/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go new file mode 100644 index 00000000..215dc79b --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go @@ -0,0 +1,162 @@ +// ©AngelaMos | 2026 +// ingest.go + +package ingest + +import ( + "bytes" + "context" + "errors" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/fetch" + "github.com/CarterPerez-dev/nadezhda/internal/normalize" + "github.com/CarterPerez-dev/nadezhda/internal/parse" + "github.com/CarterPerez-dev/nadezhda/internal/source" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +type SourceResult struct { + Name string + Parsed int + New int + Duplicates int + ItemErrors int + NotModified bool + Err error +} + +type Summary struct { + Results []SourceResult +} + +func (s Summary) Totals() (newArticles, duplicates, failed int) { + for _, r := range s.Results { + newArticles += r.New + duplicates += r.Duplicates + if r.Err != nil { + failed++ + } + } + return newArticles, duplicates, failed +} + +func Run(ctx context.Context, fc *fetch.Client, st *store.Store, cfg config.Config, targets []source.Source, now time.Time) (Summary, error) { + results := make([]SourceResult, len(targets)) + + workers := cfg.Fetch.Workers + if workers < 1 { + workers = 1 + } + + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(workers) + + for i, src := range targets { + results[i].Name = src.Name + g.Go(func() error { + process(gctx, fc, st, cfg, src, now, &results[i]) + return nil + }) + } + if err := g.Wait(); err != nil { + return Summary{Results: results}, err + } + return Summary{Results: results}, nil +} + +func process(ctx context.Context, fc *fetch.Client, st *store.Store, cfg config.Config, src source.Source, now time.Time, out *SourceResult) { + ctx, cancel := context.WithTimeout(ctx, time.Duration(cfg.Fetch.SourceTimeoutSeconds)*time.Second) + defer cancel() + + id, err := st.UpsertSource(store.SourceInput{ + Name: src.Name, Title: src.Title, URL: src.URL, Type: string(src.Type), + Weight: src.Weight, Tags: src.Tags, Enabled: src.Enabled, + }) + if err != nil { + out.Err = err + return + } + + prev, _, err := st.GetFetchState(id) + if err != nil { + out.Err = err + return + } + + res, err := fc.Fetch(ctx, fetch.Request{ + URL: src.URL, ETag: prev.ETag, LastModified: prev.LastModified, + }) + if err != nil { + out.Err = err + return + } + + if res.NotModified { + out.NotModified = true + out.Err = st.UpsertFetchState(id, store.FetchState{ + ETag: prev.ETag, LastModified: prev.LastModified, + LastFetched: now.Unix(), LastStatus: int64(res.Status), + }) + return + } + + items, err := parse.Feed(bytes.NewReader(res.Body)) + if err != nil { + out.Err = err + return + } + out.Parsed = len(items) + + for _, it := range items { + storeItem(st, cfg, id, it, now, out) + } + + if err := st.UpsertFetchState(id, store.FetchState{ + ETag: res.ETag, LastModified: res.LastModified, + LastFetched: now.Unix(), LastStatus: int64(res.Status), + }); err != nil && out.Err == nil { + out.Err = err + } +} + +func storeItem(st *store.Store, cfg config.Config, sourceID int64, it parse.Item, now time.Time, out *SourceResult) { + if it.Link == "" { + out.ItemErrors++ + return + } + canonical, err := normalize.CanonicalURL(it.Link, cfg.Cluster.TrackingParams) + if err != nil { + out.ItemErrors++ + return + } + + var publishedAt int64 + if !it.Published.IsZero() { + publishedAt = it.Published.Unix() + } + + _, err = st.InsertArticle(store.Article{ + SourceID: sourceID, + CanonicalURL: canonical, + ContentHash: normalize.ContentHash(canonical), + TitleHash: normalize.TitleHash(normalize.NormalizeTitle(it.Title)), + Title: it.Title, + Summary: normalize.StripHTML(it.Summary), + Body: normalize.StripHTML(it.Body), + Author: it.Author, + PublishedAt: publishedAt, + FetchedAt: now.Unix(), + }) + switch { + case err == nil: + out.New++ + case errors.Is(err, store.ErrDuplicate): + out.Duplicates++ + default: + out.ItemErrors++ + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest_test.go b/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest_test.go new file mode 100644 index 00000000..0aa0f3e1 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest_test.go @@ -0,0 +1,199 @@ +// ©AngelaMos | 2026 +// ingest_test.go + +package ingest + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/fetch" + "github.com/CarterPerez-dev/nadezhda/internal/normalize" + "github.com/CarterPerez-dev/nadezhda/internal/source" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const fixture = "../../testdata/feeds/thehackernews.xml" + +func loadFixture(t *testing.T) []byte { + t.Helper() + b, err := os.ReadFile(fixture) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + return b +} + +func newStore(t *testing.T) *store.Store { + t.Helper() + st, err := store.Open(filepath.Join(t.TempDir(), "ingest.db")) + if err != nil { + t.Fatalf("open store: %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + return st +} + +func newClient() *fetch.Client { + return fetch.New(fetch.Options{ + UserAgent: "nadezhda-test/1.0", + PerHostRate: 1e6, + PerHostBurst: 1, + Timeout: 5 * time.Second, + MaxRetries: 2, + }) +} + +func target(url string) []source.Source { + return []source.Source{{ + Name: "test", Title: "Test Feed", URL: url, + Type: source.KindRSS, Weight: 1.0, Tags: []string{"news"}, Enabled: true, + }} +} + +func TestRunIngestsAndDedups(t *testing.T) { + body := loadFixture(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(body) + })) + defer srv.Close() + + st := newStore(t) + cfg := config.Default() + targets := target(srv.URL + "/feed") + now := time.Unix(1_800_000_000, 0) + + first, err := Run(context.Background(), newClient(), st, cfg, targets, now) + if err != nil { + t.Fatalf("first Run: %v", err) + } + r0 := first.Results[0] + if r0.Err != nil { + t.Fatalf("source error: %v", r0.Err) + } + if r0.New != 3 || r0.Parsed != 3 { + t.Fatalf("first run: parsed=%d new=%d, want 3/3", r0.Parsed, r0.New) + } + + count, err := st.CountArticles() + if err != nil { + t.Fatal(err) + } + if count != 3 { + t.Fatalf("stored articles = %d, want 3", count) + } + + var title, canonical, contentHash, titleHash string + if err := st.DB().QueryRow( + `SELECT title, canonical_url, content_hash, title_hash FROM articles LIMIT 1`, + ).Scan(&title, &canonical, &contentHash, &titleHash); err != nil { + t.Fatal(err) + } + if contentHash != normalize.ContentHash(canonical) { + t.Errorf("content_hash = %q, want sha256(canonical)", contentHash) + } + if titleHash != normalize.TitleHash(normalize.NormalizeTitle(title)) { + t.Errorf("title_hash = %q, want sha256(normalized title)", titleHash) + } + + second, err := Run(context.Background(), newClient(), st, cfg, targets, now) + if err != nil { + t.Fatalf("second Run: %v", err) + } + r1 := second.Results[0] + if r1.New != 0 || r1.Duplicates != 3 { + t.Fatalf("second run: new=%d dup=%d, want 0/3", r1.New, r1.Duplicates) + } + + count, _ = st.CountArticles() + if count != 3 { + t.Fatalf("after re-run stored = %d, want 3 (idempotent)", count) + } +} + +func TestRunConditionalNotModified(t *testing.T) { + body := loadFixture(t) + const etag = `"abc123"` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("If-None-Match") == etag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", etag) + _, _ = w.Write(body) + })) + defer srv.Close() + + st := newStore(t) + cfg := config.Default() + targets := target(srv.URL + "/feed") + now := time.Unix(1_800_000_000, 0) + + first, err := Run(context.Background(), newClient(), st, cfg, targets, now) + if err != nil { + t.Fatalf("first Run: %v", err) + } + if first.Results[0].New != 3 { + t.Fatalf("first run new = %d, want 3", first.Results[0].New) + } + var storedETag string + if err := st.DB().QueryRow(`SELECT etag FROM fetch_state LIMIT 1`).Scan(&storedETag); err != nil { + t.Fatal(err) + } + if storedETag != etag { + t.Errorf("persisted etag = %q, want %q", storedETag, etag) + } + + second, err := Run(context.Background(), newClient(), st, cfg, targets, now) + if err != nil { + t.Fatalf("second Run: %v", err) + } + r := second.Results[0] + if !r.NotModified { + t.Errorf("expected NotModified on second run, got %+v", r) + } + if r.New != 0 { + t.Errorf("new = %d on 304, want 0", r.New) + } +} + +func TestRunFailsSoftOnBadSource(t *testing.T) { + body := loadFixture(t) + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(body) + })) + defer good.Close() + bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer bad.Close() + + st := newStore(t) + cfg := config.Default() + targets := []source.Source{ + {Name: "bad", URL: bad.URL + "/feed", Type: source.KindRSS, Weight: 1, Enabled: true}, + {Name: "good", URL: good.URL + "/feed", Type: source.KindRSS, Weight: 1, Enabled: true}, + } + now := time.Unix(1_800_000_000, 0) + + summary, err := Run(context.Background(), newClient(), st, cfg, targets, now) + if err != nil { + t.Fatalf("Run returned error, should fail soft: %v", err) + } + byName := map[string]SourceResult{} + for _, r := range summary.Results { + byName[r.Name] = r + } + if byName["bad"].Err == nil { + t.Error("bad source should record an error") + } + if byName["good"].New != 3 { + t.Errorf("good source new = %d, want 3 (bad source must not abort run)", byName["good"].New) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize.go b/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize.go new file mode 100644 index 00000000..47fd8c7e --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize.go @@ -0,0 +1,111 @@ +// ©AngelaMos | 2026 +// normalize.go + +package normalize + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "net/url" + "strings" + "unicode" + + "github.com/PuerkitoBio/goquery" +) + +const ( + wildcardSuffix = "*" + trailingSlash = "/" +) + +func CanonicalURL(raw string, trackingParams []string) (string, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return "", fmt.Errorf("normalize: parse url %q: %w", raw, err) + } + if u.Host == "" { + return "", fmt.Errorf("normalize: url %q has no host", raw) + } + + u.Scheme = strings.ToLower(u.Scheme) + u.Host = strings.ToLower(u.Host) + u.Fragment = "" + u.RawFragment = "" + + if q := u.Query(); len(q) > 0 { + for key := range q { + if isTracking(key, trackingParams) { + q.Del(key) + } + } + u.RawQuery = q.Encode() + } + + u.Path = strings.TrimRight(u.Path, trailingSlash) + if u.RawPath != "" { + u.RawPath = strings.TrimRight(u.RawPath, trailingSlash) + } + + return u.String(), nil +} + +func isTracking(key string, trackingParams []string) bool { + lowered := strings.ToLower(key) + for _, p := range trackingParams { + if strings.HasSuffix(p, wildcardSuffix) { + if strings.HasPrefix(lowered, strings.TrimSuffix(p, wildcardSuffix)) { + return true + } + continue + } + if lowered == p { + return true + } + } + return false +} + +func NormalizeTitle(title string) string { + var b strings.Builder + b.Grow(len(title)) + prevSpace := false + for _, r := range strings.ToLower(title) { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + b.WriteRune(r) + prevSpace = false + continue + } + if !prevSpace { + b.WriteByte(' ') + prevSpace = true + } + } + return strings.TrimSpace(b.String()) +} + +func StripHTML(s string) string { + doc, err := goquery.NewDocumentFromReader(strings.NewReader(s)) + if err != nil { + return collapseWhitespace(s) + } + doc.Find("script,style").Remove() + return collapseWhitespace(doc.Text()) +} + +func collapseWhitespace(s string) string { + return strings.Join(strings.Fields(s), " ") +} + +func ContentHash(canonicalURL string) string { + return sha256Hex(canonicalURL) +} + +func TitleHash(normalizedTitle string) string { + return sha256Hex(normalizedTitle) +} + +func sha256Hex(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize_test.go b/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize_test.go new file mode 100644 index 00000000..364e5039 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize_test.go @@ -0,0 +1,111 @@ +// ©AngelaMos | 2026 +// normalize_test.go + +package normalize + +import "testing" + +var params = []string{"utm_*", "gclid", "fbclid", "ref", "mc_cid", "mc_eid"} + +func TestCanonicalURL(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"lowercase host+scheme", "HTTPS://Example.COM/Path", "https://example.com/Path"}, + {"drop fragment", "https://example.com/a#section", "https://example.com/a"}, + {"strip utm params", "https://example.com/a?utm_source=x&utm_medium=y&id=7", "https://example.com/a?id=7"}, + {"strip gclid fbclid ref", "https://example.com/a?gclid=1&fbclid=2&ref=z&keep=1", "https://example.com/a?keep=1"}, + {"strip mailchimp", "https://example.com/a?mc_cid=1&mc_eid=2", "https://example.com/a"}, + {"drop trailing slash", "https://example.com/a/b/", "https://example.com/a/b"}, + {"root trailing slash", "https://example.com/", "https://example.com"}, + {"sorted query", "https://example.com/a?b=2&a=1", "https://example.com/a?a=1&b=2"}, + {"tracking case insensitive", "https://example.com/a?UTM_Source=x&id=1", "https://example.com/a?id=1"}, + {"path preserved case", "https://example.com/Foo/Bar", "https://example.com/Foo/Bar"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := CanonicalURL(tc.in, params) + if err != nil { + t.Fatalf("CanonicalURL(%q): %v", tc.in, err) + } + if got != tc.want { + t.Errorf("CanonicalURL(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +func TestCanonicalURLCollapsesTracking(t *testing.T) { + a, _ := CanonicalURL("https://example.com/story?utm_source=twitter", params) + b, _ := CanonicalURL("https://example.com/story", params) + if a != b { + t.Errorf("tracking variants did not collapse: %q vs %q", a, b) + } + if ContentHash(a) != ContentHash(b) { + t.Error("content hashes differ for tracking variants") + } +} + +func TestCanonicalURLErrors(t *testing.T) { + for _, in := range []string{"", "not a url", "/relative/only"} { + if _, err := CanonicalURL(in, params); err == nil { + t.Errorf("CanonicalURL(%q) expected error", in) + } + } +} + +func TestNormalizeTitle(t *testing.T) { + cases := []struct { + in string + want string + }{ + {"CVE-2021-44228: Log4Shell!", "cve 2021 44228 log4shell"}, + {" Multiple Spaces ", "multiple spaces"}, + {"Punctuation, and; stuff.", "punctuation and stuff"}, + {"Café déjà vu", "café déjà vu"}, + {"", ""}, + } + for _, tc := range cases { + if got := NormalizeTitle(tc.in); got != tc.want { + t.Errorf("NormalizeTitle(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestNormalizeTitleStableHash(t *testing.T) { + a := TitleHash(NormalizeTitle("Breaking: Big Hack!!!")) + b := TitleHash(NormalizeTitle("breaking big hack")) + if a != b { + t.Error("normalized-title hashes differ for equivalent titles") + } +} + +func TestStripHTML(t *testing.T) { + cases := []struct { + in string + want string + }{ + {"

Hello world

", "Hello world"}, + {"
a
\n
b
", "a b"}, + {"visible", "visible"}, + {"plain text", "plain text"}, + {"link and text", "link and text"}, + } + for _, tc := range cases { + if got := StripHTML(tc.in); got != tc.want { + t.Errorf("StripHTML(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestContentHashDeterministic(t *testing.T) { + const u = "https://example.com/a" + if ContentHash(u) != ContentHash(u) { + t.Error("ContentHash not deterministic") + } + if len(ContentHash(u)) != 64 { + t.Errorf("ContentHash length = %d, want 64 hex chars", len(ContentHash(u))) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/parse/parse.go b/PROJECTS/intermediate/security-news-scraper/internal/parse/parse.go new file mode 100644 index 00000000..c8d02b8a --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/parse/parse.go @@ -0,0 +1,74 @@ +// ©AngelaMos | 2026 +// parse.go + +package parse + +import ( + "fmt" + "io" + "strings" + "time" + + "github.com/mmcdole/gofeed" +) + +type Item struct { + Title string + Link string + Summary string + Body string + Author string + Published time.Time + Categories []string +} + +var timeLayouts = []string{time.RFC1123Z, time.RFC1123, time.RFC822Z, time.RFC822, time.RFC3339} + +func Feed(r io.Reader) ([]Item, error) { + feed, err := gofeed.NewParser().Parse(r) + if err != nil { + return nil, fmt.Errorf("parse: %w", err) + } + items := make([]Item, 0, len(feed.Items)) + for _, it := range feed.Items { + items = append(items, Item{ + Title: strings.TrimSpace(it.Title), + Link: strings.TrimSpace(it.Link), + Summary: it.Description, + Body: it.Content, + Author: author(it), + Published: published(it), + Categories: it.Categories, + }) + } + return items, nil +} + +func author(it *gofeed.Item) string { + if it.Author != nil && it.Author.Name != "" { + return it.Author.Name + } + if len(it.Authors) > 0 { + return it.Authors[0].Name + } + return "" +} + +func published(it *gofeed.Item) time.Time { + if it.PublishedParsed != nil { + return it.PublishedParsed.UTC() + } + if it.UpdatedParsed != nil { + return it.UpdatedParsed.UTC() + } + raw := strings.TrimSpace(it.Published) + if raw == "" { + raw = strings.TrimSpace(it.Updated) + } + for _, layout := range timeLayouts { + if t, err := time.Parse(layout, raw); err == nil { + return t.UTC() + } + } + return time.Time{} +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/parse/parse_test.go b/PROJECTS/intermediate/security-news-scraper/internal/parse/parse_test.go new file mode 100644 index 00000000..4c21c79e --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/parse/parse_test.go @@ -0,0 +1,105 @@ +// ©AngelaMos | 2026 +// parse_test.go + +package parse + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +const feedsDir = "../../testdata/feeds" + +func loadFeed(t *testing.T, name string) []Item { + t.Helper() + f, err := os.Open(filepath.Join(feedsDir, name)) + if err != nil { + t.Fatalf("open fixture %s: %v", name, err) + } + t.Cleanup(func() { _ = f.Close() }) + items, err := Feed(f) + if err != nil { + t.Fatalf("Feed(%s): %v", name, err) + } + return items +} + +func TestFeedGolden(t *testing.T) { + cases := []struct { + file string + wantItems int + fullBody bool + }{ + {"krebs.xml", 3, true}, + {"theregister.xml", 3, true}, + {"thehackernews.xml", 3, false}, + {"bleepingcomputer.xml", 3, false}, + {"securityweek.xml", 3, false}, + {"darkreading.xml", 3, false}, + {"cisa.xml", 3, false}, + } + for _, tc := range cases { + t.Run(tc.file, func(t *testing.T) { + items := loadFeed(t, tc.file) + if len(items) != tc.wantItems { + t.Fatalf("items = %d, want %d", len(items), tc.wantItems) + } + for i, it := range items { + if it.Title == "" { + t.Errorf("item %d: empty title", i) + } + if it.Link == "" { + t.Errorf("item %d: empty link", i) + } + if it.Published.IsZero() { + t.Errorf("item %d (%q): published time did not parse", i, it.Title) + } + if tc.fullBody && it.Body == "" { + t.Errorf("item %d (%q): expected full body, got empty", i, it.Title) + } + } + }) + } +} + +func TestFeedExactPublishedTime(t *testing.T) { + cases := []struct { + file string + raw string + }{ + {"thehackernews.xml", "Sat, 04 Jul 2026 18:17:53 +0530"}, + {"krebs.xml", "Thu, 02 Jul 2026 19:27:33 +0000"}, + } + for _, tc := range cases { + t.Run(tc.file, func(t *testing.T) { + want, err := time.Parse(time.RFC1123Z, tc.raw) + if err != nil { + t.Fatalf("parse expected time: %v", err) + } + items := loadFeed(t, tc.file) + if !items[0].Published.Equal(want) { + t.Errorf("published = %s, want %s", items[0].Published, want.UTC()) + } + }) + } +} + +func TestFeedExactTitles(t *testing.T) { + cases := []struct { + file string + title string + }{ + {"krebs.xml", "FBI Seizes NetNut Proxy Platform, Popa Botnet"}, + {"thehackernews.xml", "U.S. Government Entity Paid Kairos $1 Million in Data-Theft Extortion Case"}, + } + for _, tc := range cases { + t.Run(tc.file, func(t *testing.T) { + items := loadFeed(t, tc.file) + if items[0].Title != tc.title { + t.Errorf("first title = %q, want %q", items[0].Title, tc.title) + } + }) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/fetch_state_test.go b/PROJECTS/intermediate/security-news-scraper/internal/store/fetch_state_test.go new file mode 100644 index 00000000..1e89e2f1 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/fetch_state_test.go @@ -0,0 +1,81 @@ +// ©AngelaMos | 2026 +// fetch_state_test.go + +package store + +import "testing" + +func seedSource(t *testing.T, s *Store) int64 { + t.Helper() + id, err := s.UpsertSource(SourceInput{ + Name: "krebs", Title: "Krebs", URL: "https://krebsonsecurity.com/feed/", + Type: "rss", Weight: 1.0, Tags: []string{"news"}, Enabled: true, + }) + if err != nil { + t.Fatalf("UpsertSource: %v", err) + } + return id +} + +func TestFetchStateMissing(t *testing.T) { + s := openTemp(t) + id := seedSource(t, s) + fs, ok, err := s.GetFetchState(id) + if err != nil { + t.Fatalf("GetFetchState: %v", err) + } + if ok { + t.Error("expected no fetch_state for fresh source") + } + if fs != (FetchState{}) { + t.Errorf("expected zero FetchState, got %+v", fs) + } +} + +func TestFetchStateRoundTrip(t *testing.T) { + s := openTemp(t) + id := seedSource(t, s) + + want := FetchState{ETag: `"v1"`, LastModified: "Wed, 01 Jul 2026 00:00:00 GMT", LastFetched: 1700, LastStatus: 200} + if err := s.UpsertFetchState(id, want); err != nil { + t.Fatalf("UpsertFetchState: %v", err) + } + + got, ok, err := s.GetFetchState(id) + if err != nil { + t.Fatalf("GetFetchState: %v", err) + } + if !ok { + t.Fatal("expected fetch_state to exist") + } + if got != want { + t.Errorf("round-trip = %+v, want %+v", got, want) + } + + updated := FetchState{ETag: `"v2"`, LastModified: "Thu, 02 Jul 2026 00:00:00 GMT", LastFetched: 1800, LastStatus: 304} + if err := s.UpsertFetchState(id, updated); err != nil { + t.Fatalf("second UpsertFetchState: %v", err) + } + got, _, _ = s.GetFetchState(id) + if got != updated { + t.Errorf("after update = %+v, want %+v", got, updated) + } +} + +func TestInsertArticleStoresTitleHash(t *testing.T) { + s := openTemp(t) + id := seedSource(t, s) + if _, err := s.InsertArticle(Article{ + SourceID: id, CanonicalURL: "https://example.com/a", ContentHash: "hash1", + TitleHash: "thash1", Title: "A", + }); err != nil { + t.Fatalf("InsertArticle: %v", err) + } + var th string + if err := s.DB().QueryRow(`SELECT title_hash FROM articles WHERE content_hash = ?`, "hash1").Scan(&th); err != nil { + t.Fatal(err) + } + if th != "thash1" { + t.Errorf("title_hash = %q, want thash1", th) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0002_article_title_hash.sql b/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0002_article_title_hash.sql new file mode 100644 index 00000000..85d0c17f --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0002_article_title_hash.sql @@ -0,0 +1,6 @@ +-- ©AngelaMos | 2026 +-- 0002_article_title_hash.sql + +ALTER TABLE articles ADD COLUMN title_hash TEXT NOT NULL DEFAULT ''; + +CREATE INDEX idx_articles_title_hash ON articles(title_hash); diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go index dc874de7..a19dddc7 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go @@ -45,6 +45,7 @@ type Article struct { SourceID int64 CanonicalURL string ContentHash string + TitleHash string Title string Summary string Body string @@ -53,6 +54,13 @@ type Article struct { FetchedAt int64 } +type FetchState struct { + ETag string + LastModified string + LastFetched int64 + LastStatus int64 +} + func Open(path string) (*Store, error) { dsn := fmt.Sprintf("file:%s?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)", path) db, err := sql.Open("sqlite", dsn) @@ -114,9 +122,9 @@ func (s *Store) GetSourceByName(name string) (SourceRow, error) { func (s *Store) InsertArticle(a Article) (int64, error) { res, err := s.db.Exec(` INSERT INTO articles - (source_id, canonical_url, content_hash, title, summary, body, author, published_at, fetched_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, - a.SourceID, a.CanonicalURL, a.ContentHash, a.Title, a.Summary, a.Body, + (source_id, canonical_url, content_hash, title_hash, title, summary, body, author, published_at, fetched_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + a.SourceID, a.CanonicalURL, a.ContentHash, a.TitleHash, a.Title, a.Summary, a.Body, a.Author, a.PublishedAt, a.FetchedAt, ) if err != nil { @@ -141,6 +149,36 @@ func (s *Store) CountArticles() (int, error) { return n, nil } +func (s *Store) GetFetchState(sourceID int64) (FetchState, bool, error) { + var fs FetchState + err := s.db.QueryRow(` + SELECT etag, last_modified, last_fetched, last_status + FROM fetch_state WHERE source_id = ?`, sourceID, + ).Scan(&fs.ETag, &fs.LastModified, &fs.LastFetched, &fs.LastStatus) + if errors.Is(err, sql.ErrNoRows) { + return FetchState{}, false, nil + } + if err != nil { + return FetchState{}, false, fmt.Errorf("get fetch_state %d: %w", sourceID, err) + } + return fs, true, nil +} + +func (s *Store) UpsertFetchState(sourceID int64, fs FetchState) error { + _, err := s.db.Exec(` + INSERT INTO fetch_state (source_id, etag, last_modified, last_fetched, last_status) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(source_id) DO UPDATE SET + etag = excluded.etag, last_modified = excluded.last_modified, + last_fetched = excluded.last_fetched, last_status = excluded.last_status`, + sourceID, fs.ETag, fs.LastModified, fs.LastFetched, fs.LastStatus, + ) + if err != nil { + return fmt.Errorf("upsert fetch_state %d: %w", sourceID, err) + } + return nil +} + func boolToInt(b bool) int { if b { return 1 diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/bleepingcomputer.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/bleepingcomputer.xml new file mode 100644 index 00000000..311dc127 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/bleepingcomputer.xml @@ -0,0 +1,62 @@ + + + + + BleepingComputer + + https://www.bleepingcomputer.com/ + BleepingComputer - All Stories + Sun, 05 Jul 2026 16:55:00 GMT + https://www.bleepingcomputer.com/ + en + + + + Flipper Zero firmware development continues with community help + https://www.bleepingcomputer.com/news/security/flipper-zero-firmware-development-continues-with-community-help/ + Sun, 05 Jul 2026 10:14:52 -0400 + Bill Toulas + + + + + + https://www.bleepingcomputer.com/news/security/flipper-zero-firmware-development-continues-with-community-help/ + + + + JadePuffer ransomware used AI agent to automate entire attack + https://www.bleepingcomputer.com/news/security/jadepuffer-ransomware-used-ai-agent-to-automate-entire-attack/ + Sat, 04 Jul 2026 10:16:38 -0400 + Bill Toulas + + + + + + + + https://www.bleepingcomputer.com/news/security/jadepuffer-ransomware-used-ai-agent-to-automate-entire-attack/ + + + + NetNut proxy network disrupted, 2 million infected devices cut off + https://www.bleepingcomputer.com/news/security/netnut-proxy-network-disrupted-2-million-infected-devices-cut-off/ + Fri, 03 Jul 2026 13:50:04 -0400 + Ionut Ilascu + + + + + + https://www.bleepingcomputer.com/news/security/netnut-proxy-network-disrupted-2-million-infected-devices-cut-off/ + + + + + diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/cisa.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/cisa.xml new file mode 100644 index 00000000..3c9971e4 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/cisa.xml @@ -0,0 +1,598 @@ + + + + All CISA Advisories + https://www.cisa.gov/ + + en + + + CubeSpace CW0057 Reaction Wheel + https://www.cisa.gov/news-events/ics-advisories/icsa-26-183-02 + <p><a href="https://github.com/cisagov/CSAF/blob/develop/csaf_files/OT/white/2026/icsa-26-183-02.json"><strong>View CSAF</strong></a></p> +<h2>Summary</h2> +<p><strong>Successful exploitation of this vulnerability could allow an attacker to upload arbitrary malicious firmware to the device.</strong></p> +<p>The following versions of CubeSpace CW0057 Reaction Wheel are affected:</p> +<ul> +<li>CW0057 Reaction Wheel</li> +</ul> +<div class="csaf-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS</th> +<th role="columnheader">Vendor</th> +<th role="columnheader">Equipment</th> +<th role="columnheader">Vulnerabilities</th> +</tr> +</thead> +<tbody> +<tr> +<td>v3 6.1</td> +<td>CubeSpace</td> +<td>CubeSpace CW0057 Reaction Wheel</td> +<td>Improper Verification of Cryptographic Signature</td> +</tr> +</tbody> +</table> +</div> +<h3>Background</h3> +<ul> +<li><strong>Critical Infrastructure Sectors: </strong>Communications</li> +<li><strong>Countries/Areas Deployed: </strong>Worldwide</li> +<li><strong>Company Headquarters Location: </strong>South Africa</li> +</ul> +<hr> +<h2>Vulnerabilities</h2> +<div class="csaf-accordion"> +<p><a class="csaf-accordion-toggle-all" href="#">Expand All +</a></p> +<div class="csaf-accordion-item"> +<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-13743</a></h3> +<div class="csaf-accordion-content"> +<p>CubeSpace CW0057 Reaction Wheel firmware versions prior to 5.0.20 are vulnerable to an Improper Verification of Cryptographic Signature vulnerability. This could allow an attacker with physical access to the product to upload arbitrary malicious firmware to the device without authentication.</p> +<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-13743">View CVE Details</a></p> +<hr> +<h4>Affected Products</h4> +<h5>CubeSpace CW0057 Reaction Wheel</h5> +<div class="ics-vendor-version-status"> +<div class="ics-vendor"><strong>Vendor:</strong><br>CubeSpace</div> +<div class="ics-version"><strong>Product Version:</strong><br>CubeSpace CW0057 Reaction Wheel: &lt;firmware_5.0.20</div> +<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div> +</div> +<div class="ics-remediations"> +<h6>Remediations</h6> +<p><strong>Vendor fix</strong><br>CubeSpace has released the following firmware versions for users to enable: Firmware version 5.0.20. Firmware version 5.0.20 introduces the capability for cryptographically verified secure boot; however, this protection is not enabled by default. Users must activate signed‑boot functionality, particularly the fully immutable mode, to achieve full security.</p> +<p><strong>Mitigation</strong><br>CubeSpace acknowledges the finding. The CW0057 reaction wheel authenticates firmware updates with a CRC-32 integrity check, which confirms image integrity but does not verify the source of an image. Exploitation requires direct physical access to the device and is not exploitable remotely. A device affected by this method remains recoverable: the bootloader operates independently of the application firmware and can reload known-good, CubeSpace-supplied images, so an affected unit cannot be permanently disabled by this method. Starting with firmware version 5.0.20, CubeSpace offers optional cryptographic secure boot of varying security levels which customers can enable. Given the physical-access prerequisite and the availability of recovery, CubeSpace assesses the practical risk as low.</p> +</div> +<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/347.html">CWE-347 Improper Verification of Cryptographic Signature</a></p> +<hr> +<h4>Metrics</h4> +<div class="csaf-table csaf-metrics-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th> +<th role="columnheader">Base Score</th> +<th role="columnheader">Base Severity</th> +<th role="columnheader">Vector String</th> +</tr> +</thead> +<tbody> +<tr> +<td>3.1</td> +<td>6.1</td> +<td>MEDIUM</td> +<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:P/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:H">CVSS:3.1/AV:P/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:H</a></td> +</tr> +<tr> +<td>4.0</td> +<td>3.3</td> +<td>LOW</td> +<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:P/AC:L/AT:N/PR:N/UI:N/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N/E:P">CVSS:4.0/AV:P/AC:L/AT:N/PR:N/UI:N/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N/E:P</a></td> +</tr> +</tbody> +</table> +</div> +</div> +</div> +</div> +<hr> +<h2>Acknowledgments</h2> +<ul> +<li>Anthony Rose reported this vulnerability to CISA</li> +</ul> +<hr> +<h2>Legal Notice and Terms of Use</h2> +<p>This product is provided subject to this Notification (https://www.cisa.gov/notification) and this Privacy &amp; Use policy (https://www.cisa.gov/privacy-policy).</p> +<hr> +<h2>Recommended Practices</h2> +<p>CISA recommends users take defensive measures to minimize the risk of exploitation of this vulnerability.</p> +<p>Minimize network exposure for all control system devices and/or systems, ensuring they are not accessible from the internet.</p> +<p>Locate control system networks and remote devices behind firewalls and isolating them from business networks.</p> +<p>When remote access is required, use more secure methods, such as Virtual Private Networks (VPNs), recognizing VPNs may have vulnerabilities and should be updated to the most current version available. Also recognize VPN is only as secure as the connected devices.</p> +<p>CISA reminds organizations to perform proper impact analysis and risk assessment prior to deploying defensive measures.</p> +<p>CISA also provides a section for control systems security recommended practices on the ICS webpage on cisa.gov/ics. Several CISA products detailing cyber defense best practices are available for reading and download, including Improving Industrial Control Systems Cybersecurity with Defense-in-Depth Strategies.</p> +<p>CISA encourages organizations to implement recommended cybersecurity strategies for proactive defense of ICS assets.</p> +<p>Additional mitigation guidance and recommended practices are publicly available on the ICS webpage at cisa.gov/ics in the technical information paper, ICS-TIP-12-146-01B--Targeted Cyber Intrusion Detection and Mitigation Strategies.</p> +<p>Organizations observing suspected malicious activity should follow established internal procedures and report findings to CISA for tracking and correlation against other incidents.</p> +<p>CISA also recommends users take the following measures to protect themselves from social engineering attacks:</p> +<p>Do not click web links or open attachments in unsolicited email messages.</p> +<p>Refer to Recognizing and Avoiding Email Scams for more information on avoiding email scams.</p> +<p>Refer to Avoiding Social Engineering and Phishing Attacks for more information on social engineering attacks.</p> +<p>No known public exploitation specifically targeting this vulnerability has been reported to CISA at this time. This vulnerability is not exploitable remotely.</p> +<hr> +<h2>Revision History</h2> +<ul> +<li><strong>Initial Release Date: </strong>2026-07-02</li> +</ul> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">Date</th> +<th role="columnheader">Revision</th> +<th role="columnheader">Summary</th> +</tr> +</thead> +<tbody> +<tr> +<td>2026-07-02</td> +<td>1</td> +<td>Initial Publication</td> +</tr> +</tbody> +</table> +<hr> +<h2>Legal Notice and Terms of Use</h2> + + Thu, 02 Jul 26 12:00:00 +0000 + CISA + /node/25107 + + + Gardyn IoT Hub + https://www.cisa.gov/news-events/ics-advisories/icsa-26-183-03 + <p><a href="https://github.com/cisagov/CSAF/blob/develop/csaf_files/OT/white/2026/icsa-26-183-03.json"><strong>View CSAF</strong></a></p> +<h2>Summary</h2> +<p><strong>Successful exploitation of these vulnerabilities could allow unauthenticated users to access and control IoT Hub managed devices.</strong></p> +<p>The following versions of Gardyn IoT Hub are affected:</p> +<ul> +<li>Home Firmware</li> +<li>Studio Firmware</li> +<li>Cloud API &lt;2.12.2026 (CVE-2026-13768, CVE-2026-55726, CVE-2026-54477)</li> +</ul> +<div class="csaf-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS</th> +<th role="columnheader">Vendor</th> +<th role="columnheader">Equipment</th> +<th role="columnheader">Vulnerabilities</th> +</tr> +</thead> +<tbody> +<tr> +<td>v3 10</td> +<td>Gardyn</td> +<td>Gardyn IoT Hub</td> +<td>Use of Hard-coded Credentials, Exposure of Sensitive System Information to an Unauthorized Control Sphere, Improper Neutralization of HTTP Headers for Scripting Syntax</td> +</tr> +</tbody> +</table> +</div> +<h3>Background</h3> +<ul> +<li><strong>Critical Infrastructure Sectors: </strong>Food and Agriculture</li> +<li><strong>Countries/Areas Deployed: </strong>United States</li> +<li><strong>Company Headquarters Location: </strong>United States</li> +</ul> +<hr> +<h2>Vulnerabilities</h2> +<div class="csaf-accordion"> +<p><a class="csaf-accordion-toggle-all" href="#">Expand All +</a></p> +<div class="csaf-accordion-item"> +<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-13768</a></h3> +<div class="csaf-accordion-content"> +<p>Gardyn devices expose a privileged iothubowner key. Access to this key will allow a malicious user to invoke an IoTHub Registry Manager function which returns connection information for all Gardyn Home Kit and Studio devices. Access to this key also allows a malicious user to execute arbitrary commands on a specific connected device and may allow the malicious user to pivot to other devices on the user's network.</p> +<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-13768">View CVE Details</a></p> +<hr> +<h4>Affected Products</h4> +<h5>Gardyn IoT Hub</h5> +<div class="ics-vendor-version-status"> +<div class="ics-vendor"><strong>Vendor:</strong><br>Gardyn</div> +<div class="ics-version"><strong>Product Version:</strong><br>Gardyn Home Firmware: &lt;master.627, Gardyn Studio Firmware: &lt;master.627, Gardyn Cloud API: &lt;2.12.2026</div> +<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div> +</div> +<div class="ics-remediations"> +<h6>Remediations</h6> +<p><strong>Mitigation</strong><br>Gardyn states that IoT Hub deployed infrastructure has been updated to fix the listed vulnerabilities.</p> +<p><strong>Mitigation</strong><br>Gardyn requests that users ensure their devices have Internet connectivity in order to automatically download needed firmware updates. Unconnected devices will automatically update when configured with a working Internet connection. Gardyn also recommends that users update their mobile application to the most recent version. The current versions of the Gardyn App and the Gardyn Home firmware can be checked in the Gardyn App.</p> +<p><strong>Mitigation</strong><br>Further information on Gardyn security can be found here: https://mygardyn.com/security/<br><a href="https://mygardyn.com/security/">https://mygardyn.com/security/</a></p> +<p><strong>Mitigation</strong><br>Further customer support can be obtained from Gardyn at: support@mygardyn.com<br><a href="mailto:support@mygardyn.com">mailto:support@mygardyn.com</a></p> +</div> +<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/798.html">CWE-798 Use of Hard-coded Credentials</a></p> +<hr> +<h4>Metrics</h4> +<div class="csaf-table csaf-metrics-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th> +<th role="columnheader">Base Score</th> +<th role="columnheader">Base Severity</th> +<th role="columnheader">Vector String</th> +</tr> +</thead> +<tbody> +<tr> +<td>3.1</td> +<td>10</td> +<td>CRITICAL</td> +<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:L">CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:L</a></td> +</tr> +<tr> +<td>4.0</td> +<td>9.5</td> +<td>CRITICAL</td> +<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:N/VC:H/VI:H/VA:L/SC:H/SI:H/SA:L">CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:N/VC:H/VI:H/VA:L/SC:H/SI:H/SA:L</a></td> +</tr> +</tbody> +</table> +</div> +</div> +</div> +<div class="csaf-accordion-item"> +<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-55726</a></h3> +<div class="csaf-accordion-content"> +<p>The Azure Blob Storage container used for Gardyn device logs is publicly listable without authentication. A malicious user would be able to access any device log file available in the blob storage container.</p> +<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-55726">View CVE Details</a></p> +<hr> +<h4>Affected Products</h4> +<h5>Gardyn IoT Hub</h5> +<div class="ics-vendor-version-status"> +<div class="ics-vendor"><strong>Vendor:</strong><br>Gardyn</div> +<div class="ics-version"><strong>Product Version:</strong><br>Gardyn Home Firmware: &lt;master.627, Gardyn Studio Firmware: &lt;master.627, Gardyn Cloud API: &lt;2.12.2026</div> +<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div> +</div> +<div class="ics-remediations"> +<h6>Remediations</h6> +<p><strong>Mitigation</strong><br>Gardyn states that IoT Hub deployed infrastructure has been updated to fix the listed vulnerabilities.</p> +<p><strong>Mitigation</strong><br>Gardyn requests that users ensure their devices have Internet connectivity in order to automatically download needed firmware updates. Unconnected devices will automatically update when configured with a working Internet connection. Gardyn also recommends that users update their mobile application to the most recent version. The current versions of the Gardyn App and the Gardyn Home firmware can be checked in the Gardyn App.</p> +<p><strong>Mitigation</strong><br>Further information on Gardyn security can be found here: https://mygardyn.com/security/<br><a href="https://mygardyn.com/security/">https://mygardyn.com/security/</a></p> +<p><strong>Mitigation</strong><br>Further customer support can be obtained from Gardyn at: support@mygardyn.com<br><a href="mailto:support@mygardyn.com">mailto:support@mygardyn.com</a></p> +</div> +<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/497.html">CWE-497 Exposure of Sensitive System Information to an Unauthorized Control Sphere</a></p> +<hr> +<h4>Metrics</h4> +<div class="csaf-table csaf-metrics-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th> +<th role="columnheader">Base Score</th> +<th role="columnheader">Base Severity</th> +<th role="columnheader">Vector String</th> +</tr> +</thead> +<tbody> +<tr> +<td>3.1</td> +<td>5.3</td> +<td>MEDIUM</td> +<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:N/A:N">CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:N/A:N</a></td> +</tr> +<tr> +<td>4.0</td> +<td>6.9</td> +<td>MEDIUM</td> +<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:L/VI:N/VA:N/SC:N/SI:N/SA:N">CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:L/VI:N/VA:N/SC:N/SI:N/SA:N</a></td> +</tr> +</tbody> +</table> +</div> +</div> +</div> +<div class="csaf-accordion-item"> +<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-54477</a></h3> +<div class="csaf-accordion-content"> +<p>The admin panel lacks standard security headers, enabling clickjacking and cross-site scripting attacks.</p> +<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-54477">View CVE Details</a></p> +<hr> +<h4>Affected Products</h4> +<h5>Gardyn IoT Hub</h5> +<div class="ics-vendor-version-status"> +<div class="ics-vendor"><strong>Vendor:</strong><br>Gardyn</div> +<div class="ics-version"><strong>Product Version:</strong><br>Gardyn Home Firmware: &lt;master.627, Gardyn Studio Firmware: &lt;master.627, Gardyn Cloud API: &lt;2.12.2026</div> +<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div> +</div> +<div class="ics-remediations"> +<h6>Remediations</h6> +<p><strong>Mitigation</strong><br>Gardyn states that IoT Hub deployed infrastructure has been updated to fix the listed vulnerabilities.</p> +<p><strong>Mitigation</strong><br>Gardyn requests that users ensure their devices have Internet connectivity in order to automatically download needed firmware updates. Unconnected devices will automatically update when configured with a working Internet connection. Gardyn also recommends that users update their mobile application to the most recent version. The current versions of the Gardyn App and the Gardyn Home firmware can be checked in the Gardyn App.</p> +<p><strong>Mitigation</strong><br>Further information on Gardyn security can be found here: https://mygardyn.com/security/<br><a href="https://mygardyn.com/security/">https://mygardyn.com/security/</a></p> +<p><strong>Mitigation</strong><br>Further customer support can be obtained from Gardyn at: support@mygardyn.com<br><a href="mailto:support@mygardyn.com">mailto:support@mygardyn.com</a></p> +</div> +<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/644.html">CWE-644 Improper Neutralization of HTTP Headers for Scripting Syntax</a></p> +<hr> +<h4>Metrics</h4> +<div class="csaf-table csaf-metrics-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th> +<th role="columnheader">Base Score</th> +<th role="columnheader">Base Severity</th> +<th role="columnheader">Vector String</th> +</tr> +</thead> +<tbody> +<tr> +<td>3.1</td> +<td>5.4</td> +<td>MEDIUM</td> +<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:L/I:L/A:N">CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:L/I:L/A:N</a></td> +</tr> +<tr> +<td>4.0</td> +<td>5.1</td> +<td>MEDIUM</td> +<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N">CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N</a></td> +</tr> +</tbody> +</table> +</div> +</div> +</div> +</div> +<hr> +<h2>Acknowledgments</h2> +<ul> +<li>Michael Groberman reported these vulnerabilities to CISA</li> +</ul> +<hr> +<h2>Legal Notice and Terms of Use</h2> +<p>This product is provided subject to this Notification (https://www.cisa.gov/notification) and this Privacy &amp; Use policy (https://www.cisa.gov/privacy-policy).</p> +<hr> +<h2>Recommended Practices</h2> +<p>CISA recommends users take defensive measures to minimize the risk of exploitation of these vulnerabilities.</p> +<p>Minimize network exposure for all control system devices and/or systems, ensuring they are not accessible from the Internet.</p> +<p>Locate control system networks and remote devices behind firewalls and isolating them from business networks.</p> +<p>When remote access is required, use more secure methods, such as Virtual Private Networks (VPNs), recognizing VPNs may have vulnerabilities and should be updated to the most current version available. Also recognize VPN is only as secure as the connected devices.</p> +<p>CISA reminds organizations to perform proper impact analysis and risk assessment prior to deploying defensive measures.</p> +<p>CISA also provides a section for control systems security recommended practices on the ICS webpage on cisa.gov/ics. Several CISA products detailing cyber defense best practices are available for reading and download, including Improving Industrial Control Systems Cybersecurity with Defense-in-Depth Strategies.</p> +<p>CISA encourages organizations to implement recommended cybersecurity strategies for proactive defense of ICS assets.</p> +<p>Additional mitigation guidance and recommended practices are publicly available on the ICS webpage at cisa.gov/ics in the technical information paper, ICS-TIP-12-146-01B--Targeted Cyber Intrusion Detection and Mitigation Strategies.</p> +<p>Organizations observing suspected malicious activity should follow established internal procedures and report findings to CISA for tracking and correlation against other incidents.</p> +<p>No known public exploitation specifically targeting these vulnerabilities has been reported to CISA at this time.</p> +<hr> +<h2>Revision History</h2> +<ul> +<li><strong>Initial Release Date: </strong>2026-07-02</li> +</ul> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">Date</th> +<th role="columnheader">Revision</th> +<th role="columnheader">Summary</th> +</tr> +</thead> +<tbody> +<tr> +<td>2026-07-02</td> +<td>1</td> +<td>Initial Publication</td> +</tr> +</tbody> +</table> +<hr> +<h2>Legal Notice and Terms of Use</h2> + + Thu, 02 Jul 26 12:00:00 +0000 + CISA + /node/25108 + + + ST Engineering iDirect iQ-Series Terminals + https://www.cisa.gov/news-events/ics-advisories/icsa-26-183-01 + <p><a href="https://github.com/cisagov/CSAF/blob/develop/csaf_files/OT/white/2026/icsa-26-183-01.json"><strong>View CSAF</strong></a></p> +<h2>Summary</h2> +<p><strong>Successful exploitation of these vulnerabilities could allow an attacker to gain unauthorized access to device information or cause a denial-of-service condition.</strong></p> +<p>The following versions of ST Engineering iDirect iQ-Series Terminals are affected:</p> +<ul> +<li>Evolution iQ‑Series terminals &lt;=4.5.2.1 (CVE-2026-38059, CVE-2026-38057)</li> +<li>3315‑Series terminals &lt;=4.5.2.1 (CVE-2026-38059, CVE-2026-38057)</li> +<li>9‑Series terminals &lt;=4.5.2.1 (CVE-2026-38059, CVE-2026-38057)</li> +</ul> +<div class="csaf-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS</th> +<th role="columnheader">Vendor</th> +<th role="columnheader">Equipment</th> +<th role="columnheader">Vulnerabilities</th> +</tr> +</thead> +<tbody> +<tr> +<td>v3 8.1</td> +<td>ST Engineering iDirect</td> +<td>ST Engineering iDirect iQ-Series Terminals</td> +<td>Missing Authentication for Critical Function, Cross-Site Request Forgery (CSRF)</td> +</tr> +</tbody> +</table> +</div> +<h3>Background</h3> +<ul> +<li><strong>Critical Infrastructure Sectors: </strong>Communications, Defense Industrial Base, Energy, Government Services and Facilities, Transportation Systems</li> +<li><strong>Countries/Areas Deployed: </strong>Worldwide</li> +<li><strong>Company Headquarters Location: </strong>United States</li> +</ul> +<hr> +<h2>Vulnerabilities</h2> +<div class="csaf-accordion"> +<p><a class="csaf-accordion-toggle-all" href="#">Expand All +</a></p> +<div class="csaf-accordion-item"> +<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-38059</a></h3> +<div class="csaf-accordion-content"> +<p>The iDirect iQ200 exposes the /api/identity and /api/ REST API endpoints without authentication. An unauthenticated attacker with network access can retrieve sensitive device information including the serial number, Device ID (DID), Terminal Private Key identifier (TPK), MAC address, and exact firmware version. The DID and TPK are used for satellite network authentication in the iDirect platform, potentially enabling terminal impersonation and network reconnaissance.</p> +<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-38059">View CVE Details</a></p> +<hr> +<h4>Affected Products</h4> +<h5>ST Engineering iDirect iQ-Series Terminals</h5> +<div class="ics-vendor-version-status"> +<div class="ics-vendor"><strong>Vendor:</strong><br>ST Engineering iDirect</div> +<div class="ics-version"><strong>Product Version:</strong><br>ST Engineering iDirect Evolution iQ‑Series terminals: &lt;=4.5.2.1, ST Engineering iDirect 3315‑Series terminals: &lt;=4.5.2.1, ST Engineering iDirect 9‑Series terminals: &lt;=4.5.2.1</div> +<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div> +</div> +<div class="ics-remediations"> +<h6>Remediations</h6> +<p><strong>Mitigation</strong><br>ST Engineering iDirect has fixed the vulnerabilities and recommend users update the software to version 4.5.2.2 or newer.</p> +<p><strong>Mitigation</strong><br>Registered users are able to download patches from the iDirect Support Portal https://support.idirect.net/s/login.<br><a href="https://support.idirect.net/s/login">https://support.idirect.net/s/login</a></p> +<p><strong>Mitigation</strong><br>Restrict management interfaces to trusted networks (e.g., VPN, ACLs).</p> +<p><strong>Mitigation</strong><br>Avoid exposing administrative APIs to the public internet.</p> +<p><strong>Mitigation</strong><br>Enforce strong authentication practices.</p> +<p><strong>Mitigation</strong><br>Monitor for anomalous API activity and unexpected device reboots.</p> +</div> +<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/306.html">CWE-306 Missing Authentication for Critical Function</a></p> +<hr> +<h4>Metrics</h4> +<div class="csaf-table csaf-metrics-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th> +<th role="columnheader">Base Score</th> +<th role="columnheader">Base Severity</th> +<th role="columnheader">Vector String</th> +</tr> +</thead> +<tbody> +<tr> +<td>3.1</td> +<td>7.5</td> +<td>HIGH</td> +<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N">CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N</a></td> +</tr> +<tr> +<td>4.0</td> +<td>8.7</td> +<td>HIGH</td> +<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:N/VA:N/SC:N/SI:N/SA:N">CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:N/VA:N/SC:N/SI:N/SA:N</a></td> +</tr> +</tbody> +</table> +</div> +</div> +</div> +<div class="csaf-accordion-item"> +<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-38057</a></h3> +<div class="csaf-accordion-content"> +<p>The iDirect iQ200 does not validate CSRF tokens on state-changing API endpoints after authentication. The /api/reboot endpoint accepts POST requests authenticated solely by a session cookie that lacks the SameSite attribute. A remote attacker can host a malicious web page that, when visited by an authenticated administrator, automatically submits a cross-site POST request causing an immediate device reboot and satellite link loss. Repeated attacks can sustain a denial-of-service condition.</p> +<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-38057">View CVE Details</a></p> +<hr> +<h4>Affected Products</h4> +<h5>ST Engineering iDirect iQ-Series Terminals</h5> +<div class="ics-vendor-version-status"> +<div class="ics-vendor"><strong>Vendor:</strong><br>ST Engineering iDirect</div> +<div class="ics-version"><strong>Product Version:</strong><br>ST Engineering iDirect Evolution iQ‑Series terminals: &lt;=4.5.2.1, ST Engineering iDirect 3315‑Series terminals: &lt;=4.5.2.1, ST Engineering iDirect 9‑Series terminals: &lt;=4.5.2.1</div> +<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div> +</div> +<div class="ics-remediations"> +<h6>Remediations</h6> +<p><strong>Mitigation</strong><br>ST Engineering iDirect has fixed the vulnerabilities and recommend users update the software to version 4.5.2.2 or newer.</p> +<p><strong>Mitigation</strong><br>Registered users are able to download patches from the iDirect Support Portal https://support.idirect.net/s/login.<br><a href="https://support.idirect.net/s/login">https://support.idirect.net/s/login</a></p> +<p><strong>Mitigation</strong><br>Restrict management interfaces to trusted networks (e.g., VPN, ACLs).</p> +<p><strong>Mitigation</strong><br>Avoid exposing administrative APIs to the public internet.</p> +<p><strong>Mitigation</strong><br>Enforce strong authentication practices.</p> +<p><strong>Mitigation</strong><br>Monitor for anomalous API activity and unexpected device reboots.</p> +</div> +<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/352.html">CWE-352 Cross-Site Request Forgery (CSRF)</a></p> +<hr> +<h4>Metrics</h4> +<div class="csaf-table csaf-metrics-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th> +<th role="columnheader">Base Score</th> +<th role="columnheader">Base Severity</th> +<th role="columnheader">Vector String</th> +</tr> +</thead> +<tbody> +<tr> +<td>3.1</td> +<td>8.1</td> +<td>HIGH</td> +<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:H/A:H">CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:H/A:H</a></td> +</tr> +<tr> +<td>4.0</td> +<td>7</td> +<td>HIGH</td> +<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N">CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N</a></td> +</tr> +</tbody> +</table> +</div> +</div> +</div> +</div> +<hr> +<h2>Acknowledgments</h2> +<ul> +<li>Ahmed Alqahtani of Aramco reported these vulnerabilities to CISA</li> +</ul> +<hr> +<h2>Legal Notice and Terms of Use</h2> +<p>This product is provided subject to this Notification (https://www.cisa.gov/notification) and this Privacy &amp; Use policy (https://www.cisa.gov/privacy-policy).</p> +<hr> +<h2>Recommended Practices</h2> +<p>CISA recommends users take defensive measures to minimize the risk of exploitation of these vulnerabilities.</p> +<p>Minimize network exposure for all control system devices and/or systems, ensuring they are not accessible from the internet.</p> +<p>Locate control system networks and remote devices behind firewalls and isolating them from business networks.</p> +<p>When remote access is required, use more secure methods, such as Virtual Private Networks (VPNs), recognizing VPNs may have vulnerabilities and should be updated to the most current version available. Also recognize VPN is only as secure as the connected devices.</p> +<p>CISA reminds organizations to perform proper impact analysis and risk assessment prior to deploying defensive measures.</p> +<p>CISA also provides a section for control systems security recommended practices on the ICS webpage on cisa.gov/ics. Several CISA products detailing cyber defense best practices are available for reading and download, including Improving Industrial Control Systems Cybersecurity with Defense-in-Depth Strategies.</p> +<p>CISA encourages organizations to implement recommended cybersecurity strategies for proactive defense of ICS assets.</p> +<p>Additional mitigation guidance and recommended practices are publicly available on the ICS webpage at cisa.gov/ics in the technical information paper, ICS-TIP-12-146-01B--Targeted Cyber Intrusion Detection and Mitigation Strategies.</p> +<p>Organizations observing suspected malicious activity should follow established internal procedures and report findings to CISA for tracking and correlation against other incidents.</p> +<p>CISA also recommends users take the following measures to protect themselves from social engineering attacks:</p> +<p>Do not click web links or open attachments in unsolicited email messages.</p> +<p>Refer to Recognizing and Avoiding Email Scams for more information on avoiding email scams.</p> +<p>Refer to Avoiding Social Engineering and Phishing Attacks for more information on social engineering attacks.</p> +<p>No known public exploitation specifically targeting these vulnerabilities has been reported to CISA at this time.</p> +<hr> +<h2>Revision History</h2> +<ul> +<li><strong>Initial Release Date: </strong>2026-07-02</li> +</ul> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">Date</th> +<th role="columnheader">Revision</th> +<th role="columnheader">Summary</th> +</tr> +</thead> +<tbody> +<tr> +<td>2026-07-02</td> +<td>1</td> +<td>Initial Publication</td> +</tr> +</tbody> +</table> +<hr> +<h2>Legal Notice and Terms of Use</h2> + + Thu, 02 Jul 26 12:00:00 +0000 + CISA + /node/25106 + + + + diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/darkreading.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/darkreading.xml new file mode 100644 index 00000000..ddbc13c3 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/darkreading.xml @@ -0,0 +1,52 @@ + + + + darkreading + https://www.darkreading.com + Public RSS feed + en + + + <![CDATA[Chinese LLMs Broaden the Gap Between Attackers & Defenders]]> + + + Fri, 03 Jul 2026 13:01:00 GMT + Robert Lemos + + + + + castle-of-the-moors-Gabriela_Beres-shutterstock.jpg + + + + <![CDATA[Aussies Face Reduced Cybercrime Risk, as Pressure Shifts to SMBs]]> + + + Thu, 02 Jul 2026 23:01:00 GMT + Nate Nelson + + + + + Australia-imaginima-Getty.jpg + + + + <![CDATA[Apple Reverses Age-Old Patch Policy to Keep Up With AI]]> + + + Thu, 02 Jul 2026 19:31:58 GMT + Nate Nelson + + + + + Apple_bandage-imtmphoto-Getty.jpg + + + diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/krebs.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/krebs.xml new file mode 100644 index 00000000..4b80de7a --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/krebs.xml @@ -0,0 +1,245 @@ + + + + Krebs on Security + + https://krebsonsecurity.com + In-depth security news and investigation + Sat, 04 Jul 2026 16:16:19 +0000 + en-US + + hourly + + 1 + https://wordpress.org/?v=6.2.2 + + FBI Seizes NetNut Proxy Platform, Popa Botnet + https://krebsonsecurity.com/2026/07/fbi-seizes-netnut-proxy-platform-popa-botnet/ + https://krebsonsecurity.com/2026/07/fbi-seizes-netnut-proxy-platform-popa-botnet/#comments + + + Thu, 02 Jul 2026 19:27:33 +0000 + + + + + + + + + + + + + + + + + + https://krebsonsecurity.com/?p=73923 + + + The Federal Bureau of Investigation (FBI) said today it worked with industry partners to seize hundreds of domains associated with NetNut, a sprawling residential proxy service operated by the publicly-traded Israeli company Alarum Technologies [NASDAQ: ALAR]. The action comes roughly two weeks after KrebsOnSecurity published findings from multiple security firms connecting NetNut to the Popa botnet, a collection of at least two million devices that have been compromised by malicious software with little or no consent from victims.

+

The NetNut homepage today was replaced by this seizure banner from the FBI.

+

On June 19, three different security firms issued similar findings: That NetNut is a residential proxy network which populates a botnet called Popa, and distributes software for devices commonly found in homes, such as smart TVs and streaming boxes. NetNut’s software turns those systems into always-on residential proxy nodes that are rented to others, who predominantly use them to relay abusive and intrusive Internet traffic, such as mass content scraping, advertising fraud, and account takeover activity.

+

Earlier today, NetNut’s homepage was replaced with a seizure notice from the FBI and the Internal Revenue Service Criminal Investigation division. The seizure notice thanked Google, Lumen, Shadowserver and other industry partners for their help in dismantling hundreds of domains tied to the Popa botnet, which experts say has long been synonymous with NetNut’s residential proxy infrastructure.

+

In a blog post published today, the Google Threat Intelligence Group (GTIG) said NetNut’s proxy network is widely resold and white-labeled by a number of third-party proxy providers, and that its services are heavily sought out by cybercriminals seeking to obfuscate the source of their malicious traffic. The GTIG said that in a single week during June 2026, they observed 316 distinct clusters of threat actors using suspected NetNut exit nodes, including cybercriminal and espionage groups.

+

“These bad actors can use NetNut to mask their origin IP address when accessing victim environments, accessing their own infrastructure, and conducting password spray attacks,” Google’s GTIG wrote. “Furthermore, when a consumer device becomes an exit node, unauthorized network traffic passes through it. This means bad actors can access other private devices on the same home network, effectively exposing them to Internet threats.”

+

Google said it disabled Google accounts and services used by NetNut for malware command and control, and that it shared technical intelligence on NetNut’s software development kits (SDKs) and backend infrastructure with platform providers, law enforcement and research firms. The company also disabled apps known to bundle NetNut’s various SDKs.

+

Omer Weiss, legal counsel for NetNut parent Alarum Technologies, said the company was aware of the FBI seizure and cooperating with investigators.

+

“Alarum takes this matter seriously and will fully cooperate with law enforcement to ensure any misuse of its infrastructure is thoroughly investigated and those responsible are held to account,” Weiss said in a written statement.
+

+

Benjamin Brundage is founder of the proxy tracking service Synthient, one of the companies that published evidence last month linking the Popa botnet to NetNut and Alarum Technologies. Brundage said the domain seizures appear to have disrupted both the Popa botnet and the NetNut proxy network that rides on top of it.

+

Brundage said NetNut’s apparent demise is likely to be a great disadvantage for the cybercrime community, which was already reeling from legal actions by Google earlier this year that seized infrastructure for NetNut’s biggest competitor — IPIDEA.

+

“I think this takedown is going to have a big impact, because NetNut gained significant popularity after the IPIDEA takedown,” he said. “Also NetNut has been incredibly common among resellers, and they were on par with IPIDEA in terms of their daily traffic, quality, size, price per gigabyte, all of it.”

+

NetNut’s infrastructure, in a nutshell. Image: Black Lotus Labs, Lumen.

+

The NetNut and Popa botnet takedown may have another added benefit, Brundage said: Lessening the impact of large distributed denial-of-service botnets that have been built on the backs of poorly configured residential proxy services. In January, Synthient revealed how cybercriminals had built the world’s largest DDoS botnet (Kimwolf) by tunneling through IPIDEA proxy connections into the local networks of TV box owners, and infecting other Android-based devices behind the victim’s firewall.

+

While many of the bigger proxy providers took steps to block this activity, resellers of the major proxy networks have been far slower to respond to the threat, Brundage said.

+

“In terms of all these TV box devices getting compromised from the proxy network, it will have an impact on the DDoS botnets out there,” he said.

+

For its part, Google reckons today’s actions have caused “significant degradation to NetNut’s proxy network and its business operations, reducing the available pool of devices for the proxy operator by millions.” But the company warns that proxy networks can rebuild themselves by effectively reselling other proxy services, as IPIDEA has done over the past few months.

+

“Google has high confidence that many popular residential proxy brands are in fact whitelabeling the NetNut botnet,” the GTIG report concludes. “While we expect this disruption to have a larger ripple effect across the residential proxy ecosystem, observations after the disruption of IPIDEA proved that individual networks can appear resilient. What we have observed is that when faced with the degradation of their own botnet, proxy operators begin buying capacity from their competitors, effectively becoming a reseller. We recognize that creating a lasting disruption in this fluid ecosystem means we must scale our efforts to target the infrastructure of several interconnected providers.”

+

As KrebsOnSecurity has warned repeatedly, most of the no-name TV streaming boxes for sale on the major e-commerce websites either come pre-installed with residential proxy software, or require the installation of proxy SDKs in order to use the device for its stated purpose (streaming pirated movies, sporting events and TV shows). Google’s advice here is sound: When it comes to TV boxes, stick to name brands from reputable manufacturers, and then be sparing and judicious with any apps you choose to install.

+

The sketchy TV boxes that are being commandeered by the Popa botnet and other threats all come with or require the user to install unofficial Android operating systems that do not operate within the confines of Google’s Official Play Protect store. Google says consumers can confirm whether or not a device is built with the official Android TV OS and Play Protect certification by following these instructions.

+

Even people without TV streaming boxes can find their smart TVs enrolled in residential proxy networks, just by installing one of thousands of apps available for download on Samsung and LG smart TVs. In a report released last month, the proxy tracking company Spur found 42 percent of apps available for download via the webOS operating system on LG smart TVs include SDKs that turn one’s television into an always-on residential proxy node. More than a quarter of the apps made for Samsung’s Tizen operating system had similar residential proxy components, Spur found.

+

Image: Spur.us.

+

Update, 4:24 p.m. ET: Included a statement shared post-publication from an attorney representing NetNut parent Alarum Technologies.

+]]> + + https://krebsonsecurity.com/2026/07/fbi-seizes-netnut-proxy-platform-popa-botnet/feed/ + 17 + + + + + Scattered Spider Hackers Plead Guilty on Day 1 of Trial + https://krebsonsecurity.com/2026/06/scattered-spider-hackers-plead-guilty-on-day-1-of-trial/ + https://krebsonsecurity.com/2026/06/scattered-spider-hackers-plead-guilty-on-day-1-of-trial/#comments + + + Tue, 23 Jun 2026 16:12:49 +0000 + + + + + + + + + + https://krebsonsecurity.com/?p=73876 + + + Two men pleaded guilty in the United Kingdom this week to criminal charges stemming from an August 2024 cyberattack that crippled Transport for London, the entity responsible for the public transport network in the Greater London area. The duo were key members of a prolific cybercrime group known as Scattered Spider, and their guilty pleas came on the first day of what was expected to be a six-week trial.

+

Owen Flowers (left) 18, and Thalha Jubair, 20. Image: UK National Crime Agency (NCA).

+

Thalha Jubair, 20, of East London and 18-year-old Owen Flowers of Walsall admitted conspiring to commit unauthorized acts against Transport for London computer systems and causing risk of serious damage to human welfare. According to a report from the BBC, Flowers alone admitted to being part of a conspiracy to hack into U.S. based healthcare providers SSM Health Care Corporation and Sutter Health in September 2024.

+

Jubair is also wanted by U.S. law enforcement agencies. In September 2025, prosecutors in New Jersey unsealed an indictment alleging Jubair and other Scattered Spider members committed computer fraud, wire fraud, and money laundering in relation to 120 computer network intrusions involving 47 U.S. entities between May 2022 and September 2025, and that the group’s victims paid at least $115 million in ransom payments.

+

In July 2025, KrebsOnSecurity reported that Flowers and Jubair were arrested in the United Kingdom in connection with Scattered Spider ransom attacks against the retailers Marks & Spencer and Harrods, and the British food retailer Co-op Group. Multiple sources familiar with those investigations said Flowers was the Scattered Spider member who anonymously gave interviews to the media in the days after the group’s September 2023 ransomware attacks disrupted operations at Las Vegas casinos operated by MGM Resorts and Caesars Entertainment.

+

According to prosecutors, Jubair co-ran a bustling Telegram channel called Star Chat, the home of a SIM-swapping group that used voice- and SMS-based phishing attacks to steal credentials from employees at the major wireless providers in the U.S. and U.K. The group would then use that access to sell a service that could redirect a target’s phone number to a device the attackers controlled and intercept the victim’s calls and text messages (including one-time codes for multi-factor authentication).

+

A receipt from Star Fraud Chat’s SIM-swapping service targeting a T-Mobile customer after the group gained access to internal T-Mobile employee tools. “Rocket Ace” was one of Jubair’s hacker handles, according to U.S. prosecutors.

+

New Jersey prosecutors also allege Jubair also was involved in a mass SMS phishing campaign during the summer of 2022 that stole single sign-on credentials from employees at hundreds of companies. That weeks-long SMS phishing campaign led to intrusions and data thefts at more than 130 organizations, including LastPass, DoorDash, Mailchimp, Plex and Signal.

+

KrebsOnSecurity reported last year that one of Jubair’s alter egos at age 15 was “Everlynn,” a hacker who sold fraudulent “emergency data requests” that used compromised police and government email addresses to demand subscriber data (e.g. username, IP/email address) from major tech companies, claiming the requests concerned urgent matters of life and death and could not wait for a court order.

+

In April 2026, 24-year-old British national and Scattered Spider member Tyler “Tylerb” Buchanan pleaded guilty to wire fraud conspiracy and aggravated identity theft for participating in the group’s SMS phishing spree in the summer of 2022. The government said Buchanan, Jubair and others used the credentials harvested in that phishing campaign to steal at least $8 million in cryptocurrency from victims throughout the United States. Buchanan is currently scheduled to be sentenced on October 2.

+

In August 2025, 20-year-old Scattered Spider member from Florida named Noah Michael Urban was sentenced to 10 years in federal prison and ordered to pay $13 million in restitution, after pleading guilty to charges of wire fraud and conspiracy.

+

The U.S. Department of Justice says three alleged Scattered Spider defendants indicted along with Buchanan still face charges, including Ahmed Hossam Eldin Elbadawy, 24, a.k.a. “AD,” of College Station, Texas; Evans Onyeaka Osiebo, 21, of Dallas, Texas; and Joel Martin Evans, 26, a.k.a. “joeleoli,” of Jacksonville, North Carolina.

+

Flowers and Jubair are slated to be sentenced in a London court on July 15, 2026.

+]]> + + https://krebsonsecurity.com/2026/06/scattered-spider-hackers-plead-guilty-on-day-1-of-trial/feed/ + 39 + + + + + ‘Popa’ Botnet Linked to Publicly-Traded Israeli Firm + https://krebsonsecurity.com/2026/06/popa-botnet-linked-to-publicly-traded-israeli-firm/ + https://krebsonsecurity.com/2026/06/popa-botnet-linked-to-publicly-traded-israeli-firm/#comments + + + Thu, 18 Jun 2026 17:37:58 +0000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + https://krebsonsecurity.com/?p=73832 + + + For the past four years, a sprawling Android-based botnet called Popa has forced millions of consumer TV boxes to relay Internet traffic linked to advertising fraud, account takeovers, and mass data-scraping efforts. This week, researchers from multiple security firms concluded that the Popa botnet is linked to NetNut, a “residential proxy” provider operated by the publicly-traded Israeli firm Alarum Technologies Ltd [NASDAQ: ALAR].

+
Malicious streaming devices sold online that enroll the user's home Internet address in a residential proxy service. Image: Synthient. Pictured are 8 different TV boxes, including the X96 Mini Box, stick, and other no-name brands.

Malicious streaming devices sold online that enroll the user’s home Internet address in a residential proxy service. Image: HUMAN Security.

+

Popa is a massive botnet, but by all accounts it is unlike traditional botnets that enlist compromised systems in destructive activities, such as coordinating huge distributed denial-of-service attacks. Rather, Popa appears designed with a singular purpose: Implementing a persistent communications layer capable of registering a device, maintaining long-lived encrypted connections, and opening communication tunnels on demand.

+

Experts say Popa is a plugin component associated with the Vo1d botnet, a large-scale malware campaign targeting unofficial Android-based TV boxes. These devices, which are marketed under thousands of brand names and model numbers and broadly available for purchase at top e-commerce destinations, all advertise the ability to stream hundreds of subscription video services for an up front one-time fee.

+

But as the FBI and security industry experts have warned repeatedly, these streaming boxes typically bundle or come pre-installed with software that turns the user’s TV into a “residential proxy” — allowing anyone to route their Internet traffic through that device for as long as it remains plugged into a wall socket and connected to a local network. More concerning, some of these proxy networks do little to stop malicious customers from communicating with and even compromising systems on the local network of the unsuspecting device owner.

+

The first clues about Popa’s origins came in a 2025 report from the Chinese security company XLAB, which flagged at least nine domain names that were used to register and direct the activities of compromised devices. In a report released today, the security firm Qurium described how it stumbled on some of those same domains while investigating a series of disruptive and expensive data scraping events targeting the company’s hosted organizations in May 2026, in which the scraping activity was scattered evenly across more than 1.4 million Internet addresses.

+

Qurium said it found several dozen domains used to control Popa that were all hosted in lockstep across multiple Internet addresses over time, including gmslb[.]net, safernetwork[.]io, tera-home[.]com, and ninjatech[.]io. Digging deeper, Qurium discovered gmslb[.]net was referenced in dozens of pirated or modded video content streaming apps, such as CRICFy, DooFlix, Sprozfy, RTS Tv, Flixoid, CyberFlix, Rapid Streamz, TvMob and HD/OceanStreams.

+

Qurium’s report notes that most of the domains long used to control the Popa botnet were seized or dismantled in July 2025, after Google, HUMAN Security and Trend Micro teamed up to disrupt Badbox 2.0, a botnet that is closely associated with Vo1d. Qurium said that immediately after that disruption, several dozen new domains were registered to serve as controllers for the Popa botnet, but that one of those control domains was not new: ninjatech[.]io.

+

Ninjatech is a company founded by Moishi Kramer, whose LinkedIn profile says he is vice president of research and development at NetNut. That resume credits Kramer for helping NetNut to build from the “ground up,” “designing the architecture,” and “scaling the NetNut” before the company was acquired by Alarum Technologies. A self-created listing at the job board F6S references Kramer as the sole owner of the Ninjatech domain (a screen capture of it is pictured below).

+

Image: F6S.com.

+

Responding via email, Mr. Kramer said Ninjatech ceased operations approximately five years ago, when the company sold a software development kit (SDK) called Popa that was designed to use a small portion of a device’s bandwidth and to run only after the host application obtained user consent.

+

“That code was sold and licensed to third parties including resellers years ago,” Kramer said. “Once software is distributed that way, the original developer has no control over how others later modify, rebrand, or deploy it.”

+

Kramer said neither he nor NetNut builds, operates or maintains the infrastructure being described as Popa, nor does he control the Ninjatech domain.

+

“I didn’t register the June 2025 domains you mention, and I don’t know who did,” he continued. “I have no control over, or visibility into, that infrastructure. I can only tell you it isn’t operated by me or by NetNut.”

+

But in a separate Popa research report released today, the proxy-tracking company Synthient said a recent analysis of the Popa SDK revealed outbound traffic clearly associated with NetNut.

+

“The research team assesses with high confidence that devices running Popa forward traffic from Netnut clients,” Synthient wrote. “This proves without a shadow of a doubt that Popa actively continues to be used by NetNut as part of their proxy pool.”

+

Synthient’s platform receiving outbound traffic from Popa. Image: Synthient.com.

+

Alarum Technologies, NetNut’s Tel Aviv-based parent company, said the reports by Synthient and Qurium contained “demonstrably inaccurate assertions and flawed deductions rather than verified facts.” Alarum shared a statement saying they reject the basic characterization of the SDKs and technologies discussed in the reports as a “botnet.”

+

“The SDKs at issue are designed to facilitate bandwidth-sharing functionality and do not transform user devices into malware-controlled systems or otherwise compromise the devices on which they operate,” the statement reads. “Netnut operates a commercial proxy network and maintains policies, procedures, and technological measures designed to promote lawful and responsible use of its services.”

+

Alarum said NetNut places “significant emphasis on appropriate notice and consent mechanisms, conducts customer due diligence, monitors for potential misuse, and takes steps intended to detect and mitigate suspicious or unauthorized activity.”

+

“This method of operation is supported both by internal procedures and policies, including performing KYC checks and additional due diligence of NetNut’s customers, as well as employing various technological measures, designed to assist in identifying and addressing suspected misuse of the network,” their statement continued.

+

However, in a report released on June 8, the proxy tracking service Spur asserted that NetNut does not require corporate verification or meaningful “know your customer” procedures before allowing customers to purchase proxy access.

+

“An individual can sign up, pay, and route traffic through partner address space, including space belonging to institutions whose users never opted in,” Spur wrote. “The ‘verified corporations only’ claim is simply marketing for bandwidth sellers, not an access control on who actually uses the proxies.”

+

“Nor is NetNut the only front door,” Spur continued. “A number of downstream white labelers and resellers repackage the same ISP proxy pool under their own brands. These outlets typically perform no KYC at all, less scrutiny than NetNut itself, who at the very least might assign an account manager to potential users. Anyone who knows where to look can buy access through a reseller with nothing more than a burner email address and $5 in crypto.”

+

Synthient found that although the most recent builds of Popa (as of three months ago) have added the ability to ask the user for consent before installing proxy components, not all variants or previous versions of Popa contain this functionality.

+

“Of the over 20 genuine Popa publishers analyzed, none of them were observed asking for user consent,” Sythient wrote.

+

THE PREVALENCE OF POPA

+

Chris Formosa is senior lead information security engineer for Black Lotus Labs, a division of the Internet backbone carrier Lumen Technologies.

+

“What especially makes Popa dangerous is just how widely used NetNut is for reselling and sharing,” Formosa said, explaining that many other proxy services simply resell NetNut proxies rather than building out their own far-flung proxy networks. “So these Popa IPs appear in tons of different services all over the ecosystem, which makes it one of the most problematic and dangerous proxy botnets on the market currently.”

+

Formosa said the Popa botnet averages between 1.5 million to 2.5 million distinct IP addresses each day, relying on between 250 and 300 Internet addresses that are used to direct its activities.

+

“That’s why Popa is so dangerous,” Formosa said. “It may not be the largest botnet we have seen, but it is spread all over the industry, making its power very amplified.”

+

Formosa said while that makes Popa one of the larger botnets out there today, its numbers pale in comparison to those previously boasted by IPIDEA, a China-based proxy provider that until recently operated a daily pool of nearly 10 million devices that they resold as proxies to anyone. In January 2026, Synthient published research showing that multiple new large DDoS botnets had grown rapidly by tunneling through IPIDEA proxies into the local networks of unsuspecting TV box owners and infecting other Android-based devices behind the user’s firewall.

+

IPIDEA is based largely on SDKs used to view pirated streaming content on a vast number of TV box devices, but the service’s numbers have dwindled since January, when Google and industry partners took legal action to seize domain names that IPIDEA used to control devices and proxy traffic through them.

+

Jérôme Meyer, a security researcher at Nokia Deepfield, said the total population of devices participating in the Popa botnet may be far higher than Lumen’s estimates. Meyer told KrebsOnSecurity that Nokia is monitoring 26 of at least 359 known relay nodes for the botnet, and estimates that each relay node handles between 35,000 and 60,000 clients simultaneously.

+

“On the relay node subset I am looking at (26 of them), 750,000 unique sources in 24 hours,” Meyer wrote in response to questions.

+

Nokia Deepfield released its own report today on RoboVPN, a VPN app tied to the Vo1d botnet’s Popa plugin that Qurium attributes to NetNut/Alarum Technologies.

+

THE SYMBIOSIS OF PROXIES AND DATA SCRAPING

+

Experts say many of the world’s largest proxy providers have updated their public-facing branding to highlight their utility for training AI platforms, implying it is a primary use case for their residential proxies. That’s because AI services tend to rely on constantly mass-scraping the Internet for new text, images and video content that can be used to train large language models (LLMs).

+

NetNut and other proxy services have recast themselves as critical infrastructure for the AI scraping economy. Image: Synthient.com.

+

“AI companies depend on web-scraped content: for pre-training, for retrieval, for agent grounding, for search,” reads a report this month from Include Security that examines the prevalence of proxy SDKs in smart TV apps. “But the modern web isn’t scrapeable from a datacenter. Cloudflare, DataDome, HUMAN, among others throttle or block requests from known cloud IPs. The workaround is residential proxies. A scraping job routed through a Comcast or T-Mobile subscriber’s connection arrives at the target site from an IP that belongs to a paying residential customer.”

+

This non-stop content scraping has spawned more than 70 copyright infringement lawsuits against major tech companies that have acknowledged large-scale data scraping as a major source of the “brains” behind their commercial AI offerings. Ironically, much of that scraping is being aided by proxy services that are intimately tied to unofficial Android TV boxes and associated SDKs whose stated purpose is streaming pirated content.

+

The scraping activity has become so aggressive that it often overwhelms the targeted websites, preventing them from being reachable by legitimate visitors. In many reported cases, nonprofit organizations, libraries and universities have complained of constantly battling to keep their services online in the face of relentless data-scraping firms hiding behind residential proxy services.

+

A survey conducted last year by the Confederation of Open Access Repositories (COAR) found while some content scraping bots are rather innocuous, “others are sufficiently aggressive that they are increasingly causing service disruptions in repositories and other scholarly communications infrastructures.” More than 90 percent of survey respondents indicated their repository is encountering aggressive bots, usually more than once a week, and often leading to slow downs and service outages.

+

“Automated web scraping is nothing new, and has been the key technology underlying search engines such as Google for over 30 years,” wrote Brendan O’Connell, platform manager at the Directory of Open Access Journals (DOAJ), a free, community-curated index of peer-reviewed academic journals. “However, the current investor-fueled AI startup craze means there are now thousands of well-funded companies developing and deploying their own scraping tools to train AI models, alongside existing major players like OpenAI and Google.”

+

DON’T TOUCH THAT DIAL!

+

Across the United States, local communities are pushing back against the proliferation of new data centers aimed primarily at improving the capabilities of AI. But security experts say the general public remains largely unaware that using one of these unsanctioned Android TV boxes means their “smart TV” is almost certainly using a significant amount of bandwidth each month to help train modern AI models.

+

Even households without these sketchy TV boxes can still have their smart TVs turned into residential proxy nodes, just by downloading one of thousands of apps made available on Samsung and LG smart TVs. Spur said it recently scraped the LG and Samsung app stores and found that each had approximately 3,000 apps available for download. Many of these apps are simple games or utilities that state in the fine print that the user’s Internet connection will be used to download data and that they can opt out at any time.

+

Spur said it found that more than 42 percent of apps available for download via the webOS operating system on LG smart TVs include SDKs that turn one’s television into an always-on residential proxy node. More than a quarter of the apps made for Samsung’s Tizen operating system had similar residential proxy components, Spur found.

+

Image: Spur.us.

+

Experts say it’s questionable whether TV apps with proxy SDKs can obtain meaningful consent from users for installing an always-on proxy connection, particularly when anyone in a household — including children — can effectively opt the family TV into a residential proxy network just by installing a simple game or app.

+

“Privacy-policy disclosure is the wrong control surface for a TV,” Include Security wrote. “It is hard to scroll through a legal document navigated by arrow keys on a remote, and the in-app consent dialog doesn’t convey that a paying customer is about to route their scraping traffic through the user’s home internet.”

+

Spur’s head of research Sean Simmons told KrebsOnSecurity that most people do not have a working mental model for what it means to sell access to their residential IP address, no matter what device they are using.

+

“And on a TV, the gap is even wider,” Simmons said. “A one-time prompt navigated with a remote can disappear into the setup flow, while the app keeps monetizing the connection long after anyone remembers what they accepted.”

+

Simmons said LG and Samsung should follow the lead of other TV platforms that have already drawn a line against residential proxy providers, pointing to policies by Amazon that prohibit apps facilitating proxy services for third parties. Likewise the TV streaming device maker Roku reportedly now bars developers from using proxy SDKs and has removed apps that bundled them.

+

Piracy related apps pushing proxy SDKs onto unconsenting users. Image: Synthient.

+

Apps that turn one’s device into a residential proxy node are not limited to smart TVs and no-name streaming boxes, of course. As noted by the security firm Infoblox, mobile app developers can embed SDKs provided by the residential proxy networks into their products to monetize their software, allowing them to receive a small amount of money on each installation.

+

The result, Infoblox said, is that devices are frequently enrolled without the owner’s knowledge, typically through free applications such as VPNs, streaming apps, screensavers and “productivity” apps such as PDF viewers and break reminders.

+

All too often, these proxy services are beaconing out from employee devices brought into the workplace, Infoblox found. In a blog post earlier this month, Infoblox said it discovered that fully 65% of its customer base was querying one or more residential proxy related domains.

+

“We saw steady growth in these queries in 2025, with a 25% increase over the year to over 500 billion per month,” Infoblox wrote. “Over 90% of our pharmaceutical and food & beverage customers have queried residential proxy indicators. Perhaps even more concerning is that over 60% of government and banking customers have as well.”

+

Infoblox researchers Nick Sundvall and David Brunsdon warned that with residential proxies in the corporate environment, external access is granted to an organization’s IP space.

+

“If threat actors were to abuse the residential proxy to attack a third party, the third party’s incident response would, correctly, identify your residential proxy as the source,” they wrote. “Untangling that, by proving that you were the conduit and not the threat actor, costs time, creates legal exposure, and can damage your reputation. The stunning prevalence of these services within customer environments warrants attention from both network defenders and policy makers who should consider how the risks posed by residential proxies could be impacting their security posture.”

+]]> + + https://krebsonsecurity.com/2026/06/popa-botnet-linked-to-publicly-traded-israeli-firm/feed/ + 16 + + + + + + + diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/securityweek.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/securityweek.xml new file mode 100644 index 00000000..d6580353 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/securityweek.xml @@ -0,0 +1,90 @@ + + + + SecurityWeek + + https://www.securityweek.com/ + Cybersecurity News, Insights & Analysis + Fri, 03 Jul 2026 15:10:44 +0000 + en-US + + hourly + + 1 + https://wordpress.org/?v=7.0 + + + https://www.securityweek.com/wp-content/uploads/2023/01/cropped-SecurityWeek-Icon-32x32.jpeg + SecurityWeek + https://www.securityweek.com/ + 32 + 32 + + + In Other News: Canadian Hacker Jailed, Open Source Zero-Days, Two Sentenced for ATM Jackpotting + https://www.securityweek.com/in-other-news-canadian-hacker-jailed-open-source-zero-days-two-sentenced-for-atm-jackpotting/ + + + Fri, 03 Jul 2026 15:10:42 +0000 + + + + + https://www.securityweek.com/?p=47625 + + Noteworthy stories that might have slipped under the radar: Anonymous-linked Canadian hacker jailed, researcher drops zero-days in open source projects, Venezuelans sentenced in the US over ATM jackpotting.

+

The post In Other News: Canadian Hacker Jailed, Open Source Zero-Days, Two Sentenced for ATM Jackpotting appeared first on SecurityWeek.

+]]> + + + + + + Agentic AI Used to Conduct Ransomware Attack via Langflow + https://www.securityweek.com/agentic-ai-used-to-conduct-ransomware-attack-via-langflow/ + + + Fri, 03 Jul 2026 11:00:00 +0000 + + + + + + https://www.securityweek.com/?p=47618 + + Attack demonstrates how LLM agents can combine known exploitation techniques with real-time reasoning to automate complex, multi-stage intrusions.

+

The post Agentic AI Used to Conduct Ransomware Attack via Langflow appeared first on SecurityWeek.

+]]> + + + + + + Medtronic Data Breach Impacts 3.8 Million People + https://www.securityweek.com/medtronic-data-breach-impacts-3-8-million-people/ + + + Fri, 03 Jul 2026 10:00:00 +0000 + + + + + + https://www.securityweek.com/?p=47616 + + In April, ShinyHunters accessed the company’s corporate IT systems and stole patients’ personal and medical information.

+

The post Medtronic Data Breach Impacts 3.8 Million People appeared first on SecurityWeek.

+]]> + + + + + + diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/thehackernews.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/thehackernews.xml new file mode 100644 index 00000000..771c4871 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/thehackernews.xml @@ -0,0 +1,9 @@ +The Hacker Newshttps://thehackernews.comMost trusted, widely-read independent cybersecurity news source for everyone; supported by hackers and IT professionals — Send TIPs to admin@thehackernews.comen-usSun, 05 Jul 2026 21:16:08 +0530hourly1U.S. Government Entity Paid Kairos $1 Million in Data-Theft Extortion Casehttps://thehackernews.com/2026/07/us-government-entity-paid-kairos-group.htmlhttps://thehackernews.com/2026/07/us-government-entity-paid-kairos-group.htmlSat, 04 Jul 2026 18:17:53 +0530info@thehackernews.com (The Hacker News) +North Korean Hackers Publish 108 Malicious Packages and Extensions in PolinRider Campaignhttps://thehackernews.com/2026/07/north-korean-hackers-publish-108.htmlhttps://thehackernews.com/2026/07/north-korean-hackers-publish-108.htmlSat, 04 Jul 2026 16:47:24 +0530info@thehackernews.com (The Hacker News) +Unpatched Flaws Disclosed in Filesystem Bundled Into Millions of Embedded Deviceshttps://thehackernews.com/2026/07/unpatched-flaws-disclosed-in-filesystem.htmlhttps://thehackernews.com/2026/07/unpatched-flaws-disclosed-in-filesystem.htmlSat, 04 Jul 2026 01:49:31 +0530info@thehackernews.com (The Hacker News) diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/theregister.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/theregister.xml new file mode 100644 index 00000000..e76ea427 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/theregister.xml @@ -0,0 +1,48 @@ + + + + www.theregister.com - Articles + https://www.theregister.com + Articles from www.theregister.com + + + https://www.theregister.com/a/5266161 + https://www.theregister.com/security/2026/07/05/mfa-optional-banks-leave-safe-doors-and-accounts-wide-open-for-thieves-to-pillage/5266161 + Sun, 05 Jul 2026 17:01:00 +0200 + MFA-optional banks leave safe doors (and accounts) wide open for thieves to pillage + + security + + Fri, 03 Jul 2026 00:01:12 +0000 + + + + + + https://www.theregister.com/a/5266056 + https://www.theregister.com/security/2026/07/04/confidential-computings-core-trust-mechanism-is-broken-the-fix-may-not-exist/5266056 + Sat, 04 Jul 2026 12:03:00 +0200 + Confidential computing's core trust mechanism is broken. The fix may not exist + + security + + Fri, 03 Jul 2026 07:47:18 +0000 + + + + + + https://www.theregister.com/a/5266512 + https://www.theregister.com/security/2026/07/03/adapthealth-crooks-stole-our-passwords-patient-health-data/5266512 + Fri, 03 Jul 2026 16:29:00 +0200 + AdaptHealth says attackers sweet-talked their way into cloud systems and stole patient data + + security + + Fri, 03 Jul 2026 13:50:25 +0000 + + + + + + From 312b13b3488fd303d4363c8335e7a7524505ead3 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Sun, 5 Jul 2026 15:11:29 -0400 Subject: [PATCH 03/15] feat(nadezhda): M2 cross-source dedup + clustering Group the same story across outlets into clusters, the velocity signal for ranking. Runs automatically after 'scrape' (recompute-from-scratch, idempotent). - internal/cluster: connected-components engine (union-find). Two articles join when, within a time window (Cluster.WindowHours, default 72h), EITHER their normalized-title token-set Jaccard >= threshold (default 0.6) AND they are from different outlets, OR they share >= 1 CVE (source-agnostic). The cross-outlet guard on the title edge kills a real false positive where two distinct CISA advisories with identical boilerplate titles were merging. Cluster key = the earliest article by time; SourceCount tracks distinct outlets so the reported 'multi-source' metric is truthful. - Shared-CVE join reads article_cves, which is empty until M3 CVE extraction, so that path is a natural no-op today and lights up in M3 with no rework. - store: ClusterCandidates (effective time = published_at else fetched_at, bounded by lookback), ArticleCVEMap, ReplaceClusters (single atomic tx: wipe + rebuild). - config: Cluster.LookbackHours (default 168h) bounds the corpus for O(n^2); validated as >= window_hours so window edges are never silently dropped. - Proven live: 215 real articles -> 137 clusters, 2 genuinely cross-outlet (FortiBleed x3, Scattered Spider x2). 17 cluster tests; suite offline + -race. One read-only audit agent run; findings (multi-source stat, lookback guard) fixed in-phase. Clustering is recompute-from-scratch each scrape; safe to re-run. --- .../cmd/nadezhda/scrape.go | 16 +- .../internal/cluster/cluster.go | 208 ++++++++++++++++++ .../internal/cluster/cluster_test.go | 148 +++++++++++++ .../internal/cluster/rebuild.go | 51 +++++ .../internal/cluster/rebuild_test.go | 133 +++++++++++ .../internal/config/config.go | 16 +- .../internal/config/config_test.go | 12 + .../internal/store/store.go | 94 ++++++++ 8 files changed, 674 insertions(+), 4 deletions(-) create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/cluster/cluster.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/cluster/cluster_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/cluster/rebuild.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/cluster/rebuild_test.go diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go index 6d3c5587..0b49e69d 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go @@ -12,12 +12,15 @@ import ( "github.com/spf13/cobra" + "github.com/CarterPerez-dev/nadezhda/internal/cluster" "github.com/CarterPerez-dev/nadezhda/internal/fetch" "github.com/CarterPerez-dev/nadezhda/internal/ingest" "github.com/CarterPerez-dev/nadezhda/internal/source" "github.com/CarterPerez-dev/nadezhda/internal/store" ) +const secondsPerHour = 3600 + const ( statusNotModified = "304" statusError = "error" @@ -70,12 +73,21 @@ func runScrape(cmd *cobra.Command, args []string) error { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() - summary, err := ingest.Run(ctx, fc, st, cfg, targets, time.Now()) + now := time.Now() + summary, err := ingest.Run(ctx, fc, st, cfg, targets, now) if err != nil { return err } - printSummary(cmd, summary) + + sinceUnix := now.Unix() - int64(cfg.Cluster.LookbackHours)*secondsPerHour + windowSeconds := int64(cfg.Cluster.WindowHours) * secondsPerHour + stats, err := cluster.Rebuild(st, cfg.Cluster.TitleJaccard, windowSeconds, sinceUnix) + if err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "%d clusters (%d multi-source, largest %d)\n", + stats.Total, stats.MultiSource, stats.LargestSize) return nil } diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cluster/cluster.go b/PROJECTS/intermediate/security-news-scraper/internal/cluster/cluster.go new file mode 100644 index 00000000..2176e2df --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/cluster/cluster.go @@ -0,0 +1,208 @@ +// ©AngelaMos | 2026 +// cluster.go + +package cluster + +import ( + "sort" + "strconv" + "strings" + + "github.com/CarterPerez-dev/nadezhda/internal/normalize" +) + +type Item struct { + ID int64 + SourceID int64 + Title string + CVEs []string + Time int64 +} + +type Cluster struct { + Key string + Members []int64 + SourceCount int + FirstSeen int64 + LastSeen int64 +} + +type tokenized struct { + item Item + tokens map[string]struct{} + cves map[string]struct{} +} + +func Compute(items []Item, jaccardThreshold float64, windowSeconds int64) []Cluster { + n := len(items) + prepared := make([]tokenized, n) + for i, it := range items { + prepared[i] = tokenized{ + item: it, + tokens: tokenSet(it.Title), + cves: stringSet(it.CVEs), + } + } + + uf := newUnionFind(n) + for i := 0; i < n; i++ { + for j := i + 1; j < n; j++ { + if abs64(prepared[i].item.Time-prepared[j].item.Time) > windowSeconds { + continue + } + sharedCVE := shareAny(prepared[i].cves, prepared[j].cves) + crossSource := prepared[i].item.SourceID != prepared[j].item.SourceID + titleMatch := crossSource && + jaccard(prepared[i].tokens, prepared[j].tokens) >= jaccardThreshold + if sharedCVE || titleMatch { + uf.union(i, j) + } + } + } + + groups := make(map[int][]int) + for i := 0; i < n; i++ { + root := uf.find(i) + groups[root] = append(groups[root], i) + } + + clusters := make([]Cluster, 0, len(groups)) + for _, members := range groups { + clusters = append(clusters, buildCluster(prepared, members)) + } + sort.Slice(clusters, func(a, b int) bool { + if clusters[a].FirstSeen != clusters[b].FirstSeen { + return clusters[a].FirstSeen < clusters[b].FirstSeen + } + return clusters[a].Members[0] < clusters[b].Members[0] + }) + return clusters +} + +func buildCluster(prepared []tokenized, members []int) Cluster { + earliest := prepared[members[0]].item + ids := make([]int64, 0, len(members)) + sources := make(map[int64]struct{}, len(members)) + first := earliest.Time + last := earliest.Time + for _, m := range members { + it := prepared[m].item + ids = append(ids, it.ID) + sources[it.SourceID] = struct{}{} + if it.Time < first { + first = it.Time + } + if it.Time > last { + last = it.Time + } + if it.Time < earliest.Time || (it.Time == earliest.Time && it.ID < earliest.ID) { + earliest = it + } + } + sort.Slice(ids, func(a, b int) bool { return ids[a] < ids[b] }) + return Cluster{ + Key: strconv.FormatInt(earliest.ID, 10), + Members: ids, + SourceCount: len(sources), + FirstSeen: first, + LastSeen: last, + } +} + +func tokenSet(title string) map[string]struct{} { + fields := strings.Fields(normalize.NormalizeTitle(title)) + set := make(map[string]struct{}, len(fields)) + for _, f := range fields { + set[f] = struct{}{} + } + return set +} + +func stringSet(values []string) map[string]struct{} { + set := make(map[string]struct{}, len(values)) + for _, v := range values { + if v != "" { + set[v] = struct{}{} + } + } + return set +} + +func jaccard(a, b map[string]struct{}) float64 { + if len(a) == 0 || len(b) == 0 { + return 0 + } + small, large := a, b + if len(small) > len(large) { + small, large = large, small + } + intersection := 0 + for k := range small { + if _, ok := large[k]; ok { + intersection++ + } + } + union := len(a) + len(b) - intersection + if union == 0 { + return 0 + } + return float64(intersection) / float64(union) +} + +func shareAny(a, b map[string]struct{}) bool { + if len(a) == 0 || len(b) == 0 { + return false + } + small, large := a, b + if len(small) > len(large) { + small, large = large, small + } + for k := range small { + if _, ok := large[k]; ok { + return true + } + } + return false +} + +func abs64(v int64) int64 { + if v < 0 { + return -v + } + return v +} + +type unionFind struct { + parent []int + rank []int +} + +func newUnionFind(n int) *unionFind { + uf := &unionFind{parent: make([]int, n), rank: make([]int, n)} + for i := range uf.parent { + uf.parent[i] = i + } + return uf +} + +func (uf *unionFind) find(x int) int { + for uf.parent[x] != x { + uf.parent[x] = uf.parent[uf.parent[x]] + x = uf.parent[x] + } + return x +} + +func (uf *unionFind) union(a, b int) { + ra, rb := uf.find(a), uf.find(b) + if ra == rb { + return + } + if uf.rank[ra] < uf.rank[rb] { + ra, rb = rb, ra + } + uf.parent[rb] = ra + if uf.rank[ra] == uf.rank[rb] { + uf.rank[ra]++ + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cluster/cluster_test.go b/PROJECTS/intermediate/security-news-scraper/internal/cluster/cluster_test.go new file mode 100644 index 00000000..4c93afb9 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/cluster/cluster_test.go @@ -0,0 +1,148 @@ +// ©AngelaMos | 2026 +// cluster_test.go + +package cluster + +import ( + "sort" + "testing" +) + +const bigWindow = 1 << 40 + +func sizes(clusters []Cluster) []int { + out := make([]int, len(clusters)) + for i, c := range clusters { + out[i] = len(c.Members) + } + sort.Ints(out) + return out +} + +func findCluster(clusters []Cluster, id int64) Cluster { + for _, c := range clusters { + for _, m := range c.Members { + if m == id { + return c + } + } + } + return Cluster{} +} + +func TestComputeTitleSimilarityGroups(t *testing.T) { + items := []Item{ + {ID: 1, SourceID: 1, Title: "Acme Corp data breach exposes millions", Time: 100}, + {ID: 2, SourceID: 2, Title: "Acme Corp data breach exposed millions", Time: 200}, + {ID: 3, SourceID: 3, Title: "Acme Corp data breach hits millions", Time: 300}, + {ID: 4, SourceID: 4, Title: "Linux kernel 7 released with new scheduler", Time: 400}, + } + clusters := Compute(items, 0.6, bigWindow) + + if got := sizes(clusters); len(got) != 2 || got[0] != 1 || got[1] != 3 { + t.Fatalf("cluster sizes = %v, want [1 3]", got) + } + c := findCluster(clusters, 1) + if len(c.Members) != 3 { + t.Errorf("article 1 cluster size = %d, want 3", len(c.Members)) + } + if c.SourceCount != 3 { + t.Errorf("SourceCount = %d, want 3 (three distinct outlets)", c.SourceCount) + } + if c.Key != "1" { + t.Errorf("cluster key = %q, want 1 (earliest article)", c.Key) + } + if c.FirstSeen != 100 || c.LastSeen != 300 { + t.Errorf("span = [%d,%d], want [100,300]", c.FirstSeen, c.LastSeen) + } +} + +func TestComputeSameSourceTitlesDoNotMerge(t *testing.T) { + items := []Item{ + {ID: 1, SourceID: 1, Title: "CISA Adds One Known Exploited Vulnerability to Catalog", Time: 100}, + {ID: 2, SourceID: 1, Title: "CISA Adds One Known Exploited Vulnerability to Catalog", Time: 200}, + } + if got := sizes(Compute(items, 0.6, bigWindow)); len(got) != 2 { + t.Errorf("same-source identical titles sizes = %v, want [1 1] (clustering is cross-outlet)", got) + } +} + +func TestComputeSharedCVEJoinsRegardlessOfSource(t *testing.T) { + items := []Item{ + {ID: 1, SourceID: 1, Title: "Router vendor patches critical flaw", Time: 100, CVEs: []string{"CVE-2026-1111"}}, + {ID: 2, SourceID: 1, Title: "Enterprise gear gets emergency update", Time: 200, CVEs: []string{"CVE-2026-1111"}}, + {ID: 3, SourceID: 2, Title: "Totally unrelated privacy story", Time: 300, CVEs: []string{"CVE-2026-9999"}}, + } + clusters := Compute(items, 0.6, bigWindow) + if got := sizes(clusters); len(got) != 2 || got[0] != 1 || got[1] != 2 { + t.Fatalf("cluster sizes = %v, want [1 2]", got) + } + joined := findCluster(clusters, 1) + if len(joined.Members) != 2 { + t.Error("shared-CVE articles should join even from the same source") + } + if joined.SourceCount != 1 { + t.Errorf("SourceCount = %d, want 1 (a same-source cluster is NOT multi-source)", joined.SourceCount) + } +} + +func TestComputeThresholdBoundary(t *testing.T) { + items := []Item{ + {ID: 1, SourceID: 1, Title: "alpha bravo charlie delta", Time: 100}, + {ID: 2, SourceID: 2, Title: "alpha bravo charlie echo", Time: 200}, + } + if got := sizes(Compute(items, 0.6, bigWindow)); len(got) != 1 || got[0] != 2 { + t.Errorf("at threshold 0.6 (jaccard==0.6) sizes = %v, want [2] (>= joins)", got) + } + if got := sizes(Compute(items, 0.61, bigWindow)); len(got) != 2 { + t.Errorf("at threshold 0.61 sizes = %v, want [1 1] (below threshold)", got) + } +} + +func TestComputeWindowSeparates(t *testing.T) { + items := []Item{ + {ID: 1, SourceID: 1, Title: "identical breaking headline text", Time: 0}, + {ID: 2, SourceID: 2, Title: "identical breaking headline text", Time: 200}, + } + if got := sizes(Compute(items, 0.6, 100)); len(got) != 2 { + t.Errorf("outside window sizes = %v, want [1 1] (time gap > window)", got) + } + if got := sizes(Compute(items, 0.6, 300)); len(got) != 1 || got[0] != 2 { + t.Errorf("inside window sizes = %v, want [2]", got) + } +} + +func TestComputeKeyIsEarliestByTime(t *testing.T) { + items := []Item{ + {ID: 10, SourceID: 1, Title: "shared story about the incident", Time: 500}, + {ID: 20, SourceID: 2, Title: "shared story about the incident", Time: 100}, + } + clusters := Compute(items, 0.6, bigWindow) + if len(clusters) != 1 { + t.Fatalf("clusters = %d, want 1", len(clusters)) + } + if clusters[0].Key != "20" { + t.Errorf("key = %q, want 20 (earliest by time, not lowest id)", clusters[0].Key) + } + if clusters[0].FirstSeen != 100 || clusters[0].LastSeen != 500 { + t.Errorf("span = [%d,%d], want [100,500]", clusters[0].FirstSeen, clusters[0].LastSeen) + } +} + +func TestComputeTransitiveChain(t *testing.T) { + items := []Item{ + {ID: 1, SourceID: 1, Title: "one two three four five", Time: 100}, + {ID: 2, SourceID: 2, Title: "one two three four six", Time: 100}, + {ID: 3, SourceID: 3, Title: "one two three six five", Time: 100}, + } + clusters := Compute(items, 0.6, bigWindow) + if len(clusters) != 1 || len(clusters[0].Members) != 3 { + t.Errorf("transitive similarity should merge all three, got %d clusters", len(clusters)) + } +} + +func TestComputeEmpty(t *testing.T) { + if c := Compute(nil, 0.6, bigWindow); len(c) != 0 { + t.Errorf("empty input -> %d clusters, want 0", len(c)) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cluster/rebuild.go b/PROJECTS/intermediate/security-news-scraper/internal/cluster/rebuild.go new file mode 100644 index 00000000..b1103f43 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/cluster/rebuild.go @@ -0,0 +1,51 @@ +// ©AngelaMos | 2026 +// rebuild.go + +package cluster + +import ( + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +type Stats struct { + Total int + MultiSource int + LargestSize int +} + +func Rebuild(st *store.Store, jaccardThreshold float64, windowSeconds, sinceUnix int64) (Stats, error) { + candidates, err := st.ClusterCandidates(sinceUnix) + if err != nil { + return Stats{}, err + } + cveMap, err := st.ArticleCVEMap() + if err != nil { + return Stats{}, err + } + + items := make([]Item, len(candidates)) + for i, c := range candidates { + items[i] = Item{ID: c.ID, SourceID: c.SourceID, Title: c.Title, Time: c.Time, CVEs: cveMap[c.ID]} + } + + clusters := Compute(items, jaccardThreshold, windowSeconds) + + rows := make([]store.ClusterRow, len(clusters)) + stats := Stats{Total: len(clusters)} + for i, c := range clusters { + rows[i] = store.ClusterRow{ + Key: c.Key, Members: c.Members, FirstSeen: c.FirstSeen, LastSeen: c.LastSeen, + } + if c.SourceCount > 1 { + stats.MultiSource++ + } + if size := len(c.Members); size > stats.LargestSize { + stats.LargestSize = size + } + } + + if err := st.ReplaceClusters(rows); err != nil { + return Stats{}, err + } + return stats, nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cluster/rebuild_test.go b/PROJECTS/intermediate/security-news-scraper/internal/cluster/rebuild_test.go new file mode 100644 index 00000000..6204d029 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/cluster/rebuild_test.go @@ -0,0 +1,133 @@ +// ©AngelaMos | 2026 +// rebuild_test.go + +package cluster + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +func openStore(t *testing.T) *store.Store { + t.Helper() + st, err := store.Open(filepath.Join(t.TempDir(), "cluster.db")) + if err != nil { + t.Fatalf("open store: %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + return st +} + +func seedArticles(t *testing.T, st *store.Store) { + t.Helper() + titles := []string{ + "Acme Corp data breach exposes millions", + "Acme Corp data breach exposed millions", + "Acme Corp data breach hits millions", + "Linux kernel 7 released with new scheduler", + } + for i, title := range titles { + sourceID, err := st.UpsertSource(store.SourceInput{ + Name: fmt.Sprintf("src%d", i), URL: fmt.Sprintf("https://outlet%d.example/feed", i), + Type: "rss", Weight: 1, Enabled: true, + }) + if err != nil { + t.Fatalf("upsert source %d: %v", i, err) + } + if _, err := st.InsertArticle(store.Article{ + SourceID: sourceID, + CanonicalURL: fmt.Sprintf("https://example.com/a/%d", i), + ContentHash: fmt.Sprintf("chash-%d", i), + TitleHash: fmt.Sprintf("thash-%d", i), + Title: title, + PublishedAt: int64(100 * (i + 1)), + FetchedAt: int64(100 * (i + 1)), + }); err != nil { + t.Fatalf("insert article %d: %v", i, err) + } + } +} + +func clusterSizes(t *testing.T, st *store.Store) []int { + t.Helper() + rows, err := st.DB().Query(`SELECT size FROM clusters ORDER BY size`) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out []int + for rows.Next() { + var n int + if err := rows.Scan(&n); err != nil { + t.Fatal(err) + } + out = append(out, n) + } + return out +} + +func countMembers(t *testing.T, st *store.Store) int { + t.Helper() + var n int + if err := st.DB().QueryRow(`SELECT COUNT(*) FROM cluster_members`).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +func TestRebuildClustersEndToEnd(t *testing.T) { + st := openStore(t) + seedArticles(t, st) + + stats, err := Rebuild(st, 0.6, bigWindow, 0) + if err != nil { + t.Fatalf("Rebuild: %v", err) + } + if stats.Total != 2 || stats.MultiSource != 1 || stats.LargestSize != 3 { + t.Fatalf("stats = %+v, want {Total:2 MultiSource:1 LargestSize:3}", stats) + } + if got := clusterSizes(t, st); len(got) != 2 || got[0] != 1 || got[1] != 3 { + t.Errorf("cluster sizes = %v, want [1 3]", got) + } + if m := countMembers(t, st); m != 4 { + t.Errorf("cluster_members = %d, want 4 (every article assigned once)", m) + } +} + +func TestRebuildIsIdempotent(t *testing.T) { + st := openStore(t) + seedArticles(t, st) + + if _, err := Rebuild(st, 0.6, bigWindow, 0); err != nil { + t.Fatalf("first Rebuild: %v", err) + } + stats, err := Rebuild(st, 0.6, bigWindow, 0) + if err != nil { + t.Fatalf("second Rebuild: %v", err) + } + if stats.Total != 2 { + t.Errorf("second run clusters = %d, want 2 (rebuild replaces, no accumulation)", stats.Total) + } + if m := countMembers(t, st); m != 4 { + t.Errorf("cluster_members = %d, want 4 after re-run", m) + } +} + +func TestRebuildLookbackExcludesOld(t *testing.T) { + st := openStore(t) + seedArticles(t, st) + + stats, err := Rebuild(st, 0.6, bigWindow, 250) + if err != nil { + t.Fatalf("Rebuild: %v", err) + } + if stats.Total != 2 { + t.Errorf("with since=250 only articles at t>=250 cluster, total = %d, want 2", stats.Total) + } + if m := countMembers(t, st); m != 2 { + t.Errorf("members = %d, want 2 (two articles below lookback excluded)", m) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go index a8692f5f..1f2677f2 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go @@ -24,8 +24,9 @@ const ( defaultCacheTTLHours = 24 defaultNegativeTTLHours = 3 - defaultTitleJaccard = 0.6 - defaultWindowHours = 72 + defaultTitleJaccard = 0.6 + defaultWindowHours = 72 + defaultLookbackHours = 168 trackingUTMPrefix = "utm_*" @@ -73,6 +74,7 @@ type Enrich struct { type Cluster struct { TitleJaccard float64 `yaml:"title_jaccard_threshold"` WindowHours int `yaml:"window_hours"` + LookbackHours int `yaml:"lookback_hours"` TrackingParams []string `yaml:"tracking_params"` } @@ -136,6 +138,7 @@ func Default() Config { Cluster: Cluster{ TitleJaccard: defaultTitleJaccard, WindowHours: defaultWindowHours, + LookbackHours: defaultLookbackHours, TrackingParams: defaultTrackingParams, }, Rank: Rank{ @@ -208,6 +211,15 @@ func (c Config) validate() error { if c.Cluster.TitleJaccard < 0 || c.Cluster.TitleJaccard > 1 { return fmt.Errorf("config: cluster.title_jaccard_threshold must be in [0,1], got %v", c.Cluster.TitleJaccard) } + if c.Cluster.WindowHours < 1 { + return fmt.Errorf("config: cluster.window_hours must be >= 1, got %d", c.Cluster.WindowHours) + } + if c.Cluster.LookbackHours < 1 { + return fmt.Errorf("config: cluster.lookback_hours must be >= 1, got %d", c.Cluster.LookbackHours) + } + if c.Cluster.LookbackHours < c.Cluster.WindowHours { + return fmt.Errorf("config: cluster.lookback_hours (%d) must be >= cluster.window_hours (%d) or window edges near the corpus boundary are silently dropped", c.Cluster.LookbackHours, c.Cluster.WindowHours) + } if c.Rank.HalfLifeHours < 1 { return fmt.Errorf("config: rank.half_life_hours must be >= 1, got %d", c.Rank.HalfLifeHours) } diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config_test.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config_test.go index 9a490a58..2eaaf370 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/config/config_test.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config_test.go @@ -92,3 +92,15 @@ func TestValidateRejectsBadProvider(t *testing.T) { t.Error("expected error for invalid ai.provider") } } + +func TestValidateRejectsLookbackBelowWindow(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "lb.yaml") + body := "cluster:\n window_hours: 72\n lookback_hours: 24\n" + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err == nil { + t.Error("expected error when lookback_hours < window_hours") + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go index a19dddc7..b56b7588 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go @@ -179,6 +179,100 @@ func (s *Store) UpsertFetchState(sourceID int64, fs FetchState) error { return nil } +type CandidateArticle struct { + ID int64 + SourceID int64 + Title string + Time int64 +} + +type ClusterRow struct { + Key string + Members []int64 + FirstSeen int64 + LastSeen int64 +} + +func (s *Store) ClusterCandidates(since int64) ([]CandidateArticle, error) { + rows, err := s.db.Query(` + SELECT id, source_id, title, COALESCE(NULLIF(published_at, 0), fetched_at) AS t + FROM articles + WHERE COALESCE(NULLIF(published_at, 0), fetched_at) >= ? + ORDER BY id`, since) + if err != nil { + return nil, fmt.Errorf("cluster candidates: %w", err) + } + defer rows.Close() + + var out []CandidateArticle + for rows.Next() { + var c CandidateArticle + if err := rows.Scan(&c.ID, &c.SourceID, &c.Title, &c.Time); err != nil { + return nil, fmt.Errorf("cluster candidates: scan: %w", err) + } + out = append(out, c) + } + return out, rows.Err() +} + +func (s *Store) ArticleCVEMap() (map[int64][]string, error) { + rows, err := s.db.Query(`SELECT article_id, cve_id FROM article_cves`) + if err != nil { + return nil, fmt.Errorf("article cve map: %w", err) + } + defer rows.Close() + + out := make(map[int64][]string) + for rows.Next() { + var articleID int64 + var cveID string + if err := rows.Scan(&articleID, &cveID); err != nil { + return nil, fmt.Errorf("article cve map: scan: %w", err) + } + out[articleID] = append(out[articleID], cveID) + } + return out, rows.Err() +} + +func (s *Store) ReplaceClusters(rows []ClusterRow) error { + tx, err := s.db.Begin() + if err != nil { + return fmt.Errorf("replace clusters: begin: %w", err) + } + defer func() { _ = tx.Rollback() }() + + if _, err := tx.Exec(`DELETE FROM cluster_members`); err != nil { + return fmt.Errorf("replace clusters: clear members: %w", err) + } + if _, err := tx.Exec(`DELETE FROM clusters`); err != nil { + return fmt.Errorf("replace clusters: clear clusters: %w", err) + } + + for _, r := range rows { + var clusterID int64 + if err := tx.QueryRow(` + INSERT INTO clusters (cluster_key, first_seen, last_seen, size) + VALUES (?, ?, ?, ?) RETURNING id`, + r.Key, r.FirstSeen, r.LastSeen, len(r.Members), + ).Scan(&clusterID); err != nil { + return fmt.Errorf("replace clusters: insert cluster %q: %w", r.Key, err) + } + for _, articleID := range r.Members { + if _, err := tx.Exec(` + INSERT INTO cluster_members (cluster_id, article_id) VALUES (?, ?)`, + clusterID, articleID, + ); err != nil { + return fmt.Errorf("replace clusters: insert member %d: %w", articleID, err) + } + } + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("replace clusters: commit: %w", err) + } + return nil +} + func boolToInt(b bool) int { if b { return 1 From 5865bb6149c4f0e47d0af64639e3169db394d93a Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Sun, 5 Jul 2026 15:54:49 -0400 Subject: [PATCH 04/15] feat(nadezhda): M3 CVE extraction + NVD/KEV/EPSS enrichment The intelligence layer. scrape now extracts CVE IDs (cheap, no network) into article_cves, which activates M2's shared-CVE clustering. A separate 'enrich' command does the slow, cached API work (Carter's split: fast scrape, deliberate enrich). - internal/cve: regex CVE-ID extraction + three clients (stdlib net/http): - nvd: API 2.0, CVSS precedence v4.0>v3.1>v3.0>v2 with nullable metrics and v2 metric-level severity fallback, totalResults==0 => not-found, apiKey header (exact case), token-bucket rate limit (5/30s anon, 50/30s keyed) + retry on 429/5xx, timeouts non-retryable. - kev: one catalog download -> membership map; knownRansomwareCampaignUse is the STRING 'Known'/'Unknown' mapped explicitly to bool. - epss: batched; epss/percentile are QUOTED STRINGS parsed with ParseFloat (a parse failure is skipped, a legitimate 0.0 is kept); partial batches survive a transient error. - internal/enrich: enriches only unenriched/stale CVEs (positive + negative TTL), KEV once per run, EPSS batched, NVD per-CVE. NVD/EPSS soft per-item (resumable), KEV fatal by design (enriching without it would persist wrong is_kev for a TTL). - store: UpsertCVEStub/LinkArticleCVE, CVEsNeedingEnrichment (TTL), UpdateCVEEnrichment (nullable *float64 -> SQL NULL), GetCVE, ArticlesForCVE, ListArticles (parameterized --source/--since/--min-cvss/--kev/--keyword via EXISTS subqueries). - commands: enrich, cve, list (cve/list off the stub list). Trimmed kev-sample.json (7KB) fixture for KAT tests; nvd/epss fixtures committed. - KAT proven offline AND live: CVE-2021-44228 -> CVSS 10.0 CRITICAL (3.1), CWE-20, KEV added 2021-12-10 ransomware yes, EPSS 0.99999. 215 articles -> 82 CVEs, and shared-CVE clustering lit up (multi-source clusters 2 -> 5). Suite offline + -race. One read-only audit agent run; 0 Crit/High/Med, Low/Nit fixes applied in-phase. NVD apiKey header set with exact case to bypass Go header canonicalization. --- .../security-news-scraper/cmd/nadezhda/cve.go | 105 +++++++++ .../cmd/nadezhda/enrich.go | 67 ++++++ .../cmd/nadezhda/list.go | 92 ++++++++ .../cmd/nadezhda/scrape.go | 12 +- .../cmd/nadezhda/stubs.go | 2 - .../internal/config/config.go | 5 +- .../internal/cve/cve_test.go | 207 ++++++++++++++++++ .../internal/cve/epss.go | 83 +++++++ .../internal/cve/extract.go | 29 +++ .../internal/cve/http.go | 53 +++++ .../security-news-scraper/internal/cve/kev.go | 65 ++++++ .../security-news-scraper/internal/cve/nvd.go | 196 +++++++++++++++++ .../internal/enrich/enrich.go | 105 +++++++++ .../internal/enrich/enrich_test.go | 121 ++++++++++ .../internal/ingest/ingest.go | 26 ++- .../internal/store/cve_test.go | 106 +++++++++ .../internal/store/store.go | 192 ++++++++++++++++ .../testdata/epss/CVE-2021-44228.json | 1 + .../testdata/kev/kev-sample.json | 85 +++++++ .../testdata/nvd/CVE-2021-44228.json | 1 + 20 files changed, 1541 insertions(+), 12 deletions(-) create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/cve.go create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/enrich.go create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/list.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/cve/cve_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/cve/epss.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/cve/extract.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/cve/http.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/cve/kev.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/cve/nvd.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/cve_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/epss/CVE-2021-44228.json create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/kev/kev-sample.json create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/nvd/CVE-2021-44228.json diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/cve.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/cve.go new file mode 100644 index 00000000..18ad9596 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/cve.go @@ -0,0 +1,105 @@ +// ©AngelaMos | 2026 +// cve.go + +package main + +import ( + "database/sql" + "errors" + "fmt" + "strings" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const naValue = "n/a" + +var cveCmd = &cobra.Command{ + Use: "cve CVE-YYYY-NNNN", + Short: "Show an enriched CVE and the articles mentioning it", + Args: cobra.ExactArgs(1), + RunE: runCVE, +} + +func init() { + rootCmd.AddCommand(cveCmd) +} + +func runCVE(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + + id := strings.ToUpper(args[0]) + c, err := st.GetCVE(id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("%s not found; run 'nadezhda scrape' to discover it", id) + } + return err + } + + out := cmd.OutOrStdout() + fmt.Fprintf(out, "%s\n", c.ID) + if c.EnrichedAt == 0 { + fmt.Fprintln(out, " not enriched yet; run 'nadezhda enrich'") + } else { + fmt.Fprintf(out, " CVSS: %s\n", cvssLine(c)) + fmt.Fprintf(out, " CWE: %s\n", orNA(c.CWE)) + fmt.Fprintf(out, " KEV: %s\n", kevLine(c)) + fmt.Fprintf(out, " EPSS: %s\n", epssLine(c)) + if c.Description != "" { + fmt.Fprintf(out, "\n %s\n", c.Description) + } + } + + articles, err := st.ArticlesForCVE(id) + if err != nil { + return err + } + fmt.Fprintf(out, "\nMentioned in %d article(s):\n", len(articles)) + for _, a := range articles { + fmt.Fprintf(out, " [%s] %s\n %s\n", a.SourceName, a.Title, a.CanonicalURL) + } + return nil +} + +func cvssLine(c store.CVE) string { + if c.CVSSScore == nil { + return naValue + } + return fmt.Sprintf("%.1f %s (%s) %s", *c.CVSSScore, orNA(c.CVSSSeverity), orNA(c.CVSSVersion), c.CVSSVector) +} + +func kevLine(c store.CVE) string { + if !c.IsKEV { + return "no" + } + ransom := "no" + if c.KEVRansomware { + ransom = "yes" + } + return fmt.Sprintf("yes (added %s, ransomware: %s)", orNA(c.KEVDateAdded), ransom) +} + +func epssLine(c store.CVE) string { + if c.EPSS == nil || c.EPSSPercentile == nil { + return naValue + } + return fmt.Sprintf("%.5f (percentile %.5f)", *c.EPSS, *c.EPSSPercentile) +} + +func orNA(s string) string { + if s == "" { + return naValue + } + return s +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/enrich.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/enrich.go new file mode 100644 index 00000000..0a35040d --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/enrich.go @@ -0,0 +1,67 @@ +// ©AngelaMos | 2026 +// enrich.go + +package main + +import ( + "context" + "fmt" + "net/http" + "os" + "os/signal" + "time" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/cve" + "github.com/CarterPerez-dev/nadezhda/internal/enrich" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const nvdAPIKeyEnv = "NVD_API_KEY" + +var enrichCmd = &cobra.Command{ + Use: "enrich", + Short: "Enrich extracted CVEs with NVD, CISA KEV, and EPSS intelligence", + RunE: runEnrich, +} + +func init() { + rootCmd.AddCommand(enrichCmd) +} + +func runEnrich(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + + httpClient := &http.Client{Timeout: time.Duration(cfg.Fetch.TimeoutSeconds) * time.Second} + apiKey := os.Getenv(nvdAPIKeyEnv) + if apiKey == "" { + apiKey = cfg.Enrich.NVDAPIKey + } + clients := enrich.Clients{ + NVD: cve.NewNVDClient(httpClient, cve.NVDEndpoint, apiKey), + KEV: cve.NewKEVClient(httpClient, cve.KEVEndpoint), + EPSS: cve.NewEPSSClient(httpClient, cve.EPSSEndpoint), + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + stats, err := enrich.Run(ctx, st, clients, time.Now(), cfg.Enrich.CacheTTLHours, cfg.Enrich.NegativeTTLHours) + if err != nil { + return err + } + + fmt.Fprintf(cmd.OutOrStdout(), + "enriched %d/%d CVEs (%d not in NVD, %d KEV, %d errors)\n", + stats.Enriched, stats.Total, stats.NotFound, stats.KEVHits, stats.Errors) + return nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/list.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/list.go new file mode 100644 index 00000000..ac91f619 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/list.go @@ -0,0 +1,92 @@ +// ©AngelaMos | 2026 +// list.go + +package main + +import ( + "fmt" + "time" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const ( + defaultListLimit = 50 + dateLayout = "2006-01-02" + noDate = "----------" +) + +var ( + listSource string + listSince string + listMinCVSS float64 + listKEV bool + listKeyword string + listLimit int +) + +var listCmd = &cobra.Command{ + Use: "list", + Short: "List stored articles with filters", + RunE: runList, +} + +func init() { + listCmd.Flags().StringVar(&listSource, "source", "", "filter by source name") + listCmd.Flags().StringVar(&listSince, "since", "", "only articles published within this window (e.g. 24h, 168h)") + listCmd.Flags().Float64Var(&listMinCVSS, "min-cvss", 0, "only articles referencing a CVE with CVSS >= this") + listCmd.Flags().BoolVar(&listKEV, "kev", false, "only articles referencing a KEV-listed CVE") + listCmd.Flags().StringVar(&listKeyword, "keyword", "", "filter by keyword in title or summary") + listCmd.Flags().IntVar(&listLimit, "limit", defaultListLimit, "maximum rows to show") + rootCmd.AddCommand(listCmd) +} + +func runList(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + + filter := store.ListFilter{ + Source: listSource, + MinCVSS: listMinCVSS, + KEV: listKEV, + Keyword: listKeyword, + Limit: listLimit, + } + if listSince != "" { + d, err := time.ParseDuration(listSince) + if err != nil { + return fmt.Errorf("invalid --since %q: %w", listSince, err) + } + filter.Since = time.Now().Add(-d).Unix() + } + + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + + articles, err := st.ListArticles(filter) + if err != nil { + return err + } + + out := cmd.OutOrStdout() + fmt.Fprintf(out, "%-10s %-16s %s\n", "DATE", "SOURCE", "TITLE") + for _, a := range articles { + fmt.Fprintf(out, "%-10s %-16s %s\n", formatDate(a.PublishedAt), a.SourceName, a.Title) + } + fmt.Fprintf(out, "\n%d article(s)\n", len(articles)) + return nil +} + +func formatDate(unix int64) string { + if unix == 0 { + return noDate + } + return time.Unix(unix, 0).UTC().Format(dateLayout) +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go index 0b49e69d..5bcbf4d4 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go @@ -105,14 +105,16 @@ func selectTargets(srcs []source.Source, only string) ([]source.Source, error) { func printSummary(cmd *cobra.Command, summary ingest.Summary) { out := cmd.OutOrStdout() - fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s\n", "SOURCE", "STATUS", "PARSED", "NEW", "DUP", "ERR") + fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s %-5s\n", "SOURCE", "STATUS", "PARSED", "NEW", "DUP", "CVE", "ERR") + totalCVEs := 0 for _, r := range summary.Results { - fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s\n", - r.Name, status(r), count(r, r.Parsed), count(r, r.New), count(r, r.Duplicates), count(r, r.ItemErrors)) + totalCVEs += r.CVEs + fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s %-5s\n", + r.Name, status(r), count(r, r.Parsed), count(r, r.New), count(r, r.Duplicates), count(r, r.CVEs), count(r, r.ItemErrors)) } newArticles, duplicates, failed := summary.Totals() - fmt.Fprintf(out, "\n%d new, %d duplicate across %d sources (%d failed)\n", - newArticles, duplicates, len(summary.Results), failed) + fmt.Fprintf(out, "\n%d new, %d duplicate, %d CVE refs across %d sources (%d failed)\n", + newArticles, duplicates, totalCVEs, len(summary.Results), failed) for _, r := range summary.Results { if r.Err != nil { fmt.Fprintf(out, " %s: %v\n", r.Name, r.Err) diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go index 3b9f1431..c1a4b829 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go @@ -21,8 +21,6 @@ func init() { short string milestone string }{ - {"list", "List stored articles with filters", "milestone M3"}, - {"cve", "Show an enriched CVE and articles mentioning it", "milestone M3"}, {"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"}, {"tui", "Browse aggregated news in an interactive terminal UI", "milestone M5"}, {"ideate", "Generate content angles from ranked clusters via an AI provider", "milestone M6"}, diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go index 1f2677f2..3a2c925d 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go @@ -67,8 +67,9 @@ type Fetch struct { } type Enrich struct { - CacheTTLHours int `yaml:"cache_ttl_hours"` - NegativeTTLHours int `yaml:"negative_ttl_hours"` + CacheTTLHours int `yaml:"cache_ttl_hours"` + NegativeTTLHours int `yaml:"negative_ttl_hours"` + NVDAPIKey string `yaml:"nvd_api_key"` } type Cluster struct { diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cve/cve_test.go b/PROJECTS/intermediate/security-news-scraper/internal/cve/cve_test.go new file mode 100644 index 00000000..be28b944 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/cve/cve_test.go @@ -0,0 +1,207 @@ +// ©AngelaMos | 2026 +// cve_test.go + +package cve + +import ( + "context" + "math" + "net/http" + "net/http/httptest" + "os" + "reflect" + "testing" + "time" + + "golang.org/x/time/rate" +) + +func fileServer(t *testing.T, path string) *httptest.Server { + t.Helper() + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read fixture %s: %v", path, err) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + return srv +} + +func jsonServer(t *testing.T, body string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestExtract(t *testing.T) { + cases := []struct { + in []string + want []string + }{ + {[]string{"See CVE-2021-44228 for details"}, []string{"CVE-2021-44228"}}, + {[]string{"cve-2021-44228 lowercase"}, []string{"CVE-2021-44228"}}, + {[]string{"CVE-2021-44228 and CVE-2021-44228 again"}, []string{"CVE-2021-44228"}}, + {[]string{"CVE-2026-1 too short", "CVE-2026-12345 ok"}, []string{"CVE-2026-12345"}}, + {[]string{"CVE-2021-45046, CVE-2021-44228"}, []string{"CVE-2021-44228", "CVE-2021-45046"}}, + {[]string{"no identifiers here"}, nil}, + } + for _, tc := range cases { + if got := Extract(tc.in...); !reflect.DeepEqual(got, tc.want) { + t.Errorf("Extract(%v) = %v, want %v", tc.in, got, tc.want) + } + } +} + +func TestNVDParsesLog4Shell(t *testing.T) { + srv := fileServer(t, "../../testdata/nvd/CVE-2021-44228.json") + c := NewNVDClient(srv.Client(), srv.URL, "") + + res, err := c.Fetch(context.Background(), "CVE-2021-44228") + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if !res.Found { + t.Fatal("expected Found") + } + if res.CVSSScore == nil || *res.CVSSScore != 10.0 { + t.Errorf("cvss score = %v, want 10.0", res.CVSSScore) + } + if res.CVSSSeverity != "CRITICAL" { + t.Errorf("severity = %q, want CRITICAL", res.CVSSSeverity) + } + if res.CVSSVersion != "3.1" { + t.Errorf("version = %q, want 3.1", res.CVSSVersion) + } + if res.CWE != "CWE-20" { + t.Errorf("cwe = %q, want CWE-20", res.CWE) + } + if res.Description == "" { + t.Error("description empty") + } +} + +func TestNVDNotFound(t *testing.T) { + srv := jsonServer(t, `{"totalResults":0,"vulnerabilities":[]}`) + c := NewNVDClient(srv.Client(), srv.URL, "") + res, err := c.Fetch(context.Background(), "CVE-2099-99999") + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if res.Found { + t.Error("expected not found for totalResults 0") + } +} + +func TestNVDPrecedencePrefersV40(t *testing.T) { + body := `{"totalResults":1,"vulnerabilities":[{"cve":{ + "id":"CVE-2026-1","descriptions":[{"lang":"en","value":"d"}], + "metrics":{ + "cvssMetricV40":[{"cvssData":{"version":"4.0","baseScore":9.3,"baseSeverity":"CRITICAL","vectorString":"CVSS:4.0/x"}}], + "cvssMetricV31":[{"cvssData":{"version":"3.1","baseScore":7.5,"baseSeverity":"HIGH","vectorString":"CVSS:3.1/y"}}] + }}}]}` + srv := jsonServer(t, body) + c := NewNVDClient(srv.Client(), srv.URL, "") + res, err := c.Fetch(context.Background(), "CVE-2026-1") + if err != nil { + t.Fatal(err) + } + if res.CVSSVersion != "4.0" || res.CVSSScore == nil || *res.CVSSScore != 9.3 { + t.Errorf("expected v4.0/9.3, got %s/%v", res.CVSSVersion, res.CVSSScore) + } +} + +func TestNVDV2SeverityFromMetricLevel(t *testing.T) { + body := `{"totalResults":1,"vulnerabilities":[{"cve":{ + "id":"CVE-2005-1","descriptions":[{"lang":"en","value":"d"}], + "metrics":{"cvssMetricV2":[{"baseSeverity":"HIGH","cvssData":{"version":"2.0","baseScore":7.5,"vectorString":"AV:N"}}]} + }}]}` + srv := jsonServer(t, body) + c := NewNVDClient(srv.Client(), srv.URL, "") + res, err := c.Fetch(context.Background(), "CVE-2005-1") + if err != nil { + t.Fatal(err) + } + if res.CVSSVersion != "2.0" || res.CVSSSeverity != "HIGH" { + t.Errorf("v2 severity should fall back to metric level, got %s/%q", res.CVSSVersion, res.CVSSSeverity) + } +} + +func TestKEVParsesSample(t *testing.T) { + srv := fileServer(t, "../../testdata/kev/kev-sample.json") + c := NewKEVClient(srv.Client(), srv.URL) + catalog, err := c.Fetch(context.Background()) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + log4, ok := catalog.Lookup("CVE-2021-44228") + if !ok { + t.Fatal("Log4Shell should be in KEV") + } + if !log4.Ransomware { + t.Error("Log4Shell knownRansomwareCampaignUse=Known should map to true") + } + unknown, ok := catalog.Lookup("CVE-2026-45659") + if !ok || unknown.Ransomware { + t.Error("Unknown ransomware entry should map to false") + } + if _, ok := catalog.Lookup("CVE-1999-0001"); ok { + t.Error("absent CVE should not be a KEV member") + } +} + +func TestEPSSParsesQuotedStrings(t *testing.T) { + srv := fileServer(t, "../../testdata/epss/CVE-2021-44228.json") + c := NewEPSSClient(srv.Client(), srv.URL) + scores, err := c.Fetch(context.Background(), []string{"CVE-2021-44228"}) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + s, ok := scores["CVE-2021-44228"] + if !ok { + t.Fatal("expected a score for Log4Shell") + } + if math.Abs(s.EPSS-0.99999) > 1e-5 { + t.Errorf("epss = %v, want ~0.99999 (string must be ParseFloat'd, not zeroed)", s.EPSS) + } + if math.Abs(s.Percentile-1.0) > 1e-9 { + t.Errorf("percentile = %v, want 1.0", s.Percentile) + } +} + +func TestChunk(t *testing.T) { + got := chunk([]string{"a", "b", "c", "d", "e"}, 2) + want := [][]string{{"a", "b"}, {"c", "d"}, {"e"}} + if !reflect.DeepEqual(got, want) { + t.Errorf("chunk = %v, want %v", got, want) + } + if chunk(nil, 2) != nil { + t.Error("chunk(nil) should be nil") + } +} + +func TestNVDRetriesOn503(t *testing.T) { + var calls int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if calls == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = w.Write([]byte(`{"totalResults":0,"vulnerabilities":[]}`)) + })) + t.Cleanup(srv.Close) + c := NewNVDClient(srv.Client(), srv.URL, "") + c.limiter = rate.NewLimiter(rate.Inf, 1) + c.backoffBase = time.Millisecond + if _, err := c.Fetch(context.Background(), "CVE-2026-1"); err != nil { + t.Fatalf("Fetch after retry: %v", err) + } + if calls != 2 { + t.Errorf("calls = %d, want 2 (one 503, one 200)", calls) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cve/epss.go b/PROJECTS/intermediate/security-news-scraper/internal/cve/epss.go new file mode 100644 index 00000000..70c7a395 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/cve/epss.go @@ -0,0 +1,83 @@ +// ©AngelaMos | 2026 +// epss.go + +package cve + +import ( + "context" + "net/http" + "net/url" + "strconv" + "strings" + + "golang.org/x/time/rate" +) + +const ( + EPSSEndpoint = "https://api.first.org/data/v1/epss" + + epssBatchSize = 100 + epssRate = 5 +) + +type EPSSScore struct { + EPSS float64 + Percentile float64 +} + +type EPSSClient struct { + http *http.Client + baseURL string + limiter *rate.Limiter +} + +func NewEPSSClient(client *http.Client, baseURL string) *EPSSClient { + return &EPSSClient{ + http: client, + baseURL: baseURL, + limiter: rate.NewLimiter(rate.Limit(epssRate), 1), + } +} + +func (c *EPSSClient) Fetch(ctx context.Context, cveIDs []string) (map[string]EPSSScore, error) { + out := make(map[string]EPSSScore, len(cveIDs)) + for _, batch := range chunk(cveIDs, epssBatchSize) { + if err := c.limiter.Wait(ctx); err != nil { + return out, err + } + endpoint := c.baseURL + "?" + url.Values{"cve": {strings.Join(batch, ",")}}.Encode() + var raw epssRaw + if err := getJSON(ctx, c.http, endpoint, nil, &raw); err != nil { + return out, err + } + for _, d := range raw.Data { + epss, err1 := strconv.ParseFloat(d.EPSS, 64) + percentile, err2 := strconv.ParseFloat(d.Percentile, 64) + if err1 != nil || err2 != nil { + continue + } + out[d.CVE] = EPSSScore{EPSS: epss, Percentile: percentile} + } + } + return out, nil +} + +func chunk(ids []string, size int) [][]string { + var out [][]string + for i := 0; i < len(ids); i += size { + end := i + size + if end > len(ids) { + end = len(ids) + } + out = append(out, ids[i:end]) + } + return out +} + +type epssRaw struct { + Data []struct { + CVE string `json:"cve"` + EPSS string `json:"epss"` + Percentile string `json:"percentile"` + } `json:"data"` +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cve/extract.go b/PROJECTS/intermediate/security-news-scraper/internal/cve/extract.go new file mode 100644 index 00000000..6c1e4a74 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/cve/extract.go @@ -0,0 +1,29 @@ +// ©AngelaMos | 2026 +// extract.go + +package cve + +import ( + "regexp" + "sort" + "strings" +) + +var pattern = regexp.MustCompile(`(?i)CVE-\d{4}-\d{4,7}`) + +func Extract(texts ...string) []string { + seen := make(map[string]struct{}) + var out []string + for _, text := range texts { + for _, match := range pattern.FindAllString(text, -1) { + id := strings.ToUpper(match) + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + out = append(out, id) + } + } + sort.Strings(out) + return out +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cve/http.go b/PROJECTS/intermediate/security-news-scraper/internal/cve/http.go new file mode 100644 index 00000000..89551d76 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/cve/http.go @@ -0,0 +1,53 @@ +// ©AngelaMos | 2026 +// http.go + +package cve + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" +) + +const ( + headerAccept = "Accept" + acceptJSON = "application/json" + maxJSONBytes = 32 << 20 +) + +type statusError struct { + code int + url string +} + +func (e *statusError) Error() string { + return fmt.Sprintf("cve: GET %s: status %d", e.url, e.code) +} + +func getJSON(ctx context.Context, client *http.Client, url string, header http.Header, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("cve: build request %s: %w", url, err) + } + req.Header.Set(headerAccept, acceptJSON) + for key, values := range header { + req.Header[key] = values + } + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("cve: GET %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxJSONBytes)) + return &statusError{code: resp.StatusCode, url: url} + } + if err := json.NewDecoder(io.LimitReader(resp.Body, maxJSONBytes)).Decode(out); err != nil { + return fmt.Errorf("cve: decode %s: %w", url, err) + } + return nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cve/kev.go b/PROJECTS/intermediate/security-news-scraper/internal/cve/kev.go new file mode 100644 index 00000000..4aaed306 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/cve/kev.go @@ -0,0 +1,65 @@ +// ©AngelaMos | 2026 +// kev.go + +package cve + +import ( + "context" + "net/http" +) + +const ( + KEVEndpoint = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" + + kevRansomwareKnown = "Known" +) + +type KEVEntry struct { + DateAdded string + Ransomware bool +} + +type KEVCatalog struct { + Version string + Released string + Entries map[string]KEVEntry +} + +func (c KEVCatalog) Lookup(cveID string) (KEVEntry, bool) { + e, ok := c.Entries[cveID] + return e, ok +} + +type KEVClient struct { + http *http.Client + baseURL string +} + +func NewKEVClient(client *http.Client, baseURL string) *KEVClient { + return &KEVClient{http: client, baseURL: baseURL} +} + +func (c *KEVClient) Fetch(ctx context.Context) (KEVCatalog, error) { + var raw kevRaw + if err := getJSON(ctx, c.http, c.baseURL, nil, &raw); err != nil { + return KEVCatalog{}, err + } + entries := make(map[string]KEVEntry, len(raw.Vulnerabilities)) + for _, v := range raw.Vulnerabilities { + entries[v.CveID] = KEVEntry{ + DateAdded: v.DateAdded, + Ransomware: v.KnownRansomwareCampaignUse == kevRansomwareKnown, + } + } + return KEVCatalog{Version: raw.CatalogVersion, Released: raw.DateReleased, Entries: entries}, nil +} + +type kevRaw struct { + CatalogVersion string `json:"catalogVersion"` + DateReleased string `json:"dateReleased"` + Vulnerabilities []struct { + CveID string `json:"cveID"` + DateAdded string `json:"dateAdded"` + KnownRansomwareCampaignUse string `json:"knownRansomwareCampaignUse"` + } `json:"vulnerabilities"` +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cve/nvd.go b/PROJECTS/intermediate/security-news-scraper/internal/cve/nvd.go new file mode 100644 index 00000000..6e2be481 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/cve/nvd.go @@ -0,0 +1,196 @@ +// ©AngelaMos | 2026 +// nvd.go + +package cve + +import ( + "context" + "errors" + "net/http" + "net/url" + "time" + + "golang.org/x/time/rate" +) + +const ( + NVDEndpoint = "https://services.nvd.nist.gov/rest/json/cves/2.0" + + nvdAPIKeyHeader = "apiKey" + nvdRateNoKey = 5.0 / 30.0 + nvdRateWithKey = 50.0 / 30.0 + nvdMaxRetries = 3 + nvdBackoffBase = 2 * time.Second + langEnglish = "en" +) + +type NVDResult struct { + Found bool + Description string + CVSSScore *float64 + CVSSVersion string + CVSSSeverity string + CVSSVector string + CWE string + Published string + Modified string +} + +type NVDClient struct { + http *http.Client + baseURL string + apiKey string + limiter *rate.Limiter + backoffBase time.Duration +} + +func NewNVDClient(client *http.Client, baseURL, apiKey string) *NVDClient { + limit := nvdRateNoKey + if apiKey != "" { + limit = nvdRateWithKey + } + return &NVDClient{ + http: client, + baseURL: baseURL, + apiKey: apiKey, + limiter: rate.NewLimiter(rate.Limit(limit), 1), + backoffBase: nvdBackoffBase, + } +} + +func (c *NVDClient) Fetch(ctx context.Context, cveID string) (NVDResult, error) { + endpoint := c.baseURL + "?" + url.Values{"cveId": {cveID}}.Encode() + header := http.Header{} + if c.apiKey != "" { + header[nvdAPIKeyHeader] = []string{c.apiKey} + } + + var env nvdEnvelope + for attempt := 0; attempt <= nvdMaxRetries; attempt++ { + if attempt > 0 { + if err := sleep(ctx, c.backoffBase*time.Duration(1<<(attempt-1))); err != nil { + return NVDResult{}, err + } + } + if err := c.limiter.Wait(ctx); err != nil { + return NVDResult{}, err + } + err := getJSON(ctx, c.http, endpoint, header, &env) + if err == nil { + break + } + if !retriable(err) || attempt == nvdMaxRetries { + return NVDResult{}, err + } + } + + if env.TotalResults == 0 || len(env.Vulnerabilities) == 0 { + return NVDResult{Found: false}, nil + } + return env.Vulnerabilities[0].CVE.toResult(), nil +} + +func retriable(err error) bool { + var se *statusError + if errors.As(err, &se) { + return se.code == http.StatusTooManyRequests || se.code >= http.StatusInternalServerError + } + return true +} + +type nvdEnvelope struct { + TotalResults int `json:"totalResults"` + Vulnerabilities []struct { + CVE nvdCVE `json:"cve"` + } `json:"vulnerabilities"` +} + +type nvdCVE struct { + ID string `json:"id"` + Published string `json:"published"` + LastModified string `json:"lastModified"` + Descriptions []nvdLangValue `json:"descriptions"` + Weaknesses []struct { + Description []nvdLangValue `json:"description"` + } `json:"weaknesses"` + Metrics nvdMetrics `json:"metrics"` +} + +type nvdLangValue struct { + Lang string `json:"lang"` + Value string `json:"value"` +} + +type nvdMetrics struct { + V40 []nvdMetric `json:"cvssMetricV40"` + V31 []nvdMetric `json:"cvssMetricV31"` + V30 []nvdMetric `json:"cvssMetricV30"` + V2 []nvdMetric `json:"cvssMetricV2"` +} + +type nvdMetric struct { + BaseSeverity string `json:"baseSeverity"` + CVSSData nvdCVSSData `json:"cvssData"` +} + +type nvdCVSSData struct { + Version string `json:"version"` + VectorString string `json:"vectorString"` + BaseScore float64 `json:"baseScore"` + BaseSeverity string `json:"baseSeverity"` +} + +func (v nvdCVE) toResult() NVDResult { + res := NVDResult{ + Found: true, + Description: english(v.Descriptions), + Published: v.Published, + Modified: v.LastModified, + } + for _, w := range v.Weaknesses { + if cwe := english(w.Description); cwe != "" { + res.CWE = cwe + break + } + } + if m, ok := v.Metrics.selected(); ok { + score := m.CVSSData.BaseScore + res.CVSSScore = &score + res.CVSSVersion = m.CVSSData.Version + res.CVSSVector = m.CVSSData.VectorString + res.CVSSSeverity = m.CVSSData.BaseSeverity + if res.CVSSSeverity == "" { + res.CVSSSeverity = m.BaseSeverity + } + } + return res +} + +func (m nvdMetrics) selected() (nvdMetric, bool) { + for _, tier := range [][]nvdMetric{m.V40, m.V31, m.V30, m.V2} { + if len(tier) > 0 { + return tier[0], true + } + } + return nvdMetric{}, false +} + +func english(values []nvdLangValue) string { + for _, lv := range values { + if lv.Lang == langEnglish { + return lv.Value + } + } + return "" +} + +func sleep(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich.go b/PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich.go new file mode 100644 index 00000000..584ab983 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich.go @@ -0,0 +1,105 @@ +// ©AngelaMos | 2026 +// enrich.go + +package enrich + +import ( + "context" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/cve" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const secondsPerHour = 3600 + +type Clients struct { + NVD *cve.NVDClient + KEV *cve.KEVClient + EPSS *cve.EPSSClient +} + +type Stats struct { + Total int + Enriched int + NotFound int + KEVHits int + Errors int +} + +func Run(ctx context.Context, st *store.Store, clients Clients, now time.Time, positiveTTLHours, negativeTTLHours int) (Stats, error) { + positiveTTL := int64(positiveTTLHours) * secondsPerHour + negativeTTL := int64(negativeTTLHours) * secondsPerHour + + ids, err := st.CVEsNeedingEnrichment(now.Unix(), positiveTTL, negativeTTL) + if err != nil { + return Stats{}, err + } + if len(ids) == 0 { + return Stats{}, nil + } + + catalog, err := clients.KEV.Fetch(ctx) + if err != nil { + return Stats{}, err + } + + epssScores, _ := clients.EPSS.Fetch(ctx, ids) + if epssScores == nil { + epssScores = map[string]cve.EPSSScore{} + } + + stats := Stats{Total: len(ids)} + for _, id := range ids { + nvdRes, err := clients.NVD.Fetch(ctx, id) + if err != nil { + if ctx.Err() != nil { + return stats, ctx.Err() + } + stats.Errors++ + continue + } + + rec := store.CVE{ID: id, EnrichedAt: now.Unix()} + if nvdRes.Found { + rec.EnrichStatus = store.EnrichStatusOK + rec.Description = nvdRes.Description + rec.CVSSScore = nvdRes.CVSSScore + rec.CVSSVersion = nvdRes.CVSSVersion + rec.CVSSSeverity = nvdRes.CVSSSeverity + rec.CVSSVector = nvdRes.CVSSVector + rec.CWE = nvdRes.CWE + rec.NVDPublished = nvdRes.Published + rec.NVDModified = nvdRes.Modified + } else { + rec.EnrichStatus = store.EnrichStatusNotFound + } + + if entry, ok := catalog.Lookup(id); ok { + rec.IsKEV = true + rec.KEVDateAdded = entry.DateAdded + rec.KEVRansomware = entry.Ransomware + } + + if score, ok := epssScores[id]; ok { + epssVal := score.EPSS + percentileVal := score.Percentile + rec.EPSS = &epssVal + rec.EPSSPercentile = &percentileVal + } + + if err := st.UpdateCVEEnrichment(rec); err != nil { + stats.Errors++ + continue + } + if nvdRes.Found { + stats.Enriched++ + } else { + stats.NotFound++ + } + if rec.IsKEV { + stats.KEVHits++ + } + } + return stats, nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich_test.go b/PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich_test.go new file mode 100644 index 00000000..6a1bdd06 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich_test.go @@ -0,0 +1,121 @@ +// ©AngelaMos | 2026 +// enrich_test.go + +package enrich + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/cve" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +func fileServer(t *testing.T, path string) *httptest.Server { + t.Helper() + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + return srv +} + +func newStore(t *testing.T) *store.Store { + t.Helper() + st, err := store.Open(filepath.Join(t.TempDir(), "enrich.db")) + if err != nil { + t.Fatalf("open store: %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + return st +} + +func TestRunEnrichesLog4Shell(t *testing.T) { + nvd := fileServer(t, "../../testdata/nvd/CVE-2021-44228.json") + kev := fileServer(t, "../../testdata/kev/kev-sample.json") + epss := fileServer(t, "../../testdata/epss/CVE-2021-44228.json") + + st := newStore(t) + if err := st.UpsertCVEStub("CVE-2021-44228"); err != nil { + t.Fatal(err) + } + + clients := Clients{ + NVD: cve.NewNVDClient(nvd.Client(), nvd.URL, ""), + KEV: cve.NewKEVClient(kev.Client(), kev.URL), + EPSS: cve.NewEPSSClient(epss.Client(), epss.URL), + } + + stats, err := Run(context.Background(), st, clients, time.Unix(1_800_000_000, 0), 24, 3) + if err != nil { + t.Fatalf("Run: %v", err) + } + if stats.Enriched != 1 || stats.KEVHits != 1 { + t.Fatalf("stats = %+v, want Enriched:1 KEVHits:1", stats) + } + + c, err := st.GetCVE("CVE-2021-44228") + if err != nil { + t.Fatal(err) + } + if c.CVSSScore == nil || *c.CVSSScore != 10.0 || c.CVSSSeverity != "CRITICAL" { + t.Errorf("cvss = %v/%q, want 10.0/CRITICAL", c.CVSSScore, c.CVSSSeverity) + } + if !c.IsKEV || !c.KEVRansomware { + t.Errorf("expected KEV + ransomware, got is_kev=%v ransomware=%v", c.IsKEV, c.KEVRansomware) + } + if c.EPSS == nil || *c.EPSS < 0.9 { + t.Errorf("epss = %v, want ~0.99999", c.EPSS) + } + if c.EnrichStatus != store.EnrichStatusOK { + t.Errorf("status = %q, want ok", c.EnrichStatus) + } +} + +func TestRunSkipsFreshAndMarksNotFound(t *testing.T) { + nvd := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"totalResults":0,"vulnerabilities":[]}`)) + })) + t.Cleanup(nvd.Close) + kev := fileServer(t, "../../testdata/kev/kev-sample.json") + epss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"data":[]}`)) + })) + t.Cleanup(epss.Close) + + st := newStore(t) + if err := st.UpsertCVEStub("CVE-2099-99999"); err != nil { + t.Fatal(err) + } + clients := Clients{ + NVD: cve.NewNVDClient(nvd.Client(), nvd.URL, ""), + KEV: cve.NewKEVClient(kev.Client(), kev.URL), + EPSS: cve.NewEPSSClient(epss.Client(), epss.URL), + } + now := time.Unix(1_800_000_000, 0) + + stats, err := Run(context.Background(), st, clients, now, 24, 3) + if err != nil { + t.Fatalf("Run: %v", err) + } + if stats.NotFound != 1 { + t.Errorf("NotFound = %d, want 1", stats.NotFound) + } + + fresh, err := Run(context.Background(), st, clients, now, 24, 3) + if err != nil { + t.Fatalf("second Run: %v", err) + } + if fresh.Total != 0 { + t.Errorf("second run Total = %d, want 0 (not_found within negative TTL is skipped)", fresh.Total) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go b/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go index 215dc79b..7a2de30c 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go @@ -12,6 +12,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/cve" "github.com/CarterPerez-dev/nadezhda/internal/fetch" "github.com/CarterPerez-dev/nadezhda/internal/normalize" "github.com/CarterPerez-dev/nadezhda/internal/parse" @@ -24,6 +25,7 @@ type SourceResult struct { Parsed int New int Duplicates int + CVEs int ItemErrors int NotModified bool Err error @@ -134,19 +136,22 @@ func storeItem(st *store.Store, cfg config.Config, sourceID int64, it parse.Item return } + summary := normalize.StripHTML(it.Summary) + body := normalize.StripHTML(it.Body) + var publishedAt int64 if !it.Published.IsZero() { publishedAt = it.Published.Unix() } - _, err = st.InsertArticle(store.Article{ + id, err := st.InsertArticle(store.Article{ SourceID: sourceID, CanonicalURL: canonical, ContentHash: normalize.ContentHash(canonical), TitleHash: normalize.TitleHash(normalize.NormalizeTitle(it.Title)), Title: it.Title, - Summary: normalize.StripHTML(it.Summary), - Body: normalize.StripHTML(it.Body), + Summary: summary, + Body: body, Author: it.Author, PublishedAt: publishedAt, FetchedAt: now.Unix(), @@ -154,9 +159,24 @@ func storeItem(st *store.Store, cfg config.Config, sourceID int64, it parse.Item switch { case err == nil: out.New++ + linkCVEs(st, id, cve.Extract(it.Title, summary, body), out) case errors.Is(err, store.ErrDuplicate): out.Duplicates++ default: out.ItemErrors++ } } + +func linkCVEs(st *store.Store, articleID int64, ids []string, out *SourceResult) { + for _, id := range ids { + if err := st.UpsertCVEStub(id); err != nil { + out.ItemErrors++ + continue + } + if err := st.LinkArticleCVE(articleID, id); err != nil { + out.ItemErrors++ + continue + } + out.CVEs++ + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/cve_test.go b/PROJECTS/intermediate/security-news-scraper/internal/store/cve_test.go new file mode 100644 index 00000000..12c28992 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/cve_test.go @@ -0,0 +1,106 @@ +// ©AngelaMos | 2026 +// cve_test.go + +package store + +import "testing" + +func seedArticleWithCVE(t *testing.T, s *Store, cveID string, cvss float64, kev bool) int64 { + t.Helper() + sourceID, err := s.UpsertSource(SourceInput{ + Name: "src", URL: "https://ex.example/feed", Type: "rss", Weight: 1, Enabled: true, + }) + if err != nil { + t.Fatal(err) + } + articleID, err := s.InsertArticle(Article{ + SourceID: sourceID, CanonicalURL: "https://ex.example/a", ContentHash: "ch", TitleHash: "th", + Title: "Exploit in the wild", Summary: "details", PublishedAt: 1000, FetchedAt: 1000, + }) + if err != nil { + t.Fatal(err) + } + if err := s.UpsertCVEStub(cveID); err != nil { + t.Fatal(err) + } + if err := s.LinkArticleCVE(articleID, cveID); err != nil { + t.Fatal(err) + } + score := cvss + if err := s.UpdateCVEEnrichment(CVE{ + ID: cveID, CVSSScore: &score, CVSSSeverity: "CRITICAL", IsKEV: kev, + EnrichedAt: 2000, EnrichStatus: EnrichStatusOK, + }); err != nil { + t.Fatal(err) + } + return articleID +} + +func TestArticlesForCVE(t *testing.T) { + s := openTemp(t) + seedArticleWithCVE(t, s, "CVE-2026-1", 9.8, true) + got, err := s.ArticlesForCVE("CVE-2026-1") + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Title != "Exploit in the wild" { + t.Errorf("ArticlesForCVE = %+v, want one article", got) + } +} + +func TestLinkArticleCVEIsIdempotent(t *testing.T) { + s := openTemp(t) + id := seedArticleWithCVE(t, s, "CVE-2026-1", 5.0, false) + if err := s.LinkArticleCVE(id, "CVE-2026-1"); err != nil { + t.Fatalf("re-link should be a no-op: %v", err) + } + got, _ := s.ArticlesForCVE("CVE-2026-1") + if len(got) != 1 { + t.Errorf("duplicate link produced %d rows, want 1", len(got)) + } +} + +func TestListArticlesFilters(t *testing.T) { + s := openTemp(t) + seedArticleWithCVE(t, s, "CVE-2026-1", 9.8, true) + + if got, _ := s.ListArticles(ListFilter{KEV: true}); len(got) != 1 { + t.Errorf("--kev returned %d, want 1", len(got)) + } + if got, _ := s.ListArticles(ListFilter{MinCVSS: 9.0}); len(got) != 1 { + t.Errorf("--min-cvss 9 returned %d, want 1", len(got)) + } + if got, _ := s.ListArticles(ListFilter{MinCVSS: 10.0}); len(got) != 0 { + t.Errorf("--min-cvss 10 returned %d, want 0", len(got)) + } + if got, _ := s.ListArticles(ListFilter{Keyword: "exploit"}); len(got) != 1 { + t.Errorf("--keyword returned %d, want 1", len(got)) + } + if got, _ := s.ListArticles(ListFilter{Source: "nope"}); len(got) != 0 { + t.Errorf("unknown source returned %d, want 0", len(got)) + } +} + +func TestCVEsNeedingEnrichment(t *testing.T) { + s := openTemp(t) + if err := s.UpsertCVEStub("CVE-2026-1"); err != nil { + t.Fatal(err) + } + need, err := s.CVEsNeedingEnrichment(10000, 3600, 1800) + if err != nil { + t.Fatal(err) + } + if len(need) != 1 { + t.Fatalf("fresh stub should need enrichment, got %d", len(need)) + } + score := 5.0 + if err := s.UpdateCVEEnrichment(CVE{ID: "CVE-2026-1", CVSSScore: &score, EnrichedAt: 9000, EnrichStatus: EnrichStatusOK}); err != nil { + t.Fatal(err) + } + if need, _ := s.CVEsNeedingEnrichment(10000, 3600, 1800); len(need) != 0 { + t.Errorf("recently enriched cve should be skipped, got %d", len(need)) + } + if need, _ := s.CVEsNeedingEnrichment(99999, 3600, 1800); len(need) != 1 { + t.Errorf("stale cve (past TTL) should need re-enrichment, got %d", len(need)) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go index b56b7588..b65e681e 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go @@ -15,6 +15,11 @@ import ( var ErrDuplicate = errors.New("store: article already exists") +const ( + EnrichStatusOK = "ok" + EnrichStatusNotFound = "not_found" +) + type Store struct { db *sql.DB version int @@ -179,6 +184,135 @@ func (s *Store) UpsertFetchState(sourceID int64, fs FetchState) error { return nil } +type CVE struct { + ID string + Description string + CVSSScore *float64 + CVSSVersion string + CVSSSeverity string + CVSSVector string + CWE string + IsKEV bool + KEVDateAdded string + KEVRansomware bool + EPSS *float64 + EPSSPercentile *float64 + NVDPublished string + NVDModified string + EnrichedAt int64 + EnrichStatus string +} + +type ArticleSummary struct { + ID int64 + SourceName string + Title string + CanonicalURL string + PublishedAt int64 +} + +type ListFilter struct { + Source string + Since int64 + MinCVSS float64 + KEV bool + Keyword string + Limit int +} + +func (s *Store) CVEsNeedingEnrichment(now, positiveTTL, negativeTTL int64) ([]string, error) { + rows, err := s.db.Query(` + SELECT id FROM cves + WHERE enriched_at = 0 + OR (enrich_status = ? AND enriched_at < ?) + OR (enrich_status = ? AND enriched_at < ?) + ORDER BY id`, EnrichStatusOK, now-positiveTTL, EnrichStatusNotFound, now-negativeTTL) + if err != nil { + return nil, fmt.Errorf("cves needing enrichment: %w", err) + } + defer rows.Close() + var out []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("cves needing enrichment: scan: %w", err) + } + out = append(out, id) + } + return out, rows.Err() +} + +func (s *Store) UpdateCVEEnrichment(c CVE) error { + _, err := s.db.Exec(` + UPDATE cves SET + description = ?, cvss_score = ?, cvss_version = ?, cvss_severity = ?, cvss_vector = ?, + cwe = ?, is_kev = ?, kev_date_added = ?, kev_ransomware = ?, + epss = ?, epss_percentile = ?, nvd_published = ?, nvd_modified = ?, + enriched_at = ?, enrich_status = ? + WHERE id = ?`, + c.Description, c.CVSSScore, c.CVSSVersion, c.CVSSSeverity, c.CVSSVector, + c.CWE, boolToInt(c.IsKEV), c.KEVDateAdded, boolToInt(c.KEVRansomware), + c.EPSS, c.EPSSPercentile, c.NVDPublished, c.NVDModified, + c.EnrichedAt, c.EnrichStatus, c.ID, + ) + if err != nil { + return fmt.Errorf("update cve enrichment %q: %w", c.ID, err) + } + return nil +} + +func (s *Store) GetCVE(id string) (CVE, error) { + var c CVE + var isKEV, ransomware int + err := s.db.QueryRow(` + SELECT id, description, cvss_score, cvss_version, cvss_severity, cvss_vector, + cwe, is_kev, kev_date_added, kev_ransomware, epss, epss_percentile, + nvd_published, nvd_modified, enriched_at, enrich_status + FROM cves WHERE id = ?`, id, + ).Scan(&c.ID, &c.Description, &c.CVSSScore, &c.CVSSVersion, &c.CVSSSeverity, &c.CVSSVector, + &c.CWE, &isKEV, &c.KEVDateAdded, &ransomware, &c.EPSS, &c.EPSSPercentile, + &c.NVDPublished, &c.NVDModified, &c.EnrichedAt, &c.EnrichStatus) + if err != nil { + return CVE{}, fmt.Errorf("get cve %q: %w", id, err) + } + c.IsKEV = isKEV != 0 + c.KEVRansomware = ransomware != 0 + return c, nil +} + +func (s *Store) ArticlesForCVE(id string) ([]ArticleSummary, error) { + rows, err := s.db.Query(` + SELECT a.id, s.name, a.title, a.canonical_url, a.published_at + FROM article_cves ac + JOIN articles a ON a.id = ac.article_id + JOIN sources s ON s.id = a.source_id + WHERE ac.cve_id = ? + ORDER BY a.published_at DESC`, id) + if err != nil { + return nil, fmt.Errorf("articles for cve %q: %w", id, err) + } + defer rows.Close() + return scanArticleSummaries(rows) +} + +func (s *Store) UpsertCVEStub(id string) error { + _, err := s.db.Exec(`INSERT INTO cves (id) VALUES (?) ON CONFLICT(id) DO NOTHING`, id) + if err != nil { + return fmt.Errorf("upsert cve %q: %w", id, err) + } + return nil +} + +func (s *Store) LinkArticleCVE(articleID int64, cveID string) error { + _, err := s.db.Exec(` + INSERT INTO article_cves (article_id, cve_id) VALUES (?, ?) + ON CONFLICT(article_id, cve_id) DO NOTHING`, articleID, cveID) + if err != nil { + return fmt.Errorf("link article %d cve %q: %w", articleID, cveID, err) + } + return nil +} + type CandidateArticle struct { ID int64 SourceID int64 @@ -273,6 +407,64 @@ func (s *Store) ReplaceClusters(rows []ClusterRow) error { return nil } +func (s *Store) ListArticles(f ListFilter) ([]ArticleSummary, error) { + query := ` + SELECT a.id, s.name, a.title, a.canonical_url, a.published_at + FROM articles a + JOIN sources s ON s.id = a.source_id + WHERE 1 = 1` + var args []any + + if f.Source != "" { + query += ` AND s.name = ?` + args = append(args, f.Source) + } + if f.Since > 0 { + query += ` AND a.published_at >= ?` + args = append(args, f.Since) + } + if f.Keyword != "" { + query += ` AND (a.title LIKE ? OR a.summary LIKE ?)` + like := "%" + f.Keyword + "%" + args = append(args, like, like) + } + if f.MinCVSS > 0 { + query += ` AND EXISTS ( + SELECT 1 FROM article_cves ac JOIN cves c ON c.id = ac.cve_id + WHERE ac.article_id = a.id AND c.cvss_score >= ?)` + args = append(args, f.MinCVSS) + } + if f.KEV { + query += ` AND EXISTS ( + SELECT 1 FROM article_cves ac JOIN cves c ON c.id = ac.cve_id + WHERE ac.article_id = a.id AND c.is_kev = 1)` + } + query += ` ORDER BY a.published_at DESC` + if f.Limit > 0 { + query += ` LIMIT ?` + args = append(args, f.Limit) + } + + rows, err := s.db.Query(query, args...) + if err != nil { + return nil, fmt.Errorf("list articles: %w", err) + } + defer rows.Close() + return scanArticleSummaries(rows) +} + +func scanArticleSummaries(rows *sql.Rows) ([]ArticleSummary, error) { + var out []ArticleSummary + for rows.Next() { + var a ArticleSummary + if err := rows.Scan(&a.ID, &a.SourceName, &a.Title, &a.CanonicalURL, &a.PublishedAt); err != nil { + return nil, fmt.Errorf("scan article summary: %w", err) + } + out = append(out, a) + } + return out, rows.Err() +} + func boolToInt(b bool) int { if b { return 1 diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/epss/CVE-2021-44228.json b/PROJECTS/intermediate/security-news-scraper/testdata/epss/CVE-2021-44228.json new file mode 100644 index 00000000..5dad8bea --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/epss/CVE-2021-44228.json @@ -0,0 +1 @@ +{"status":"OK","status-code":200,"version":"1.0","access":"public","total":1,"offset":0,"limit":100,"data":[{"cve":"CVE-2021-44228","epss":"0.999990000","percentile":"1.000000000","date":"2026-07-05"}]} diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/kev/kev-sample.json b/PROJECTS/intermediate/security-news-scraper/testdata/kev/kev-sample.json new file mode 100644 index 00000000..d0e9e4b1 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/kev/kev-sample.json @@ -0,0 +1,85 @@ +{ + "title": "CISA Catalog of Known Exploited Vulnerabilities", + "catalogVersion": "2026.07.01", + "dateReleased": "2026-07-01T19:00:06.9016Z", + "count": 5, + "vulnerabilities": [ + { + "cveID": "CVE-2021-44228", + "vendorProject": "Apache", + "product": "Log4j2", + "vulnerabilityName": "Apache Log4j2 Remote Code Execution Vulnerability", + "dateAdded": "2021-12-10", + "shortDescription": "Apache Log4j2 contains a vulnerability where JNDI features do not protect against attacker-controlled JNDI-related endpoints, allowing for remote code execution.", + "requiredAction": "For all affected software assets for which updates exist, the only acceptable remediation actions are: 1) Apply updates; OR 2) remove affected assets from agency networks. Temporary mitigations using one of the measures provided at https://www.cisa.gov/uscert/ed-22-02-apache-log4j-recommended-mitigation-measures are only acceptable until updates are available.", + "dueDate": "2021-12-24", + "knownRansomwareCampaignUse": "Known", + "notes": "https://nvd.nist.gov/vuln/detail/CVE-2021-44228", + "cwes": [ + "CWE-20", + "CWE-400", + "CWE-502" + ] + }, + { + "cveID": "CVE-2026-45659", + "vendorProject": "Microsoft", + "product": "SharePoint Server", + "vulnerabilityName": "Microsoft SharePoint Server Deserialization of Untrusted Data Vulnerability", + "dateAdded": "2026-07-01", + "shortDescription": "Microsoft SharePoint Server contains a deserialization of untrusted data vulnerability which allows an authorized attacker to execute code over a network.", + "requiredAction": "Apply mitigations in accordance with vendor instructions, ensuring compliance with CISA\u2019s BOD 26-04 Prioritizing Security Updates Based on Risk (see URL in Notes) guidance and CISA\u2019s \u201cForensics Triage Requirements\u201d (see URL in Notes). Follow applicable BOD 26-04 guidance for cloud services or discontinue use of the product if mitigations are unavailable. Stakeholders are responsible for evaluating each asset's internet exposure and ensuring adherence to BOD 26-04 patching guidelines.", + "dueDate": "2026-07-04", + "knownRansomwareCampaignUse": "Unknown", + "notes": "https://msrc.microsoft.com/update-guide/vulnerability/CVE-2026-45659 ; BOD 26-04: https://www.cisa.gov/news-events/directives/bod-26-04-prioritizing-security-updates-based-risk ; Forensics Triage Requirements: https://www.cisa.gov/news-events/directives/bod-26-04-implementation-guidance-prioritizing-security-updates-based-risk ; https://nvd.nist.gov/vuln/detail/CVE-2026-45659", + "cwes": [ + "CWE-502" + ] + }, + { + "cveID": "CVE-2026-48558", + "vendorProject": "SimpleHelp ", + "product": "SimpleHelp", + "vulnerabilityName": "SimpleHelp Authentication Bypass Vulnerability", + "dateAdded": "2026-06-29", + "shortDescription": "SimpleHelp contains an authentication bypass vulnerability in the OIDC authentication flow. When OIDC authentication is configured, identity tokens submitted during login are accepted without verifying their cryptographic signature. In a vulnerable configuration, a remote, unauthenticated attacker can submit a forged token containing arbitrary identity claims to obtain a fully authenticated technician session. In some configurations, this may also allow bypass of multi-factor authentication.", + "requiredAction": "Apply mitigations in accordance with vendor instructions, ensuring compliance with CISA\u2019s BOD 26-04 Prioritizing Security Updates Based on Risk (see URL in Notes) guidance and CISA\u2019s \u201cForensics Triage Requirements\u201d (see URL in Notes). Follow applicable BOD 26-04 guidance for cloud services or discontinue use of the product if mitigations are unavailable. Stakeholders are responsible for evaluating each asset's internet exposure and ensuring adherence to BOD 26-04 patching guidelines.", + "dueDate": "2026-07-02", + "knownRansomwareCampaignUse": "Unknown", + "notes": "https://simple-help.com/security/simplehelp-security-update-2026-05 ; BOD 26-04: https://www.cisa.gov/news-events/directives/bod-26-04-prioritizing-security-updates-based-risk ; Forensics Triage Requirements: https://www.cisa.gov/news-events/directives/bod-26-04-implementation-guidance-prioritizing-security-updates-based-risk ; https://nvd.nist.gov/vuln/detail/CVE-2026-48558", + "cwes": [ + "CWE-347" + ] + }, + { + "cveID": "CVE-2026-35273", + "vendorProject": "Oracle", + "product": " PeopleSoft Enterprise PeopleTools", + "vulnerabilityName": "Oracle PeopleSoft Enterprise PeopleTools Missing Authentication for Critical Function Vulnerability", + "dateAdded": "2026-06-12", + "shortDescription": "Oracle PeopleSoft Enterprise PeopleTools contains a missing authentication for critical function vulnerability which could allow an unauthenticated attacker to obtain takeover of PeopleSoft Enterprise PeopleTools.", + "requiredAction": "Apply mitigations in accordance with vendor instructions, ensuring compliance with CISA\u2019s BOD 26-04 Prioritizing Security Updates Based on Risk (see URL in Notes) guidance and CISA\u2019s \u201cForensics Triage Requirements\u201d (see URL in Notes). Follow applicable BOD 26-04 guidance for cloud services or discontinue use of the product if mitigations are unavailable. Stakeholders are responsible for evaluating each asset's internet exposure and ensuring adherence to BOD 26-04 patching guidelines.", + "dueDate": "2026-06-15", + "knownRansomwareCampaignUse": "Known", + "notes": "https://www.oracle.com/security-alerts/alert-cve-2026-35273.html ; https://support.oracle.com/signin/ ; BOD 26-04: https://www.cisa.gov/news-events/directives/bod-26-04-prioritizing-security-updates-based-risk ; Forensics Triage Requirements: https://www.cisa.gov/news-events/directives/bod-26-04-implementation-guidance-prioritizing-security-updates-based-risk ; https://nvd.nist.gov/vuln/detail/CVE-2026-35273", + "cwes": [ + "CWE-306" + ] + }, + { + "cveID": "CVE-2026-50751", + "vendorProject": "Check Point", + "product": "Security Gateway", + "vulnerabilityName": "Check Point Security Gateway Improper Authentication Vulnerability", + "dateAdded": "2026-06-08", + "shortDescription": "Check Point Security Gateway contains an improper authentication vulnerability in IKEv1 key exchange that could allow an unauthenticated remote attacker to bypass user authentication and establish a remote access VPN connection without a valid user password.", + "requiredAction": "Apply mitigations per vendor instructions, follow applicable BOD 22-01 guidance for cloud services, or discontinue use of the product if mitigations are unavailable.", + "dueDate": "2026-06-11", + "knownRansomwareCampaignUse": "Known", + "notes": "https://blog.checkpoint.com/security/check-point-releases-important-hotfix-for-vulnerabilities-in-deprecated-ikev1-vpn-protocol/ ; https://support.checkpoint.com/results/sk/sk185033?_gl=1*1wqeqhc*_gcl_au*MTI1MzE5MjI2LjE3ODA5MzQ1NTM. ; https://nvd.nist.gov/vuln/detail/CVE-2026-50751", + "cwes": [ + "CWE-287" + ] + } + ] +} diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/nvd/CVE-2021-44228.json b/PROJECTS/intermediate/security-news-scraper/testdata/nvd/CVE-2021-44228.json new file mode 100644 index 00000000..21ee2221 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/nvd/CVE-2021-44228.json @@ -0,0 +1 @@ +{"resultsPerPage":1,"startIndex":0,"totalResults":1,"format":"NVD_CVE","version":"2.0","timestamp":"2026-07-05T17:02:19.125","vulnerabilities":[{"cve":{"id":"CVE-2021-44228","sourceIdentifier":"security@apache.org","published":"2021-12-10T10:15:09.143","lastModified":"2026-06-17T04:12:05.460","vulnStatus":"Analyzed","cveTags":[],"descriptions":[{"lang":"en","value":"Apache Log4j2 2.0-beta9 through 2.15.0 (excluding security releases 2.12.2, 2.12.3, and 2.3.1) JNDI features used in configuration, log messages, and parameters do not protect against attacker controlled LDAP and other JNDI related endpoints. An attacker who can control log messages or log message parameters can execute arbitrary code loaded from LDAP servers when message lookup substitution is enabled. From log4j 2.15.0, this behavior has been disabled by default. From version 2.16.0 (along with 2.12.2, 2.12.3, and 2.3.1), this functionality has been completely removed. Note that this vulnerability is specific to log4j-core and does not affect log4net, log4cxx, or other Apache Logging Services projects."},{"lang":"es","value":"Las características JNDI de Apache Log4j2 2.0-beta9 hasta 2.15.0 (excluyendo las versiones de seguridad 2.12.2, 2.12.3 y 2.3.1) utilizadas en la configuración, los mensajes de registro y los parámetros no protegen contra LDAP controlado por un atacante y otros puntos finales relacionados con JNDI. Un atacante que pueda controlar los mensajes de registro o los parámetros de los mensajes de registro puede ejecutar código arbitrario cargado desde servidores LDAP cuando la sustitución de la búsqueda de mensajes está habilitada. A partir de la versión 2.15.0 de log4j, este comportamiento ha sido deshabilitado por defecto. A partir de la versión 2.16.0 (junto con las versiones 2.12.2, 2.12.3 y 2.3.1), esta funcionalidad se ha eliminado por completo. Tenga en cuenta que esta vulnerabilidad es específica de log4j-core y no afecta a log4net, log4cxx u otros proyectos de Apache Logging Services"}],"affected":[{"source":"security@apache.org","affectedData":[{"vendor":"Apache Software Foundation","product":"Apache Log4j2","versions":[{"version":"2.0-beta9","lessThan":"log4j-core*","versionType":"custom","status":"affected","changes":[{"at":"2.3.1","status":"unaffected"},{"at":"2.4","status":"affected"},{"at":"2.12.2","status":"unaffected"},{"at":"2.13.0","status":"affected"},{"at":"2.15.0","status":"unaffected"}]}]}]}],"metrics":{"cvssMetricV31":[{"source":"nvd@nist.gov","type":"Primary","cvssData":{"version":"3.1","vectorString":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H","baseScore":10.0,"baseSeverity":"CRITICAL","attackVector":"NETWORK","attackComplexity":"LOW","privilegesRequired":"NONE","userInteraction":"NONE","scope":"CHANGED","confidentialityImpact":"HIGH","integrityImpact":"HIGH","availabilityImpact":"HIGH"},"exploitabilityScore":3.9,"impactScore":6.0},{"source":"134c704f-9b21-4f2e-91b3-4a467353bcc0","type":"Secondary","cvssData":{"version":"3.1","vectorString":"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H","baseScore":10.0,"baseSeverity":"CRITICAL","attackVector":"NETWORK","attackComplexity":"LOW","privilegesRequired":"NONE","userInteraction":"NONE","scope":"CHANGED","confidentialityImpact":"HIGH","integrityImpact":"HIGH","availabilityImpact":"HIGH"},"exploitabilityScore":3.9,"impactScore":6.0}],"cvssMetricV2":[{"source":"nvd@nist.gov","type":"Primary","cvssData":{"version":"2.0","vectorString":"AV:N/AC:M/Au:N/C:C/I:C/A:C","baseScore":9.3,"accessVector":"NETWORK","accessComplexity":"MEDIUM","authentication":"NONE","confidentialityImpact":"COMPLETE","integrityImpact":"COMPLETE","availabilityImpact":"COMPLETE"},"baseSeverity":"HIGH","exploitabilityScore":8.6,"impactScore":10.0,"acInsufInfo":false,"obtainAllPrivilege":false,"obtainUserPrivilege":false,"obtainOtherPrivilege":false,"userInteractionRequired":false}],"ssvcV203":[{"source":"134c704f-9b21-4f2e-91b3-4a467353bcc0","ssvcData":{"timestamp":"2025-02-04T14:25:34.416117Z","id":"CVE-2021-44228","options":[{"exploitation":"active"},{"automatable":"yes"},{"technicalImpact":"total"}],"role":"CISA Coordinator","version":"2.0.3"}}]},"cisaExploitAdd":"2021-12-10","cisaActionDue":"2021-12-24","cisaRequiredAction":"For all affected software assets for which updates exist, the only acceptable remediation actions are: 1) Apply updates; OR 2) remove affected assets from agency networks. Temporary mitigations using one of the measures provided at https://www.cisa.gov/uscert/ed-22-02-apache-log4j-recommended-mitigation-measures are only acceptable until updates are available.","cisaVulnerabilityName":"Apache Log4j2 Remote Code Execution Vulnerability","weaknesses":[{"source":"security@apache.org","type":"Secondary","description":[{"lang":"en","value":"CWE-20"},{"lang":"en","value":"CWE-400"},{"lang":"en","value":"CWE-502"}]},{"source":"nvd@nist.gov","type":"Secondary","description":[{"lang":"en","value":"CWE-917"}]}],"configurations":[{"operator":"AND","nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:o:siemens:6bk1602-0aa12-0tp0_firmware:*:*:*:*:*:*:*:*","versionEndExcluding":"2.7.0","matchCriteriaId":"BD64FC36-CC7B-4FD7-9845-7EA1DDB0E627"}]},{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":false,"criteria":"cpe:2.3:h:siemens:6bk1602-0aa12-0tp0:-:*:*:*:*:*:*:*","matchCriteriaId":"CF99FE8F-40D0-48A8-9A40-43119B259535"}]}]},{"operator":"AND","nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:o:siemens:6bk1602-0aa22-0tp0_firmware:*:*:*:*:*:*:*:*","versionEndExcluding":"2.7.0","matchCriteriaId":"D0012304-B1C8-460A-B891-42EBF96504F5"}]},{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":false,"criteria":"cpe:2.3:h:siemens:6bk1602-0aa22-0tp0:-:*:*:*:*:*:*:*","matchCriteriaId":"F3F61BCB-64FA-463C-8B95-8868995EDBC0"}]}]},{"operator":"AND","nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:o:siemens:6bk1602-0aa32-0tp0_firmware:*:*:*:*:*:*:*:*","versionEndExcluding":"2.7.0","matchCriteriaId":"B02BCF56-D9D3-4BF3-85A2-D445E997F5EC"}]},{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":false,"criteria":"cpe:2.3:h:siemens:6bk1602-0aa32-0tp0:-:*:*:*:*:*:*:*","matchCriteriaId":"B5A189B7-DDBF-4B84-997F-637CEC5FF12B"}]}]},{"operator":"AND","nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:o:siemens:6bk1602-0aa42-0tp0_firmware:*:*:*:*:*:*:*:*","versionEndExcluding":"2.7.0","matchCriteriaId":"4A2DB5BA-1065-467A-8FB6-81B5EC29DC0C"}]},{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":false,"criteria":"cpe:2.3:h:siemens:6bk1602-0aa42-0tp0:-:*:*:*:*:*:*:*","matchCriteriaId":"035AFD6F-E560-43C8-A283-8D80DAA33025"}]}]},{"operator":"AND","nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:o:siemens:6bk1602-0aa52-0tp0_firmware:*:*:*:*:*:*:*:*","versionEndExcluding":"2.7.0","matchCriteriaId":"809EB87E-561A-4DE5-9FF3-BBEE0FA3706E"}]},{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":false,"criteria":"cpe:2.3:h:siemens:6bk1602-0aa52-0tp0:-:*:*:*:*:*:*:*","matchCriteriaId":"4594FF76-A1F8-4457-AE90-07D051CD0DCB"}]}]},{"nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*","versionStartIncluding":"2.0.1","versionEndExcluding":"2.3.1","matchCriteriaId":"03FA5E81-F9C0-403E-8A4B-E4284E4E7B72"},{"vulnerable":true,"criteria":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*","versionStartIncluding":"2.4.0","versionEndExcluding":"2.12.2","matchCriteriaId":"AED3D5EC-DAD5-4E5F-8BBD-B4E3349D84FC"},{"vulnerable":true,"criteria":"cpe:2.3:a:apache:log4j:*:*:*:*:*:*:*:*","versionStartIncluding":"2.13.0","versionEndExcluding":"2.15.0","matchCriteriaId":"D31D423D-FC4D-428A-B863-55AF472B80DC"},{"vulnerable":true,"criteria":"cpe:2.3:a:apache:log4j:2.0:-:*:*:*:*:*:*","matchCriteriaId":"17854E42-7063-4A55-BF2A-4C7074CC2D60"},{"vulnerable":true,"criteria":"cpe:2.3:a:apache:log4j:2.0:beta9:*:*:*:*:*:*","matchCriteriaId":"53F32FB2-6970-4975-8BD0-EAE12E9AD03A"},{"vulnerable":true,"criteria":"cpe:2.3:a:apache:log4j:2.0:rc1:*:*:*:*:*:*","matchCriteriaId":"B773ED91-1D39-42E6-9C52-D02210DE1A94"},{"vulnerable":true,"criteria":"cpe:2.3:a:apache:log4j:2.0:rc2:*:*:*:*:*:*","matchCriteriaId":"EF24312D-1A62-482E-8078-7EC24758B710"}]}]},{"operator":"AND","nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:o:siemens:sppa-t3000_ses3000_firmware:*:*:*:*:*:*:*:*","matchCriteriaId":"E8320869-CBF4-4C92-885C-560C09855BFA"}]},{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":false,"criteria":"cpe:2.3:h:siemens:sppa-t3000_ses3000:-:*:*:*:*:*:*:*","matchCriteriaId":"755BA221-33DD-40A2-A517-8574D042C261"}]}]},{"nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:capital:*:*:*:*:*:*:*:*","versionEndExcluding":"2019.1","matchCriteriaId":"9AAF12D5-7961-4344-B0CC-BE1C673BFE1F"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:capital:2019.1:-:*:*:*:*:*:*","matchCriteriaId":"19CB7B44-1877-4739-AECB-3E995ED03FC9"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:capital:2019.1:sp1912:*:*:*:*:*:*","matchCriteriaId":"A883D9C2-F2A4-459F-8000-EE288DC0DD17"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:comos:*:*:*:*:*:*:*:*","versionEndExcluding":"10.4.2","matchCriteriaId":"9CD4AC6F-B8D3-4588-B3BD-55C9BAF4AAAC"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:desigo_cc_advanced_reports:3.0:*:*:*:*:*:*:*","matchCriteriaId":"8AFD64AC-0826-48FB-91B0-B8DF5ECC8775"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:desigo_cc_advanced_reports:4.0:*:*:*:*:*:*:*","matchCriteriaId":"BB524B33-68E7-46A2-B5CE-BCD9C3194B8B"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:desigo_cc_advanced_reports:4.1:*:*:*:*:*:*:*","matchCriteriaId":"5F852C6D-44A0-4CCE-83C7-4501CAD73F9F"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:desigo_cc_advanced_reports:4.2:*:*:*:*:*:*:*","matchCriteriaId":"AA61161C-C2E7-4852-963E-E2D3DFBFDC7B"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:desigo_cc_advanced_reports:5.0:*:*:*:*:*:*:*","matchCriteriaId":"A76AA04A-BB43-4027-895E-D1EACFCDF41B"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:desigo_cc_advanced_reports:5.1:*:*:*:*:*:*:*","matchCriteriaId":"2A6B60F3-327B-49B7-B5E4-F1C60896C9BB"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:desigo_cc_info_center:5.0:*:*:*:*:*:*:*","matchCriteriaId":"4BCF281E-B0A2-49E2-AEF8-8691BDCE08D5"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:desigo_cc_info_center:5.1:*:*:*:*:*:*:*","matchCriteriaId":"A87EFCC4-4BC1-4FEA-BAA4-8FF221838EBD"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:e-car_operation_center:*:*:*:*:*:*:*:*","versionEndExcluding":"2021-12-13","matchCriteriaId":"B678380B-E95E-4A8B-A49D-D13B62AA454E"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:energy_engage:3.1:*:*:*:*:*:*:*","matchCriteriaId":"4557476B-0157-44C2-BB50-299E7C7E1E72"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:energyip:8.5:*:*:*:*:*:*:*","matchCriteriaId":"991B2959-5AA3-4B68-A05A-42D9860FAA9D"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:energyip:8.6:*:*:*:*:*:*:*","matchCriteriaId":"7E5948A0-CA31-41DF-85B6-1E6D09E5720B"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:energyip:8.7:*:*:*:*:*:*:*","matchCriteriaId":"4C08D302-EEAC-45AA-9943-3A5F09E29FAB"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:energyip:9.0:*:*:*:*:*:*:*","matchCriteriaId":"D53BA68C-B653-4507-9A2F-177CF456960F"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:energyip_prepay:*:*:*:*:*:*:*:*","versionEndExcluding":"3.8.0.12","matchCriteriaId":"536C7527-27E6-41C9-8ED8-564DD0DC4EA0"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:gma-manager:*:*:*:*:*:*:*:*","versionEndExcluding":"8.6.2j-398","matchCriteriaId":"0E180527-5C36-4158-B017-5BEDC0412FD6"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:head-end_system_universal_device_integration_system:*:*:*:*:*:*:*:*","matchCriteriaId":"AFDADA98-1CD0-45DA-9082-BFC383F7DB97"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:industrial_edge_management:*:*:*:*:*:*:*:*","matchCriteriaId":"E33D707F-100E-4DE7-A05B-42467DE75EAC"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:industrial_edge_management_hub:*:*:*:*:*:*:*:*","versionEndExcluding":"2021-12-13","matchCriteriaId":"DD3EAC80-44BE-41D2-8D57-0EE3DBA1E1B1"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:logo\\!_soft_comfort:*:*:*:*:*:*:*:*","matchCriteriaId":"2AC8AB52-F4F4-440D-84F5-2776BFE1957A"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:mendix:*:*:*:*:*:*:*:*","matchCriteriaId":"6AF6D774-AC8C-49CA-A00B-A2740CA8FA91"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:mindsphere:*:*:*:*:*:*:*:*","versionEndExcluding":"2021-12-16","matchCriteriaId":"25FADB1B-988D-4DB9-9138-7542AFDEB672"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:navigator:*:*:*:*:*:*:*:*","versionEndExcluding":"2021-12-13","matchCriteriaId":"48C6A61B-2198-4B9E-8BCF-824643C81EC3"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:nx:*:*:*:*:*:*:*:*","matchCriteriaId":"BEE2F7A1-8281-48F1-8BFB-4FE0D7E1AEF4"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:opcenter_intelligence:*:*:*:*:*:*:*:*","versionStartIncluding":"3.2","versionEndExcluding":"3.5","matchCriteriaId":"C07AFA19-21AE-4C7E-AA95-69599834C0EC"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:operation_scheduler:*:*:*:*:*:*:*:*","versionEndIncluding":"1.1.3","matchCriteriaId":"74D1F4AD-9A60-4432-864F-4505B3C60659"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:sentron_powermanager:4.1:*:*:*:*:*:*:*","matchCriteriaId":"7ABA5332-8D1E-4129-A557-FCECBAC12827"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:sentron_powermanager:4.2:*:*:*:*:*:*:*","matchCriteriaId":"9C3AA865-5570-4C8B-99DE-431AD7B163F1"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:siguard_dsa:*:*:*:*:*:*:*:*","versionStartIncluding":"4.2","versionEndExcluding":"4.4.1","matchCriteriaId":"9A4B950B-4527-491B-B111-046DB1CCC037"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:sipass_integrated:2.80:*:*:*:*:*:*:*","matchCriteriaId":"83E77D85-0AE8-41D6-AC0C-983A8B73C831"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:sipass_integrated:2.85:*:*:*:*:*:*:*","matchCriteriaId":"02B28A44-3708-480D-9D6D-DDF8C21A15EC"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:siveillance_command:*:*:*:*:*:*:*:*","versionEndIncluding":"4.16.2.1","matchCriteriaId":"2FC0A575-F771-4B44-A0C6-6A5FD98E5134"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:siveillance_control_pro:*:*:*:*:*:*:*:*","matchCriteriaId":"6D1D6B61-1F17-4008-9DFB-EF419777768E"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:siveillance_identity:1.5:*:*:*:*:*:*:*","matchCriteriaId":"9772EE3F-FFC5-4611-AD9A-8AD8304291BB"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:siveillance_identity:1.6:*:*:*:*:*:*:*","matchCriteriaId":"CF524892-278F-4373-A8A3-02A30FA1AFF4"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:siveillance_vantage:*:*:*:*:*:*:*:*","matchCriteriaId":"F30DE588-9479-46AA-8346-EA433EE83A5F"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:siveillance_viewpoint:*:*:*:*:*:*:*:*","matchCriteriaId":"4941EAD6-8759-4C72-ABA6-259C0E838216"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:solid_edge_cam_pro:*:*:*:*:*:*:*:*","matchCriteriaId":"5BF2708F-0BD9-41BF-8CB1-4D06C4EFB777"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:solid_edge_harness_design:*:*:*:*:*:*:*:*","versionEndExcluding":"2020","matchCriteriaId":"0762031C-DFF1-4962-AE05-0778B27324B9"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:solid_edge_harness_design:2020:*:*:*:*:*:*:*","matchCriteriaId":"96271088-1D1B-4378-8ABF-11DAB3BB4DDC"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:solid_edge_harness_design:2020:-:*:*:*:*:*:*","matchCriteriaId":"2595AD24-2DF2-4080-B780-BC03F810B9A9"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:solid_edge_harness_design:2020:sp2002:*:*:*:*:*:*","matchCriteriaId":"88096F08-F261-4E3E-9EEB-2AB0225CD6F3"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:spectrum_power_4:*:*:*:*:*:*:*:*","versionEndExcluding":"4.70","matchCriteriaId":"044994F7-8127-4F03-AA1A-B2AB41D68AF5"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:spectrum_power_4:4.70:-:*:*:*:*:*:*","matchCriteriaId":"A6CB3A8D-9577-41FB-8AC4-0DF8DE6A519C"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:spectrum_power_4:4.70:sp7:*:*:*:*:*:*","matchCriteriaId":"17B7C211-6339-4AF2-9564-94C7DE52EEB7"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:spectrum_power_4:4.70:sp8:*:*:*:*:*:*","matchCriteriaId":"DBCCBBBA-9A4F-4354-91EE-10A1460BBA3F"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:spectrum_power_7:*:*:*:*:*:*:*:*","versionEndExcluding":"2.30","matchCriteriaId":"12F81F6B-E455-4367-ADA4-8A5EC7F4754A"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:spectrum_power_7:2.30:*:*:*:*:*:*:*","matchCriteriaId":"A5EF509E-3799-4718-B361-EFCBA17AEEF3"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:spectrum_power_7:2.30:-:*:*:*:*:*:*","matchCriteriaId":"8CA31645-29FC-4432-9BFC-C98A808DB8CF"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:spectrum_power_7:2.30:sp2:*:*:*:*:*:*","matchCriteriaId":"BB424991-0B18-4FFC-965F-FCF4275F56C5"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:teamcenter:*:*:*:*:*:*:*:*","matchCriteriaId":"1B209EFE-77F2-48CD-A880-ABA0A0A81AB1"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:vesys:*:*:*:*:*:*:*:*","versionEndExcluding":"2019.1","matchCriteriaId":"72D238AB-4A1F-458D-897E-2C93DCD7BA6C"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:vesys:2019.1:*:*:*:*:*:*:*","matchCriteriaId":"9778339A-EA93-4D18-9A03-4EB4CBD25459"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:vesys:2019.1:-:*:*:*:*:*:*","matchCriteriaId":"1747F127-AB45-4325-B9A1-F3D12E69FFC8"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:vesys:2019.1:sp1912:*:*:*:*:*:*","matchCriteriaId":"18BBEF7C-F686-4129-8EE9-0F285CE38845"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:vesys:2020.1:-:*:*:*:*:*:*","matchCriteriaId":"264C7817-0CD5-4370-BC39-E1DF3E932E16"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:vesys:2021.1:-:*:*:*:*:*:*","matchCriteriaId":"C7442C42-D493-46B9-BCC2-2C62EAD5B945"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:xpedition_enterprise:-:*:*:*:*:*:*:*","matchCriteriaId":"AD525494-2807-48EA-AED0-11B9CB5A6A9B"},{"vulnerable":true,"criteria":"cpe:2.3:a:siemens:xpedition_package_integrator:-:*:*:*:*:*:*:*","matchCriteriaId":"1EDCBF98-A857-48BC-B04D-6F36A1975AA5"}]}]},{"nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:a:intel:computer_vision_annotation_tool:-:*:*:*:*:*:*:*","matchCriteriaId":"12A06BF8-E4DC-4389-8A91-8AC7598E0009"},{"vulnerable":true,"criteria":"cpe:2.3:a:intel:datacenter_manager:*:*:*:*:*:*:*:*","versionEndExcluding":"5.1","matchCriteriaId":"EAD1E1F3-F06B-4D17-8854-2CDA7E6D872D"},{"vulnerable":true,"criteria":"cpe:2.3:a:intel:genomics_kernel_library:-:*:*:*:*:*:*:*","matchCriteriaId":"18989EBC-E1FB-473B-83E0-48C8896C2E96"},{"vulnerable":true,"criteria":"cpe:2.3:a:intel:oneapi_sample_browser:-:*:*:*:*:eclipse:*:*","matchCriteriaId":"EDE66B6C-25E5-49AE-B35F-582130502222"},{"vulnerable":true,"criteria":"cpe:2.3:a:intel:secure_device_onboard:-:*:*:*:*:*:*:*","matchCriteriaId":"22BEE177-D117-478C-8EAD-9606DEDF9FD5"},{"vulnerable":true,"criteria":"cpe:2.3:a:intel:system_studio:-:*:*:*:*:*:*:*","matchCriteriaId":"FC619106-991C-413A-809D-C2410EBA4CDB"}]}]},{"nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:o:debian:debian_linux:9.0:*:*:*:*:*:*:*","matchCriteriaId":"DEECE5FC-CACF-4496-A3E7-164736409252"},{"vulnerable":true,"criteria":"cpe:2.3:o:debian:debian_linux:10.0:*:*:*:*:*:*:*","matchCriteriaId":"07B237A9-69A3-4A9C-9DA0-4E06BD37AE73"},{"vulnerable":true,"criteria":"cpe:2.3:o:debian:debian_linux:11.0:*:*:*:*:*:*:*","matchCriteriaId":"FA6FEEC2-9F11-4643-8827-749718254FED"}]}]},{"nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:o:fedoraproject:fedora:34:*:*:*:*:*:*:*","matchCriteriaId":"A930E247-0B43-43CB-98FF-6CE7B8189835"},{"vulnerable":true,"criteria":"cpe:2.3:o:fedoraproject:fedora:35:*:*:*:*:*:*:*","matchCriteriaId":"80E516C0-98A4-4ADE-B69F-66A772E2BAAA"}]}]},{"nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:a:sonicwall:email_security:*:*:*:*:*:*:*:*","versionEndExcluding":"10.0.13","matchCriteriaId":"CA7D45EF-18F7-43C6-9B51-ABAB7B0CA3CD"}]}]},{"nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:a:netapp:active_iq_unified_manager:-:*:*:*:*:linux:*:*","matchCriteriaId":"F3E0B672-3E06-4422-B2A4-0BD073AEC2A1"},{"vulnerable":true,"criteria":"cpe:2.3:a:netapp:active_iq_unified_manager:-:*:*:*:*:vmware_vsphere:*:*","matchCriteriaId":"3A756737-1CC4-42C2-A4DF-E1C893B4E2D5"},{"vulnerable":true,"criteria":"cpe:2.3:a:netapp:active_iq_unified_manager:-:*:*:*:*:windows:*:*","matchCriteriaId":"B55E8D50-99B4-47EC-86F9-699B67D473CE"},{"vulnerable":true,"criteria":"cpe:2.3:a:netapp:brocade_san_navigator:-:*:*:*:*:*:*:*","matchCriteriaId":"25FA7A4D-B0E2-423E-8146-E221AE2D6120"},{"vulnerable":true,"criteria":"cpe:2.3:a:netapp:cloud_insights:-:*:*:*:*:*:*:*","matchCriteriaId":"26FCA75B-4282-4E0F-95B4-640A82C8E91C"},{"vulnerable":true,"criteria":"cpe:2.3:a:netapp:cloud_manager:-:*:*:*:*:*:*:*","matchCriteriaId":"197D0D80-6702-4B61-B681-AFDBA7D69067"},{"vulnerable":true,"criteria":"cpe:2.3:a:netapp:cloud_secure_agent:-:*:*:*:*:*:*:*","matchCriteriaId":"F0F202E8-97E6-4BBB-A0B6-4CA3F5803C08"},{"vulnerable":true,"criteria":"cpe:2.3:a:netapp:oncommand_insight:-:*:*:*:*:*:*:*","matchCriteriaId":"F1BE6C1F-2565-4E97-92AA-16563E5660A5"},{"vulnerable":true,"criteria":"cpe:2.3:a:netapp:ontap_tools:-:*:*:*:*:vmware_vsphere:*:*","matchCriteriaId":"CBCC384C-5DF0-41AB-B17B-6E9B6CAE8065"},{"vulnerable":true,"criteria":"cpe:2.3:a:netapp:snapcenter:-:*:*:*:*:vmware_vsphere:*:*","matchCriteriaId":"F3A48D58-4291-4D3C-9CEA-BF12183468A7"},{"vulnerable":true,"criteria":"cpe:2.3:a:netapp:solidfire_\\&_hci_storage_node:-:*:*:*:*:*:*:*","matchCriteriaId":"D452B464-1200-4B72-9A89-42DC58486191"},{"vulnerable":true,"criteria":"cpe:2.3:a:netapp:solidfire_enterprise_sds:-:*:*:*:*:*:*:*","matchCriteriaId":"5D18075A-E8D6-48B8-A7FA-54E336A434A2"}]}]},{"nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:advanced_malware_protection_virtual_private_cloud_appliance:*:*:*:*:*:*:*:*","versionEndExcluding":"3.5.4","matchCriteriaId":"4E52AF19-0158-451B-8E36-02CB6406083F"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:automated_subsea_tuning:*:*:*:*:*:*:*:*","versionEndExcluding":"2.1.0","matchCriteriaId":"CB21CFB4-4492-4C5D-BD07-FFBE8B5D92B6"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:broadworks:*:*:*:*:*:*:*:*","versionEndExcluding":"2021.11_1.162","matchCriteriaId":"97426511-9B48-46F5-AC5C-F9781F1BAE2F"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:business_process_automation:*:*:*:*:*:*:*:*","versionEndExcluding":"3.0.000.115","matchCriteriaId":"82306B9F-AE97-4E29-A8F7-2E5BA52998A7"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:business_process_automation:*:*:*:*:*:*:*:*","versionStartIncluding":"3.1.000.000","versionEndExcluding":"3.1.000.044","matchCriteriaId":"4C903C85-DC0F-47D8-B8BE-7A666877B017"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:business_process_automation:*:*:*:*:*:*:*:*","versionStartIncluding":"3.2.000.000","versionEndExcluding":"3.2.000.009","matchCriteriaId":"E4C6F9E0-5DCE-431D-AE7E-B680AC1F9332"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:cloud_connect:*:*:*:*:*:*:*:*","versionEndExcluding":"12.6\\(1\\)","matchCriteriaId":"52CF6199-8028-4076-952B-855984F30129"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:cloudcenter:*:*:*:*:*:*:*:*","versionEndExcluding":"4.10.0.16","matchCriteriaId":"622BB8D9-AC81-4C0F-A5C5-C5E51F0BC0D1"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:cloudcenter_cost_optimizer:*:*:*:*:*:*:*:*","versionEndExcluding":"5.5.2","matchCriteriaId":"38FB3CE1-5F62-4798-A825-4E3DB07E868F"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:cloudcenter_suite_admin:*:*:*:*:*:*:*:*","versionEndExcluding":"5.3.1","matchCriteriaId":"29CDB878-B085-448E-AB84-25B1E2D024F8"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:cloudcenter_workload_manager:*:*:*:*:*:*:*:*","versionEndExcluding":"5.5.2","matchCriteriaId":"C25FDA96-9490-431F-B8B6-CC2CC272670E"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:common_services_platform_collector:*:*:*:*:*:*:*:*","versionEndExcluding":"2.9.1.3","matchCriteriaId":"51CD9E4C-9385-435C-AD18-6C36C8DF7B65"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:common_services_platform_collector:*:*:*:*:*:*:*:*","versionStartIncluding":"2.10.0","versionEndExcluding":"2.10.0.1","matchCriteriaId":"FC0AC4C1-CB06-4084-BFBB-5B702C384C53"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:connected_mobile_experiences:-:*:*:*:*:*:*:*","matchCriteriaId":"3871EBD2-F270-435A-B98C-A282E1C52693"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:contact_center_domain_manager:*:*:*:*:*:*:*:*","versionEndExcluding":"12.5\\(1\\)","matchCriteriaId":"8D4DF34B-E8C2-41C8-90E2-D119B50E4E7E"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:contact_center_management_portal:*:*:*:*:*:*:*:*","versionEndExcluding":"12.5\\(1\\)","matchCriteriaId":"C8EF64DA-73E4-4E5E-8F9A-B837C947722E"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_data_gateway:*:*:*:*:*:*:*:*","versionEndExcluding":"2.0.2","matchCriteriaId":"66E1E4FC-0B6E-4CFA-B003-91912F8785B2"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_data_gateway:3.0.0:*:*:*:*:*:*:*","matchCriteriaId":"1B2390C3-C319-4F05-8CF0-0D30F9931507"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_network_controller:*:*:*:*:*:*:*:*","versionEndExcluding":"2.0.1","matchCriteriaId":"C154491E-06C7-48B0-AC1D-89BBDBDB902E"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_network_controller:3.0.0:*:*:*:*:*:*:*","matchCriteriaId":"1E98EC48-0CED-4E02-9CCB-06EF751F2BDC"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_optimization_engine:*:*:*:*:*:*:*:*","versionEndExcluding":"2.0.1","matchCriteriaId":"C569DC2A-CFF6-4E13-A50C-E215A4F96D99"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_optimization_engine:3.0.0:*:*:*:*:*:*:*","matchCriteriaId":"258A51AC-6649-4F67-A842-48A7AE4DCEE1"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_platform_infrastructure:*:*:*:*:*:*:*:*","versionEndExcluding":"4.0.1","matchCriteriaId":"8DC22505-DE11-4A1B-8C06-1E306419B031"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_platform_infrastructure:4.1.0:*:*:*:*:*:*:*","matchCriteriaId":"9E31AC54-B928-48B5-8293-F5F4A7A8C293"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_zero_touch_provisioning:*:*:*:*:*:*:*:*","versionEndExcluding":"2.0.1","matchCriteriaId":"5B8AE870-6FD0-40D2-958B-548E2D7A7B75"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_zero_touch_provisioning:3.0.0:*:*:*:*:*:*:*","matchCriteriaId":"68E7D83B-B6AC-45B1-89A4-D18D7A6018DD"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:customer_experience_cloud_agent:*:*:*:*:*:*:*:*","versionEndExcluding":"1.12.1","matchCriteriaId":"17660B09-47AA-42A2-B5FF-8EBD8091C661"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:cyber_vision_sensor_management_extension:*:*:*:*:*:*:*:*","versionEndExcluding":"4.0.3","matchCriteriaId":"FBEF9A82-16AE-437A-B8CF-CC7E9B6C4E44"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:data_center_network_manager:*:*:*:*:*:*:*:*","versionEndExcluding":"11.3\\(1\\)","matchCriteriaId":"843147AE-8117-4FE9-AE74-4E1646D55642"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:data_center_network_manager:11.3\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"7EB871C9-CA14-4829-AED3-CC2B35E99E92"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:dna_center:*:*:*:*:*:*:*:*","versionEndExcluding":"2.1.2.8","matchCriteriaId":"4FF8A83D-A282-4661-B133-213A8838FB27"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:dna_center:*:*:*:*:*:*:*:*","versionStartIncluding":"2.2.2.0","versionEndExcluding":"2.2.2.8","matchCriteriaId":"139CDAA5-63E9-4E56-AF72-745BD88E4B49"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:dna_center:*:*:*:*:*:*:*:*","versionStartIncluding":"2.2.3.0","versionEndExcluding":"2.2.3.4","matchCriteriaId":"01FD99C4-BCB1-417E-ADCE-73314AD2E857"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:dna_spaces\\:_connector:*:*:*:*:*:*:*:*","versionEndExcluding":"2.5","matchCriteriaId":"9031BE8A-646A-4581-BDE5-750FB0CE04CB"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:emergency_responder:*:*:*:*:*:*:*:*","versionEndExcluding":"11.5\\(4\\)","matchCriteriaId":"15BED3E2-46FF-4E58-8C5D-4D8FE5B0E527"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:enterprise_chat_and_email:*:*:*:*:*:*:*:*","versionEndExcluding":"12.0\\(1\\)","matchCriteriaId":"7C950436-2372-4C4B-9B56-9CB48D843045"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:evolved_programmable_network_manager:*:*:*:*:*:*:*:*","versionEndIncluding":"4.1.1","matchCriteriaId":"0B61F186-D943-4711-B3E0-875BB570B142"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:finesse:*:*:*:*:*:*:*:*","versionEndExcluding":"12.6\\(1\\)","matchCriteriaId":"2A285C40-170D-4C95-8031-2C6E4D5FB1D4"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:finesse:12.6\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"3C0F02B5-AA2A-48B2-AE43-38B45532C563"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:fog_director:-:*:*:*:*:*:*:*","matchCriteriaId":"830BDB28-963F-46C3-8D50-638FDABE7F64"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:identity_services_engine:*:*:*:*:*:*:*:*","versionEndExcluding":"2.4.0","matchCriteriaId":"54553C65-6BFA-40B1-958D-A4E3289D6B1D"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:identity_services_engine:2.4.0:-:*:*:*:*:*:*","matchCriteriaId":"439948AD-C95D-4FC3-ADD1-C3D241529F12"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:integrated_management_controller_supervisor:*:*:*:*:*:*:*:*","versionEndExcluding":"2.3.2.1","matchCriteriaId":"9C2002AE-0F3C-4A06-9B9A-F77A9F700EB2"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:intersight_virtual_appliance:*:*:*:*:*:*:*:*","versionEndExcluding":"1.0.9-361","matchCriteriaId":"596A986D-E7DC-4FC4-A776-6FE87A91D7E4"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:iot_operations_dashboard:-:*:*:*:*:*:*:*","matchCriteriaId":"DD93434E-8E75-469C-B12B-7E2B6EDCAA79"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_assurance_engine:*:*:*:*:*:*:*:*","versionEndExcluding":"6.0.2","matchCriteriaId":"78684844-4974-41AD-BBC1-961F60025CD2"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_services_orchestrator:*:*:*:*:*:*:*:*","versionEndExcluding":"5.3.5.1","matchCriteriaId":"3A00D235-FC9C-4EB7-A16C-BB0B09802E61"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_services_orchestrator:*:*:*:*:*:*:*:*","versionStartIncluding":"5.4","versionEndExcluding":"5.4.5.2","matchCriteriaId":"C60FDD1B-898E-4FCB-BDE2-45A7CBDBAF4F"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_services_orchestrator:*:*:*:*:*:*:*:*","versionStartIncluding":"5.5","versionEndExcluding":"5.5.4.1","matchCriteriaId":"E7A33E5F-BBC7-4917-9C63-900248B546D9"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_services_orchestrator:*:*:*:*:*:*:*:*","versionStartIncluding":"5.6","versionEndExcluding":"5.6.3.1","matchCriteriaId":"12D98A7C-4992-4E58-A6BD-3D8173C8F2B0"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:nexus_dashboard:*:*:*:*:*:*:*:*","versionEndExcluding":"2.1.2","matchCriteriaId":"E2DDC1AF-31B5-4F05-B84F-8FD23BE163DA"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:nexus_insights:*:*:*:*:*:*:*:*","versionEndExcluding":"6.0.2","matchCriteriaId":"A4540CF6-D33E-4D33-8608-11129D6591FA"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:optical_network_controller:*:*:*:*:*:*:*:*","versionEndExcluding":"1.1.0","matchCriteriaId":"129A7615-99E7-41F8-8EBC-CEDA10AD89AD"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:packaged_contact_center_enterprise:*:*:*:*:*:*:*:*","versionEndExcluding":"11.6","matchCriteriaId":"5F46A7AC-C133-442D-984B-BA278951D0BF"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:packaged_contact_center_enterprise:11.6\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"A1A75AB6-C3A7-4299-B35A-46A4BCD00816"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:paging_server:*:*:*:*:*:*:*:*","versionEndExcluding":"14.4.1","matchCriteriaId":"0A73E888-C8C2-4AFD-BA60-566D45214BCA"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:prime_service_catalog:*:*:*:*:*:*:*:*","versionEndExcluding":"12.1","matchCriteriaId":"4B0D0FD0-ABC6-465F-AB8D-FA8788B1B2DD"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:sd-wan_vmanage:*:*:*:*:*:*:*:*","versionEndExcluding":"20.3.4.1","matchCriteriaId":"D673F6F7-C42A-4538-96F0-34CB4F0CB080"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:sd-wan_vmanage:*:*:*:*:*:*:*:*","versionStartIncluding":"20.4","versionEndExcluding":"20.4.2.1","matchCriteriaId":"FD374819-3CED-4260-90B6-E3C1333EAAD2"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:sd-wan_vmanage:*:*:*:*:*:*:*:*","versionStartIncluding":"20.5","versionEndExcluding":"20.5.1.1","matchCriteriaId":"D2D89973-94AF-4BE7-8245-275F3FEB30F4"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:sd-wan_vmanage:*:*:*:*:*:*:*:*","versionStartIncluding":"20.6","versionEndExcluding":"20.6.2.1","matchCriteriaId":"91A9A889-2C2B-4147-8108-C35291761C15"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:smart_phy:*:*:*:*:*:*:*:*","versionEndExcluding":"3.2.1","matchCriteriaId":"D0EEA1EC-C63C-4C7D-BFAE-BA4556332242"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:ucs_central:*:*:*:*:*:*:*:*","versionEndExcluding":"2.0\\(1p\\)","matchCriteriaId":"ACE22D97-42FA-4179-99E5-C2EE582DB7FF"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:ucs_director:*:*:*:*:*:*:*:*","versionEndExcluding":"6.8.2.0","matchCriteriaId":"F6B5DB6D-9E7D-4403-8028-D7DA7493716B"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager:*:*:*:*:-:*:*:*","versionEndExcluding":"11.5\\(1\\)","matchCriteriaId":"B98D7AD5-0590-43FB-8AC0-376C9C500C15"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager:*:*:*:*:session_management:*:*:*","versionEndExcluding":"11.5\\(1\\)","matchCriteriaId":"D9DA1900-9972-4DFD-BE2E-74DABA1ED9A9"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager:11.5\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"42A41C41-A370-4C0E-A49D-AD42B2F3FB5C"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager:11.5\\(1\\):*:*:*:-:*:*:*","matchCriteriaId":"7E958AFF-185D-4D55-B74B-485BEAEC42FD"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager:11.5\\(1\\):*:*:*:session_management:*:*:*","matchCriteriaId":"F770709C-FFB2-4A4E-A2D8-2EAA23F2E87C"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager:11.5\\(1\\)su3:*:*:*:*:*:*:*","matchCriteriaId":"B85B81F9-8837-426E-8639-AB0712CD1A96"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager_im_and_presence_service:*:*:*:*:*:*:*:*","versionEndExcluding":"11.5\\(1\\)","matchCriteriaId":"C1CCCD27-A247-4720-A2FE-C8ED55D1D0DE"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager_im_and_presence_service:11.5\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"34D89C42-AAD9-4B04-9F95-F77681E39553"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_contact_center_enterprise:*:*:*:*:*:*:*:*","versionEndExcluding":"11.6\\(2\\)","matchCriteriaId":"897C8893-B0B6-4D6E-8D70-31B421D80B9A"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_contact_center_enterprise:11.6\\(2\\):*:*:*:*:*:*:*","matchCriteriaId":"91D62A73-21B5-4D16-A07A-69AED2D40CC0"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_contact_center_express:*:*:*:*:*:*:*:*","versionEndExcluding":"12.5\\(1\\)","matchCriteriaId":"B0492049-D3AC-4512-A4BF-C9C26DA72CB0"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_customer_voice_portal:*:*:*:*:*:*:*:*","versionEndExcluding":"11.6","matchCriteriaId":"3868A8AA-6660-4332-AB0C-089C150D00E7"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_customer_voice_portal:11.6:*:*:*:*:*:*:*","matchCriteriaId":"58BD72D6-4A79-49C9-9652-AB0136A591FA"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_customer_voice_portal:12.0:*:*:*:*:*:*:*","matchCriteriaId":"A32761FD-B435-4E51-807C-2B245857F90E"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_customer_voice_portal:12.5:*:*:*:*:*:*:*","matchCriteriaId":"154F7F71-53C5-441C-8F5C-0A82CB0DEC43"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_intelligence_center:*:*:*:*:*:*:*:*","versionEndExcluding":"12.6\\(1\\)","matchCriteriaId":"8BD68514-1566-4E7C-879C-76D35084F7BE"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unity_connection:*:*:*:*:*:*:*:*","versionEndExcluding":"11.5\\(1\\)","matchCriteriaId":"65FD3873-2663-4C49-878F-7C65D4B8E455"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:video_surveillance_operations_manager:*:*:*:*:*:*:*:*","versionEndExcluding":"7.14.4","matchCriteriaId":"0886FB04-24AA-4995-BA53-1E44F94E114E"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:virtual_topology_system:*:*:*:*:*:*:*:*","versionEndExcluding":"2.6.7","matchCriteriaId":"C61805C1-1F73-462C-A9CA-BB0CA4E57D0B"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:virtualized_infrastructure_manager:*:*:*:*:*:*:*:*","versionEndExcluding":"3.2.0","matchCriteriaId":"5EB39834-0F6D-4BD7-AFEC-DD8BEE46DA50"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:virtualized_infrastructure_manager:*:*:*:*:*:*:*:*","versionStartIncluding":"3.4.0","versionEndExcluding":"3.4.4","matchCriteriaId":"0B78DD21-15F2-47A4-8A99-6DB6756920AC"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:virtualized_voice_browser:*:*:*:*:*:*:*:*","versionEndExcluding":"12.5\\(1\\)","matchCriteriaId":"7C6222EB-36E1-4CD5-BD69-5A921ED5DA6A"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:wan_automation_engine:*:*:*:*:*:*:*:*","versionEndExcluding":"7.3.0.2","matchCriteriaId":"C200CABD-F91B-49C4-A262-C56370E44B4C"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:*:*:*:*:*:*:*:*","versionEndExcluding":"3.0","matchCriteriaId":"DE22BE9B-374E-43DC-BA91-E3B9699A4C7C"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:3.0:-:*:*:*:*:*:*","matchCriteriaId":"61D1081F-87E8-4E8B-BEBD-0F239E745586"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:3.0:maintenance_release1:*:*:*:*:*:*","matchCriteriaId":"8D138973-02B0-4FEC-A646-FF1278DA1EDF"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:3.0:maintenance_release2:*:*:*:*:*:*","matchCriteriaId":"30B55A5B-8C5E-4ECB-9C85-A8A3A3030850"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:3.0:maintenance_release3:*:*:*:*:*:*","matchCriteriaId":"14DBEC10-0641-441C-BE15-8F72C1762DCE"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:3.0:maintenance_release3:-:*:*:*:*:*","matchCriteriaId":"205C1ABA-2A4F-480F-9768-7E3EC43B03F5"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:3.0:maintenance_release3_security_patch4:*:*:*:*:*:*","matchCriteriaId":"D36FE453-C43F-448B-8A59-668DE95468C0"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:3.0:maintenance_release3_security_patch5:*:*:*:*:*:*","matchCriteriaId":"E8DF0944-365F-4149-9059-BDFD6B131DC5"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:3.0:maintenance_release3_service_pack_2:*:*:*:*:*:*","matchCriteriaId":"6B37AA08-13C7-4FD0-8402-E344A270C8F7"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:3.0:maintenance_release3_service_pack_3:*:*:*:*:*:*","matchCriteriaId":"2AA56735-5A5E-4D8C-B09D-DBDAC2B5C8E9"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:3.0:maintenance_release4:*:*:*:*:*:*","matchCriteriaId":"4646849B-8190-4798-833C-F367E28C1881"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:4.0:-:*:*:*:*:*:*","matchCriteriaId":"4D6CF856-093A-4E89-A71D-50A2887C265B"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:4.0:maintenance_release1:*:*:*:*:*:*","matchCriteriaId":"B36A9043-0621-43CD-BFCD-66529F937859"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:4.0:maintenance_release2:*:*:*:*:*:*","matchCriteriaId":"8842B42E-C412-4356-9F54-DFC53B683D3E"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:4.0:maintenance_release3:*:*:*:*:*:*","matchCriteriaId":"D25BC647-C569-46E5-AD45-7E315EBEB784"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:workload_optimization_manager:*:*:*:*:*:*:*:*","versionEndExcluding":"3.2.1","matchCriteriaId":"B468EDA1-CDEF-44D4-9D62-C433CF27F631"},{"vulnerable":true,"criteria":"cpe:2.3:o:cisco:unified_sip_proxy:*:*:*:*:*:*:*:*","versionEndExcluding":"10.2.1v2","matchCriteriaId":"9E4905E2-2129-469C-8BBD-EDA258815E2B"},{"vulnerable":true,"criteria":"cpe:2.3:o:cisco:unified_workforce_optimization:*:*:*:*:*:*:*:*","versionEndExcluding":"11.5\\(1\\)","matchCriteriaId":"EC86AC6C-7C08-4EB9-A588-A034113E4BB1"}]}]},{"operator":"AND","nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_1010:-:*:*:*:*:*:*:*","matchCriteriaId":"7FFE3880-4B85-4E23-9836-70875D5109F7"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_1120:-:*:*:*:*:*:*:*","matchCriteriaId":"727A02E8-40A1-4DFE-A3A2-91D628D3044F"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_1140:-:*:*:*:*:*:*:*","matchCriteriaId":"19F6546E-28F4-40DC-97D6-E0E023FE939B"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_1150:-:*:*:*:*:*:*:*","matchCriteriaId":"EB3B0EC3-4654-4D90-9D41-7EC2AD1DDF99"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_2110:-:*:*:*:*:*:*:*","matchCriteriaId":"52D96810-5F79-4A83-B8CA-D015790FCF72"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_2120:-:*:*:*:*:*:*:*","matchCriteriaId":"16FE2945-4975-4003-AE48-7E134E167A7F"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_2130:-:*:*:*:*:*:*:*","matchCriteriaId":"DCE7122A-5AA7-4ECD-B024-E27C9D0CFB7B"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_2140:-:*:*:*:*:*:*:*","matchCriteriaId":"976901BF-C52C-4F81-956A-711AF8A60140"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_4110:-:*:*:*:*:*:*:*","matchCriteriaId":"A0CBC7F5-7767-43B6-9384-BE143FCDBD7F"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_4112:-:*:*:*:*:*:*:*","matchCriteriaId":"957D64EB-D60E-4775-B9A8-B21CA48ED3B1"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_4115:-:*:*:*:*:*:*:*","matchCriteriaId":"A694AD51-9008-4AE6-8240-98B17AB527EE"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_4120:-:*:*:*:*:*:*:*","matchCriteriaId":"38AE6DC0-2B03-4D36-9856-42530312CC46"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_4125:-:*:*:*:*:*:*:*","matchCriteriaId":"71DCEF22-ED20-4330-8502-EC2DD4C9838F"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_4140:-:*:*:*:*:*:*:*","matchCriteriaId":"3DB2822B-B752-4CD9-A178-934957E306B4"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_4145:-:*:*:*:*:*:*:*","matchCriteriaId":"81F4868A-6D62-479C-9C19-F9AABDBB6B24"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_4150:-:*:*:*:*:*:*:*","matchCriteriaId":"65378F3A-777C-4AE2-87FB-1E7402F9EA1B"},{"vulnerable":false,"criteria":"cpe:2.3:h:cisco:firepower_9300:-:*:*:*:*:*:*:*","matchCriteriaId":"07DAFDDA-718B-4B69-A524-B0CEB80FE960"}]},{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:o:cisco:fxos:6.2.3:*:*:*:*:*:*:*","matchCriteriaId":"82C8AD48-0130-4C20-ADEC-697668E2293B"},{"vulnerable":true,"criteria":"cpe:2.3:o:cisco:fxos:6.3.0:*:*:*:*:*:*:*","matchCriteriaId":"4E75EF7C-8D71-4D70-91F0-74FC99A90CC3"},{"vulnerable":true,"criteria":"cpe:2.3:o:cisco:fxos:6.4.0:*:*:*:*:*:*:*","matchCriteriaId":"2DB7EE7D-8CB4-4804-9F9D-F235608E86E1"},{"vulnerable":true,"criteria":"cpe:2.3:o:cisco:fxos:6.5.0:*:*:*:*:*:*:*","matchCriteriaId":"77571973-2A94-4E15-AC5B-155679C3C565"},{"vulnerable":true,"criteria":"cpe:2.3:o:cisco:fxos:6.6.0:*:*:*:*:*:*:*","matchCriteriaId":"CA405A50-3F31-48ED-9AF1-4B02F5B367DE"},{"vulnerable":true,"criteria":"cpe:2.3:o:cisco:fxos:6.7.0:*:*:*:*:*:*:*","matchCriteriaId":"D3753953-04E8-4382-A6EC-CD334DD83CF4"},{"vulnerable":true,"criteria":"cpe:2.3:o:cisco:fxos:7.0.0:*:*:*:*:*:*:*","matchCriteriaId":"B4A5F89F-1296-4A0F-A36D-082A481F190F"},{"vulnerable":true,"criteria":"cpe:2.3:o:cisco:fxos:7.1.0:*:*:*:*:*:*:*","matchCriteriaId":"F50F48AF-44FF-425C-9685-E386F956C901"}]}]},{"nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:automated_subsea_tuning:02.01.00:*:*:*:*:*:*:*","matchCriteriaId":"A4D28E76-56D4-4C9A-A660-7CD7E0A1AC9F"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:broadworks:-:*:*:*:*:*:*:*","matchCriteriaId":"CD975A0E-00A6-475E-9064-1D64E4291499"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:cloudcenter_suite:4.10.0.15:*:*:*:*:*:*:*","matchCriteriaId":"9C6EE38C-5396-4ACE-8FA3-B147251C6D9F"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:cloudcenter_suite:5.3.0:*:*:*:*:*:*:*","matchCriteriaId":"E5569201-9A97-481E-96FD-7F41697BD5EA"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:cloudcenter_suite:5.4.1:*:*:*:*:*:*:*","matchCriteriaId":"10A33C3D-5890-430B-9284-3E28F03ADAE8"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:cloudcenter_suite:5.5.0:*:*:*:*:*:*:*","matchCriteriaId":"777C92AC-A9B6-4204-99D1-EA7E3EE366A3"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:cloudcenter_suite:5.5.1:*:*:*:*:*:*:*","matchCriteriaId":"5A908014-8E8B-477D-9E81-3801F43723BB"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:common_services_platform_collector:002.009\\(000.000\\):*:*:*:*:*:*:*","matchCriteriaId":"BAC1A386-04C7-45B2-A883-1CD9AB60C14B"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:common_services_platform_collector:002.009\\(000.001\\):*:*:*:*:*:*:*","matchCriteriaId":"3F0F1639-D69E-473A-8926-827CCF73ACC9"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:common_services_platform_collector:002.009\\(000.002\\):*:*:*:*:*:*:*","matchCriteriaId":"F4FDF900-E9D6-454A-BF6B-821620CA59F4"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:common_services_platform_collector:002.009\\(001.000\\):*:*:*:*:*:*:*","matchCriteriaId":"1859BD43-BA2B-45A5-B523-C6BFD34C7B01"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:common_services_platform_collector:002.009\\(001.001\\):*:*:*:*:*:*:*","matchCriteriaId":"1EBC145C-9A2F-4B76-953E-0F690314511C"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:common_services_platform_collector:002.009\\(001.002\\):*:*:*:*:*:*:*","matchCriteriaId":"158B7A53-FEC1-4B42-A1E2-E83E99564B07"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:common_services_platform_collector:002.010\\(000.000\\):*:*:*:*:*:*:*","matchCriteriaId":"3A378971-1A08-4914-B012-8E24DCDEFC68"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_network_automation:-:*:*:*:*:*:*:*","matchCriteriaId":"2F429F37-3576-4D8A-9901-359D65EC3CF4"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_network_automation:2.0.0:*:*:*:*:*:*:*","matchCriteriaId":"F526DEF1-4A3E-4FE1-8153-E9252DAE5B92"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_network_automation:3.0.0:*:*:*:*:*:*:*","matchCriteriaId":"C19679D0-F4DC-4130-AFFD-692E5130531A"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_network_automation:4.1.0:*:*:*:*:*:*:*","matchCriteriaId":"60D2FBF3-D8AB-41F0-B170-9E56FBF7E2F7"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:crosswork_network_automation:4.1.1:*:*:*:*:*:*:*","matchCriteriaId":"F60324DD-8450-4B14-A7A1-0D5EA5163580"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:cx_cloud_agent:001.012:*:*:*:*:*:*:*","matchCriteriaId":"12F6DFD1-273B-4292-A22C-F2BE0DD3FB3F"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:cyber_vision:4.0.2:*:*:*:*:*:*:*","matchCriteriaId":"13EA024C-97A4-4D33-BC3E-51DB77C51E76"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:cyber_vision_sensor_management_extension:4.0.2:*:*:*:*:*:*:*","matchCriteriaId":"85289E35-C7C2-46D0-9BDC-10648DD2C86F"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:dna_center:2.2.2.8:*:*:*:*:*:*:*","matchCriteriaId":"17282822-C082-4FBC-B46D-468DCF8EF6B8"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:dna_spaces:-:*:*:*:*:*:*:*","matchCriteriaId":"F5463DA6-5D44-4C32-B46C-E8A2ADD7646B"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:dna_spaces_connector:-:*:*:*:*:*:*:*","matchCriteriaId":"54A237CF-A439-4114-AF81-D75582F29573"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:emergency_responder:11.5:*:*:*:*:*:*:*","matchCriteriaId":"A37D19BF-E4F5-4AF4-8942-0C3B62C4BF2B"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:emergency_responder:11.5\\(4.65000.14\\):*:*:*:*:*:*:*","matchCriteriaId":"EF25688B-6659-4C7C-866D-79AA1166AD7A"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:emergency_responder:11.5\\(4.66000.14\\):*:*:*:*:*:*:*","matchCriteriaId":"47B70741-90D9-4676-BF16-8A21E147F532"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:enterprise_chat_and_email:12.0\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"ED862A1B-E558-4D44-839C-270488E735BB"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:enterprise_chat_and_email:12.5\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"2678AF98-1194-4810-9933-5BA50E409F88"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:enterprise_chat_and_email:12.6\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"37E7DEBD-9E47-4D08-86BC-D1B013450A98"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:evolved_programmable_network_manager:3.0:*:*:*:*:*:*:*","matchCriteriaId":"1A935862-18F7-45FE-B647-1A9BA454E304"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:evolved_programmable_network_manager:3.1:*:*:*:*:*:*:*","matchCriteriaId":"69594997-2568-4C10-A411-69A50BFD175F"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:evolved_programmable_network_manager:4.0:*:*:*:*:*:*:*","matchCriteriaId":"1EC39E2D-C47B-4311-BC7B-130D432549F4"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:evolved_programmable_network_manager:4.1:*:*:*:*:*:*:*","matchCriteriaId":"EE5E6CBE-D82C-4001-87CB-73DF526F0AB1"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:evolved_programmable_network_manager:5.0:*:*:*:*:*:*:*","matchCriteriaId":"460E6456-0E51-45BC-868E-DEEA5E3CD366"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:evolved_programmable_network_manager:5.1:*:*:*:*:*:*:*","matchCriteriaId":"F7F58659-A318-42A0-83C5-8F09FCD78982"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:finesse:12.5\\(1\\):su1:*:*:*:*:*:*","matchCriteriaId":"D8A49E46-8501-4697-A17A-249A7D9F5A0B"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:finesse:12.5\\(1\\):su2:*:*:*:*:*:*","matchCriteriaId":"5D81E7A9-0C2B-4603-91F0-ABF2380DBBA3"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:finesse:12.6\\(1\\):-:*:*:*:*:*:*","matchCriteriaId":"4DFCE723-9359-40C7-BA35-B71BDF8E3CF3"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:finesse:12.6\\(1\\):es01:*:*:*:*:*:*","matchCriteriaId":"28B1524E-FDCA-4570-86DD-CE396271B232"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:finesse:12.6\\(1\\):es02:*:*:*:*:*:*","matchCriteriaId":"74DC6F28-BFEF-4D89-93D5-10072DAC39C8"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:finesse:12.6\\(1\\):es03:*:*:*:*:*:*","matchCriteriaId":"BA1D60D7-1B4A-4EEE-A26C-389D9271E005"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:firepower_threat_defense:6.2.3:*:*:*:*:*:*:*","matchCriteriaId":"1D726F07-06F1-4B0A-B010-E607E0C2A280"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:firepower_threat_defense:6.3.0:*:*:*:*:*:*:*","matchCriteriaId":"3ED58B0E-FCC7-48E3-A5C0-6CC54A38BAE3"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:firepower_threat_defense:6.4.0:*:*:*:*:*:*:*","matchCriteriaId":"B2DF0B07-8C2A-4341-8AFF-DE7E5E5B3A43"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:firepower_threat_defense:6.5.0:*:*:*:*:*:*:*","matchCriteriaId":"41E168ED-D664-4749-805E-77644407EAFE"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:firepower_threat_defense:6.6.0:*:*:*:*:*:*:*","matchCriteriaId":"DCD69468-8067-4A5D-B2B0-EC510D889AA0"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:firepower_threat_defense:6.7.0:*:*:*:*:*:*:*","matchCriteriaId":"85F22403-B4EE-4303-9C94-915D3E0AC944"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:firepower_threat_defense:7.0.0:*:*:*:*:*:*:*","matchCriteriaId":"BBCA75A6-0A3E-4393-8884-9F3CE190641E"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:firepower_threat_defense:7.1.0:*:*:*:*:*:*:*","matchCriteriaId":"D619BF54-1BA9-45D0-A876-92D7010088A0"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:identity_services_engine:002.004\\(000.914\\):-:*:*:*:*:*:*","matchCriteriaId":"808F8065-BD3A-4802-83F9-CE132EDB8D34"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:identity_services_engine:002.006\\(000.156\\):-:*:*:*:*:*:*","matchCriteriaId":"B236B13E-93B9-424E-926C-95D3DBC6CA5D"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:identity_services_engine:002.007\\(000.356\\):-:*:*:*:*:*:*","matchCriteriaId":"8A63CC83-0A6E-4F33-A1BE-214A33B51518"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:identity_services_engine:003.000\\(000.458\\):-:*:*:*:*:*:*","matchCriteriaId":"37DB7759-6529-46DE-B384-10F060D86A97"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:identity_services_engine:003.001\\(000.518\\):-:*:*:*:*:*:*","matchCriteriaId":"8C640AD9-146E-488A-B166-A6BB940F97D3"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:identity_services_engine:003.002\\(000.116\\):-:*:*:*:*:*:*","matchCriteriaId":"DAC1FA7E-CB1B-46E5-A248-ABACECFBD6E8"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:integrated_management_controller_supervisor:002.003\\(002.000\\):*:*:*:*:*:*:*","matchCriteriaId":"7C3BD5AF-9FC1-494B-A676-CC3D4B8EAC8D"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:integrated_management_controller_supervisor:2.3.2.0:*:*:*:*:*:*:*","matchCriteriaId":"F477CACA-2AA0-417C-830D-F2D3AE93153A"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:intersight_virtual_appliance:1.0.9-343:*:*:*:*:*:*:*","matchCriteriaId":"7E3BE5E1-A6B6-46C7-B93B-8A9F5AEA2731"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:mobility_services_engine:-:*:*:*:*:*:*:*","matchCriteriaId":"04E0BB7B-0716-4DBD-89B9-BA11AAD77C00"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_assurance_engine:6.0\\(2.1912\\):*:*:*:*:*:*:*","matchCriteriaId":"64C98A76-0C31-45E7-882B-35AE0D2C5430"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_dashboard_fabric_controller:11.0\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"379F8D86-BE87-4250-9E85-494D331A0398"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_dashboard_fabric_controller:11.1\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"71F69E51-E59D-4AE3-B242-D6D2CFDB3F46"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_dashboard_fabric_controller:11.2\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"578DA613-8E15-4748-A4B7-646415449609"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_dashboard_fabric_controller:11.3\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"544EFAD6-CE2F-4E1D-9A00-043454B72889"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_dashboard_fabric_controller:11.4\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"2E16DF9C-3B64-4220-82B6-6E20C7807BAA"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_dashboard_fabric_controller:11.5\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"B9CD5B8A-9846-48F1-9495-77081E44CBFC"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_dashboard_fabric_controller:11.5\\(2\\):*:*:*:*:*:*:*","matchCriteriaId":"68E6CD49-6F71-4E17-B046-FBE91CE91CB7"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_dashboard_fabric_controller:11.5\\(3\\):*:*:*:*:*:*:*","matchCriteriaId":"0BDD8018-7E77-4C89-917E-ACDC678A7DE2"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_insights_for_data_center:6.0\\(2.1914\\):*:*:*:*:*:*:*","matchCriteriaId":"A7D39156-A47D-405E-8C02-CAE7D637F99A"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:network_services_orchestrator:-:*:*:*:*:*:*:*","matchCriteriaId":"5426FC59-411D-4963-AFEF-5B55F68B8958"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:optical_network_controller:1.1:*:*:*:*:*:*:*","matchCriteriaId":"810E9A92-4302-4396-94D3-3003947DB2A7"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:paging_server:8.3\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"522C36A5-7520-4368-BD92-9AB577756493"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:paging_server:8.4\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"CB2EC4BE-FFAF-4605-8A96-2FEF35975540"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:paging_server:8.5\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"CA1D3C2A-E5FA-400C-AC01-27A3E5160477"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:paging_server:9.0\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"63B27050-997B-4D54-8E5A-CE9E33904318"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:paging_server:9.0\\(2\\):*:*:*:*:*:*:*","matchCriteriaId":"5ABF05B8-1B8A-4CCF-A1AD-D8602A247718"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:paging_server:9.1\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"2F74580D-0011-4ED9-9A00-B4CDB6685154"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:paging_server:12.5\\(2\\):*:*:*:*:*:*:*","matchCriteriaId":"17A3C22E-1980-49B6-8985-9FA76A77A836"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:paging_server:14.0\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"B1AB42DC-CE58-448A-A6B5-56F31B15F4A0"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:prime_service_catalog:12.1:*:*:*:*:*:*:*","matchCriteriaId":"9DC32B55-0C76-4669-8EAD-DCC16355E887"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:sd-wan_vmanage:20.3:*:*:*:*:*:*:*","matchCriteriaId":"6CDA737F-337E-4C30-B68D-EF908A8D6840"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:sd-wan_vmanage:20.4:*:*:*:*:*:*:*","matchCriteriaId":"9DC5A89C-CCCF-49EC-B4FC-AB98ACB79233"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:sd-wan_vmanage:20.5:*:*:*:*:*:*:*","matchCriteriaId":"4BA4F513-CBA1-4523-978B-D498CEDAE0CF"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:sd-wan_vmanage:20.6:*:*:*:*:*:*:*","matchCriteriaId":"6C53C6FD-B98E-4F7E-BA4D-391C90CF9E83"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:sd-wan_vmanage:20.6.1:*:*:*:*:*:*:*","matchCriteriaId":"D00F6719-2C73-4D8D-8505-B9922E8A4627"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:sd-wan_vmanage:20.7:*:*:*:*:*:*:*","matchCriteriaId":"EFE9210F-39C5-4828-9608-6905C1D378D4"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:sd-wan_vmanage:20.8:*:*:*:*:*:*:*","matchCriteriaId":"A1CEDCE4-CFD1-434B-B157-D63329CBA24A"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:smart_phy:3.1.2:*:*:*:*:*:*:*","matchCriteriaId":"33660EB8-2984-4258-B8AD-141B7065C85E"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:smart_phy:3.1.3:*:*:*:*:*:*:*","matchCriteriaId":"0ACA346D-5103-47F0-8BD9-7A8AD9B92E98"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:smart_phy:3.1.4:*:*:*:*:*:*:*","matchCriteriaId":"A38BDF03-23C8-4BB6-A44D-68818962E7CB"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:smart_phy:3.1.5:*:*:*:*:*:*:*","matchCriteriaId":"3104C099-FEDA-466B-93CC-D55F058F7CD3"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:smart_phy:3.2.1:*:*:*:*:*:*:*","matchCriteriaId":"890EA1C7-5990-4C71-857F-197E6F5B4089"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:smart_phy:21.3:*:*:*:*:*:*:*","matchCriteriaId":"56F21CF4-83FE-4529-9871-0FDD70D3095E"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:ucs_central_software:2.0:*:*:*:*:*:*:*","matchCriteriaId":"B9331834-9EAD-46A1-9BD4-F4027E49D0C3"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:ucs_central_software:2.0\\(1a\\):*:*:*:*:*:*:*","matchCriteriaId":"0E707E44-12CD-46C3-9124-639D0265432E"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:ucs_central_software:2.0\\(1b\\):*:*:*:*:*:*:*","matchCriteriaId":"2FEE8482-DB64-4421-B646-9E5F560D1712"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:ucs_central_software:2.0\\(1c\\):*:*:*:*:*:*:*","matchCriteriaId":"4385CE6E-6283-4621-BBD9-8E66E2A34843"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:ucs_central_software:2.0\\(1d\\):*:*:*:*:*:*:*","matchCriteriaId":"9A6CDBD4-889B-442D-B272-C8E9A1B6AEC0"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:ucs_central_software:2.0\\(1e\\):*:*:*:*:*:*:*","matchCriteriaId":"FF1E59F9-CF4F-4EFB-872C-5F503A04CCF4"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:ucs_central_software:2.0\\(1f\\):*:*:*:*:*:*:*","matchCriteriaId":"1782219F-0C3D-45B7-80C7-D1DAA70D90B1"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:ucs_central_software:2.0\\(1g\\):*:*:*:*:*:*:*","matchCriteriaId":"DDAB3BAD-1EC6-4101-A58D-42DA48D04D0C"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:ucs_central_software:2.0\\(1h\\):*:*:*:*:*:*:*","matchCriteriaId":"8F7AA674-6BC2-490F-8D8A-F575B11F4BE0"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:ucs_central_software:2.0\\(1k\\):*:*:*:*:*:*:*","matchCriteriaId":"6945C4DE-C070-453E-B641-2F5B9CFA3B6D"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:ucs_central_software:2.0\\(1l\\):*:*:*:*:*:*:*","matchCriteriaId":"DAB8C7C0-D09B-4232-A88E-57D25AF45457"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager:11.5\\(1.17900.52\\):*:*:*:*:*:*:*","matchCriteriaId":"ACEDB7B4-EBD4-4A37-9EE3-07EE3B46BE44"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager:11.5\\(1.18119.2\\):*:*:*:*:*:*:*","matchCriteriaId":"820D579C-AA45-4DC1-945A-748FFCD51CA2"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager:11.5\\(1.18900.97\\):*:*:*:*:*:*:*","matchCriteriaId":"7B23A9A6-CD04-4D76-BE3F-AFAFBB525F5E"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager:11.5\\(1.21900.40\\):*:*:*:*:*:*:*","matchCriteriaId":"A44E6007-7A3A-4AD3-9A65-246C59B73FB6"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager:11.5\\(1.22900.28\\):*:*:*:*:*:*:*","matchCriteriaId":"3D508E51-4075-4E34-BB7C-65AF9D56B49F"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager_im_\\&_presence_service:11.5\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"376D06D5-D68E-4FF0-97E5-CBA2165A05CF"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_communications_manager_im_\\&_presence_service:11.5\\(1.22900.6\\):*:*:*:*:*:*:*","matchCriteriaId":"18ED6B8F-2064-4BBA-A78D-4408F13C724D"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_computing_system:006.008\\(001.000\\):*:*:*:*:*:*:*","matchCriteriaId":"94091FE3-AB88-4CF5-8C4C-77B349E716A9"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_contact_center_enterprise:11.6\\(2\\):*:*:*:*:*:*:*","matchCriteriaId":"91D62A73-21B5-4D16-A07A-69AED2D40CC0"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_contact_center_enterprise:12.0\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"53F1314A-9A2C-43DC-8203-E4654EF013CC"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_contact_center_enterprise:12.5\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"0ADE468B-8F0C-490D-BB4C-358D947BA8E4"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_contact_center_enterprise:12.6\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"32FEE78D-309E-491D-9AB6-98005F1CBF49"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_contact_center_enterprise:12.6\\(2\\):*:*:*:*:*:*:*","matchCriteriaId":"878D9901-675D-4444-B094-0BA505E7433F"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_contact_center_express:12.5\\(1\\):-:*:*:*:*:*:*","matchCriteriaId":"66E25EE4-AB7B-42BF-A703-0C2E83E83577"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_contact_center_express:12.5\\(1\\):su1:*:*:*:*:*:*","matchCriteriaId":"D8F35520-F04A-4863-A1BC-0EDD2D1804F7"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_contact_center_express:12.6\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"EF9855FD-7747-4D9E-9542-703B1EC9A382"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_contact_center_express:12.6\\(2\\):*:*:*:*:*:*:*","matchCriteriaId":"E07AF386-D8A5-44F5-A418-940C9F88A36A"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_contact_center_management_portal:12.6\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"113C77DA-AC22-4D67-9812-8510EFC0A95F"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_customer_voice_portal:11.6\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"4BE221AB-A3B0-4CFF-9BC0-777773C2EF63"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_customer_voice_portal:12.0\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"15941265-1E7E-4C3E-AF1D-027C5E0D3141"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_customer_voice_portal:12.5\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"54AA2B0C-92A1-4B53-88D7-6E31120F5041"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_customer_voice_portal:12.6\\(1\\):*:*:*:*:*:*:*","matchCriteriaId":"F9BD7207-85FB-4484-8720-4D11F296AC10"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_intelligence_center:12.6\\(1\\):-:*:*:*:*:*:*","matchCriteriaId":"62E009C4-BE3E-4A14-91EF-8F667B2220A7"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_intelligence_center:12.6\\(1\\):es01:*:*:*:*:*:*","matchCriteriaId":"088512E1-434D-4685-992E-192A98ECAD9A"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_intelligence_center:12.6\\(1\\):es02:*:*:*:*:*:*","matchCriteriaId":"50A7BBC6-077C-4182-AA7A-577C4AAC3CD8"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_intelligence_center:12.6\\(2\\):-:*:*:*:*:*:*","matchCriteriaId":"E0536F45-3A49-4F93-942E-AF679DFC7017"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_sip_proxy:010.000\\(000\\):*:*:*:*:*:*:*","matchCriteriaId":"3D54794B-6CD5-46D7-B9E9-62A642143562"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_sip_proxy:010.000\\(001\\):*:*:*:*:*:*:*","matchCriteriaId":"BE844DCA-FF52-43F5-BDD9-836A812A8CFF"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_sip_proxy:010.002\\(000\\):*:*:*:*:*:*:*","matchCriteriaId":"07B261EB-CA63-4796-BD15-A6770FD68B34"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_sip_proxy:010.002\\(001\\):*:*:*:*:*:*:*","matchCriteriaId":"29F9067A-B86C-4A6B-ACB7-DB125E04B795"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unified_workforce_optimization:11.5\\(1\\):sr7:*:*:*:*:*:*","matchCriteriaId":"FAC4CC92-8BA0-4D96-9C48-5E311CDED53F"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unity_connection:11.5:*:*:*:*:*:*:*","matchCriteriaId":"8F2437A5-217A-4CD1-9B72-A31BDDC81F42"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:unity_connection:11.5\\(1.10000.6\\):*:*:*:*:*:*:*","matchCriteriaId":"9C3CFF0D-BD70-4353-AE2F-6C55F8DE56A2"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:video_surveillance_manager:7.14\\(1.26\\):*:*:*:*:*:*:*","matchCriteriaId":"2CE47760-0E71-4FCA-97D1-CF0BB71CAC17"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:video_surveillance_manager:7.14\\(2.26\\):*:*:*:*:*:*:*","matchCriteriaId":"89B2D4F5-CB86-4B25-8C14-CED59E8A3F22"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:video_surveillance_manager:7.14\\(3.025\\):*:*:*:*:*:*:*","matchCriteriaId":"B150B636-6267-4504-940F-DC37ABEFB082"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:video_surveillance_manager:7.14\\(4.018\\):*:*:*:*:*:*:*","matchCriteriaId":"D00B9911-A7CA-467E-B7A3-3AF31828D5D9"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:virtual_topology_system:2.6.6:*:*:*:*:*:*:*","matchCriteriaId":"B67C08C3-412F-4B7F-B98C-EEAEE77CBE4B"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:wan_automation_engine:7.1.3:*:*:*:*:*:*:*","matchCriteriaId":"6D428C9B-53E1-4D26-BB4D-57FDE02FA613"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:wan_automation_engine:7.2.1:*:*:*:*:*:*:*","matchCriteriaId":"CDB41596-FACF-440A-BB6C-8CAD792EC186"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:wan_automation_engine:7.2.2:*:*:*:*:*:*:*","matchCriteriaId":"D8C88EE2-5702-4E8B-A144-CB485435FD62"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:wan_automation_engine:7.2.3:*:*:*:*:*:*:*","matchCriteriaId":"1BC62844-C608-4DB1-A1AD-C1B55128C560"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:wan_automation_engine:7.3:*:*:*:*:*:*:*","matchCriteriaId":"EFF2FFA4-358A-4F33-BC67-A9EF8A30714E"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:wan_automation_engine:7.4:*:*:*:*:*:*:*","matchCriteriaId":"53C0BBDE-795E-4754-BB96-4D6D4B5A804F"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:wan_automation_engine:7.5:*:*:*:*:*:*:*","matchCriteriaId":"7A41E377-16F9-423F-8DC2-F6EDD54E1069"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:wan_automation_engine:7.6:*:*:*:*:*:*:*","matchCriteriaId":"F0C2789E-255B-45D9-9469-B5B549A01F53"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:3.0:*:*:*:*:*:*:*","matchCriteriaId":"EFAFEC61-2128-4BFA-992D-54742BD4911A"},{"vulnerable":true,"criteria":"cpe:2.3:a:cisco:webex_meetings_server:4.0:*:*:*:*:*:*:*","matchCriteriaId":"F12AF70E-2201-4F5D-A929-A1A057B74252"}]}]},{"nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:a:snowsoftware:snow_commander:*:*:*:*:*:*:*:*","versionEndExcluding":"8.10.0","matchCriteriaId":"A2CBCDC4-02DF-47F4-A01C-7CBCB2FF0163"},{"vulnerable":true,"criteria":"cpe:2.3:a:snowsoftware:vm_access_proxy:*:*:*:*:*:*:*:*","versionEndExcluding":"3.6","matchCriteriaId":"C42D44C8-9894-4183-969B-B38FDA1FEDF9"}]}]},{"nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:a:bentley:synchro:*:*:*:*:pro:*:*:*","versionStartIncluding":"6.1","versionEndExcluding":"6.2.4.2","matchCriteriaId":"452D8730-F273-4AB4-9221-E82EC2CAAFD8"},{"vulnerable":true,"criteria":"cpe:2.3:a:bentley:synchro_4d:*:*:*:*:pro:*:*:*","versionEndExcluding":"6.4.3.2","matchCriteriaId":"F2EF5054-EECB-4489-B27A-AACB96B25B97"}]}]},{"nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:a:percussion:rhythmyx:*:*:*:*:*:*:*:*","versionEndIncluding":"7.3.2","matchCriteriaId":"16E0A04D-30BE-4AB3-85A1-13AF614C425C"}]}]},{"nodes":[{"operator":"OR","negate":false,"cpeMatch":[{"vulnerable":true,"criteria":"cpe:2.3:a:apple:xcode:*:*:*:*:*:*:*:*","versionEndExcluding":"13.3","matchCriteriaId":"E0755E91-2F36-4EC3-8727-E8BF0427E663"}]}]}],"references":[{"url":"http://packetstormsecurity.com/files/165225/Apache-Log4j2-2.14.1-Remote-Code-Execution.html","source":"security@apache.org","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165260/VMware-Security-Advisory-2021-0028.html","source":"security@apache.org","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165261/Apache-Log4j2-2.14.1-Information-Disclosure.html","source":"security@apache.org","tags":["Exploit","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165270/Apache-Log4j2-2.14.1-Remote-Code-Execution.html","source":"security@apache.org","tags":["Exploit","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165281/Log4j2-Log4Shell-Regexes.html","source":"security@apache.org","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165282/Log4j-Payload-Generator.html","source":"security@apache.org","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165306/L4sh-Log4j-Remote-Code-Execution.html","source":"security@apache.org","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165307/Log4j-Remote-Code-Execution-Word-Bypassing.html","source":"security@apache.org","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165311/log4j-scan-Extensive-Scanner.html","source":"security@apache.org","tags":["Broken Link","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165371/VMware-Security-Advisory-2021-0028.4.html","source":"security@apache.org","tags":["Exploit","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165532/Log4Shell-HTTP-Header-Injection.html","source":"security@apache.org","tags":["Exploit","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165642/VMware-vCenter-Server-Unauthenticated-Log4Shell-JNDI-Injection-Remote-Code-Execution.html","source":"security@apache.org","tags":["Exploit","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165673/UniFi-Network-Application-Unauthenticated-Log4Shell-Remote-Code-Execution.html","source":"security@apache.org","tags":["Exploit","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/167794/Open-Xchange-App-Suite-7.10.x-Cross-Site-Scripting-Command-Injection.html","source":"security@apache.org","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/167917/MobileIron-Log4Shell-Remote-Command-Execution.html","source":"security@apache.org","tags":["Exploit","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/171626/AD-Manager-Plus-7122-Remote-Code-Execution.html","source":"security@apache.org","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://seclists.org/fulldisclosure/2022/Dec/2","source":"security@apache.org","tags":["Exploit","Mailing List","Third Party Advisory"]},{"url":"http://seclists.org/fulldisclosure/2022/Jul/11","source":"security@apache.org","tags":["Mailing List","Third Party Advisory"]},{"url":"http://seclists.org/fulldisclosure/2022/Mar/23","source":"security@apache.org","tags":["Mailing List","Third Party Advisory"]},{"url":"http://www.openwall.com/lists/oss-security/2021/12/10/1","source":"security@apache.org","tags":["Mailing List","Mitigation","Third Party Advisory"]},{"url":"http://www.openwall.com/lists/oss-security/2021/12/10/2","source":"security@apache.org","tags":["Mailing List","Mitigation","Third Party Advisory"]},{"url":"http://www.openwall.com/lists/oss-security/2021/12/10/3","source":"security@apache.org","tags":["Mailing List","Third Party Advisory"]},{"url":"http://www.openwall.com/lists/oss-security/2021/12/13/1","source":"security@apache.org","tags":["Mailing List","Third Party Advisory"]},{"url":"http://www.openwall.com/lists/oss-security/2021/12/13/2","source":"security@apache.org","tags":["Mailing List","Third Party Advisory"]},{"url":"http://www.openwall.com/lists/oss-security/2021/12/14/4","source":"security@apache.org","tags":["Mailing List","Third Party Advisory"]},{"url":"http://www.openwall.com/lists/oss-security/2021/12/15/3","source":"security@apache.org","tags":["Mailing List","Third Party Advisory"]},{"url":"https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf","source":"security@apache.org","tags":["Third Party Advisory"]},{"url":"https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf","source":"security@apache.org","tags":["Third Party Advisory"]},{"url":"https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf","source":"security@apache.org","tags":["Third Party Advisory"]},{"url":"https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf","source":"security@apache.org","tags":["Third Party Advisory"]},{"url":"https://github.com/cisagov/log4j-affected-db","source":"security@apache.org","tags":["Third Party Advisory"]},{"url":"https://github.com/cisagov/log4j-affected-db/blob/develop/SOFTWARE-LIST.md","source":"security@apache.org","tags":["Broken Link","Product","US Government Resource"]},{"url":"https://github.com/nu11secur1ty/CVE-mitre/tree/main/CVE-2021-44228","source":"security@apache.org","tags":["Exploit","Third Party Advisory"]},{"url":"https://lists.debian.org/debian-lts-announce/2021/12/msg00007.html","source":"security@apache.org","tags":["Mailing List","Third Party Advisory"]},{"url":"https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/M5CSVUNV4HWZZXGOKNSK6L7RPM7BOKIB/","source":"security@apache.org","tags":["Release Notes"]},{"url":"https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/VU57UJDCFIASIO35GC55JMKSRXJMCDFM/","source":"security@apache.org","tags":["Release Notes"]},{"url":"https://logging.apache.org/log4j/2.x/security.html","source":"security@apache.org","tags":["Release Notes","Vendor Advisory"]},{"url":"https://msrc-blog.microsoft.com/2021/12/11/microsofts-response-to-cve-2021-44228-apache-log4j2/","source":"security@apache.org","tags":["Patch","Third Party Advisory","Vendor Advisory"]},{"url":"https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032","source":"security@apache.org","tags":["Third Party Advisory"]},{"url":"https://security.netapp.com/advisory/ntap-20211210-0007/","source":"security@apache.org","tags":["Third Party Advisory"]},{"url":"https://support.apple.com/kb/HT213189","source":"security@apache.org","tags":["Third Party Advisory"]},{"url":"https://tools.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd","source":"security@apache.org","tags":["Third Party Advisory"]},{"url":"https://twitter.com/kurtseifried/status/1469345530182455296","source":"security@apache.org","tags":["Broken Link","Exploit","Third Party Advisory"]},{"url":"https://www.bentley.com/en/common-vulnerability-exposure/be-2022-0001","source":"security@apache.org","tags":["Third Party Advisory"]},{"url":"https://www.debian.org/security/2021/dsa-5020","source":"security@apache.org","tags":["Mailing List","Third Party Advisory"]},{"url":"https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html","source":"security@apache.org","tags":["Third Party Advisory"]},{"url":"https://www.kb.cert.org/vuls/id/930724","source":"security@apache.org","tags":["Third Party Advisory","US Government Resource"]},{"url":"https://www.nu11secur1ty.com/2021/12/cve-2021-44228.html","source":"security@apache.org","tags":["Exploit","Third Party Advisory"]},{"url":"https://www.oracle.com/security-alerts/alert-cve-2021-44228.html","source":"security@apache.org","tags":["Third Party Advisory"]},{"url":"https://www.oracle.com/security-alerts/cpuapr2022.html","source":"security@apache.org","tags":["Patch","Third Party Advisory"]},{"url":"https://www.oracle.com/security-alerts/cpujan2022.html","source":"security@apache.org","tags":["Patch","Third Party Advisory"]},{"url":"http://packetstormsecurity.com/files/165225/Apache-Log4j2-2.14.1-Remote-Code-Execution.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165260/VMware-Security-Advisory-2021-0028.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165261/Apache-Log4j2-2.14.1-Information-Disclosure.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Exploit","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165270/Apache-Log4j2-2.14.1-Remote-Code-Execution.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Exploit","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165281/Log4j2-Log4Shell-Regexes.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165282/Log4j-Payload-Generator.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165306/L4sh-Log4j-Remote-Code-Execution.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165307/Log4j-Remote-Code-Execution-Word-Bypassing.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165311/log4j-scan-Extensive-Scanner.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Broken Link","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165371/VMware-Security-Advisory-2021-0028.4.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Exploit","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165532/Log4Shell-HTTP-Header-Injection.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Exploit","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165642/VMware-vCenter-Server-Unauthenticated-Log4Shell-JNDI-Injection-Remote-Code-Execution.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Exploit","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/165673/UniFi-Network-Application-Unauthenticated-Log4Shell-Remote-Code-Execution.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Exploit","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/167794/Open-Xchange-App-Suite-7.10.x-Cross-Site-Scripting-Command-Injection.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/167917/MobileIron-Log4Shell-Remote-Command-Execution.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Exploit","Third Party Advisory","VDB Entry"]},{"url":"http://packetstormsecurity.com/files/171626/AD-Manager-Plus-7122-Remote-Code-Execution.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory","VDB Entry"]},{"url":"http://seclists.org/fulldisclosure/2022/Dec/2","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Exploit","Mailing List","Third Party Advisory"]},{"url":"http://seclists.org/fulldisclosure/2022/Jul/11","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Mailing List","Third Party Advisory"]},{"url":"http://seclists.org/fulldisclosure/2022/Mar/23","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Mailing List","Third Party Advisory"]},{"url":"http://www.openwall.com/lists/oss-security/2021/12/10/1","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Mailing List","Mitigation","Third Party Advisory"]},{"url":"http://www.openwall.com/lists/oss-security/2021/12/10/2","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Mailing List","Mitigation","Third Party Advisory"]},{"url":"http://www.openwall.com/lists/oss-security/2021/12/10/3","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Mailing List","Third Party Advisory"]},{"url":"http://www.openwall.com/lists/oss-security/2021/12/13/1","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Mailing List","Third Party Advisory"]},{"url":"http://www.openwall.com/lists/oss-security/2021/12/13/2","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Mailing List","Third Party Advisory"]},{"url":"http://www.openwall.com/lists/oss-security/2021/12/14/4","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Mailing List","Third Party Advisory"]},{"url":"http://www.openwall.com/lists/oss-security/2021/12/15/3","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Mailing List","Third Party Advisory"]},{"url":"https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory"]},{"url":"https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory"]},{"url":"https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory"]},{"url":"https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory"]},{"url":"https://github.com/cisagov/log4j-affected-db","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory"]},{"url":"https://github.com/cisagov/log4j-affected-db/blob/develop/SOFTWARE-LIST.md","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Broken Link","Product","US Government Resource"]},{"url":"https://github.com/nu11secur1ty/CVE-mitre/tree/main/CVE-2021-44228","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Exploit","Third Party Advisory"]},{"url":"https://lists.debian.org/debian-lts-announce/2021/12/msg00007.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Mailing List","Third Party Advisory"]},{"url":"https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/M5CSVUNV4HWZZXGOKNSK6L7RPM7BOKIB/","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Release Notes"]},{"url":"https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/VU57UJDCFIASIO35GC55JMKSRXJMCDFM/","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Release Notes"]},{"url":"https://logging.apache.org/log4j/2.x/security.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Release Notes","Vendor Advisory"]},{"url":"https://msrc-blog.microsoft.com/2021/12/11/microsofts-response-to-cve-2021-44228-apache-log4j2/","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Patch","Third Party Advisory","Vendor Advisory"]},{"url":"https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory"]},{"url":"https://security.netapp.com/advisory/ntap-20211210-0007/","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory"]},{"url":"https://support.apple.com/kb/HT213189","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory"]},{"url":"https://tools.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory"]},{"url":"https://twitter.com/kurtseifried/status/1469345530182455296","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Broken Link","Exploit","Third Party Advisory"]},{"url":"https://www.bentley.com/en/common-vulnerability-exposure/be-2022-0001","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory"]},{"url":"https://www.debian.org/security/2021/dsa-5020","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Mailing List","Third Party Advisory"]},{"url":"https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory"]},{"url":"https://www.kb.cert.org/vuls/id/930724","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory","US Government Resource"]},{"url":"https://www.nu11secur1ty.com/2021/12/cve-2021-44228.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Exploit","Third Party Advisory"]},{"url":"https://www.oracle.com/security-alerts/alert-cve-2021-44228.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Third Party Advisory"]},{"url":"https://www.oracle.com/security-alerts/cpuapr2022.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Patch","Third Party Advisory"]},{"url":"https://www.oracle.com/security-alerts/cpujan2022.html","source":"af854a3a-2127-422b-91ae-364da2661108","tags":["Patch","Third Party Advisory"]},{"url":"https://www.cisa.gov/known-exploited-vulnerabilities-catalog?field_cve=CVE-2021-44228","source":"134c704f-9b21-4f2e-91b3-4a467353bcc0","tags":["Third Party Advisory","US Government Resource"]}]}}]} From 8713712bf9d6d07f986999f62e04f2551a8c29c2 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Sun, 5 Jul 2026 18:42:57 -0400 Subject: [PATCH 05/15] feat(nadezhda): M4 ranking + digest + Markdown/JSON export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turn the aggregated store into a ranked, exportable digest of story clusters. - internal/rank: pure deterministic Score(Signals, cfg.Rank) — recency (exp half-life decay), cvss/10, kev, epss, velocity (cluster size/age), source weight, watchlist keyword. Every signal clamped to [0,1]; all weights + half_life + velocity_norm from config, no literals. recency/velocity are total (guard half_life<1 and norm<=0). Rank() scores each cluster and stable-sorts desc, tiebreak freshest. Research 06's two worked examples are the golden-order test (A ~0.99 strictly above B ~0.13). - internal/export: per-cluster (story) digest. Headline = freshest member, distinct sorted outlets, CVEs sorted KEV-first then CVSS. Markdown + deterministic JSON. - store.DigestClusters: 3-query per-cluster aggregation (clusters by since, member articles, union of CVE signals) with explicit ORDER BY; per-cluster CVE dedup via SELECT DISTINCT. - digest command: --top/--since/--format md|json/--out. Off the stub list. - Proven live: scrape -> digest --format md and json render correctly, per-cluster (FortiBleed shows 3 outlets). Suite offline + -race. One read-only audit agent; 0 Crit/High/Med, golden-order math verified; Low/Nit fixed in-phase (top<=0 count, pure-fn guards, ORDER BY, headline sanitize, 2-cluster attach test). Digest ranks per-cluster (Carter's choice), not per-article. --- .../cmd/nadezhda/digest.go | 101 ++++++++++ .../cmd/nadezhda/stubs.go | 1 - .../internal/export/export.go | 172 ++++++++++++++++++ .../internal/export/export_test.go | 104 +++++++++++ .../internal/rank/rank.go | 159 ++++++++++++++++ .../internal/rank/rank_test.go | 106 +++++++++++ .../internal/store/digest_test.go | 110 +++++++++++ .../internal/store/store.go | 116 ++++++++++++ 8 files changed, 868 insertions(+), 1 deletion(-) create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/digest.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/export/export.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/export/export_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/rank/rank.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/rank/rank_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/digest_test.go diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/digest.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/digest.go new file mode 100644 index 00000000..b4aed201 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/digest.go @@ -0,0 +1,101 @@ +// ©AngelaMos | 2026 +// digest.go + +package main + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/export" + "github.com/CarterPerez-dev/nadezhda/internal/rank" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const ( + defaultDigestTop = 20 + formatMarkdown = "md" + formatJSON = "json" + outFilePerm = 0o644 +) + +var ( + digestTop int + digestSince string + digestFormat string + digestOut string +) + +var digestCmd = &cobra.Command{ + Use: "digest", + Short: "Render a ranked digest of story clusters to Markdown or JSON", + RunE: runDigest, +} + +func init() { + digestCmd.Flags().IntVar(&digestTop, "top", defaultDigestTop, "show the top N ranked clusters") + digestCmd.Flags().StringVar(&digestSince, "since", "", "only clusters active within this window (e.g. 24h, 168h)") + digestCmd.Flags().StringVar(&digestFormat, "format", formatMarkdown, "output format: md or json") + digestCmd.Flags().StringVar(&digestOut, "out", "", "write to this file instead of stdout") + rootCmd.AddCommand(digestCmd) +} + +func runDigest(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + if digestFormat != formatMarkdown && digestFormat != formatJSON { + return fmt.Errorf("invalid --format %q: want md or json", digestFormat) + } + + var since int64 + now := time.Now() + if digestSince != "" { + d, err := time.ParseDuration(digestSince) + if err != nil { + return fmt.Errorf("invalid --since %q: %w", digestSince, err) + } + since = now.Add(-d).Unix() + } + + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + + clusters, err := st.DigestClusters(since) + if err != nil { + return err + } + scored := rank.Rank(clusters, cfg.Rank, cfg.Watchlist, now) + + shown := len(scored) + if digestTop > 0 && digestTop < shown { + shown = digestTop + } + + var rendered string + if digestFormat == formatJSON { + rendered, err = export.JSON(scored, digestTop) + if err != nil { + return err + } + } else { + rendered = export.Markdown(scored, digestTop) + } + + if digestOut != "" { + if err := os.WriteFile(digestOut, []byte(rendered), outFilePerm); err != nil { + return fmt.Errorf("write digest to %s: %w", digestOut, err) + } + fmt.Fprintf(cmd.OutOrStdout(), "wrote %d clusters to %s\n", shown, digestOut) + return nil + } + fmt.Fprint(cmd.OutOrStdout(), rendered) + return nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go index c1a4b829..e12b52fe 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go @@ -21,7 +21,6 @@ func init() { short string milestone string }{ - {"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"}, {"tui", "Browse aggregated news in an interactive terminal UI", "milestone M5"}, {"ideate", "Generate content angles from ranked clusters via an AI provider", "milestone M6"}, {"watch", "Run as a daemon, re-ingesting on an interval", "milestone M7"}, diff --git a/PROJECTS/intermediate/security-news-scraper/internal/export/export.go b/PROJECTS/intermediate/security-news-scraper/internal/export/export.go new file mode 100644 index 00000000..38e0a148 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/export/export.go @@ -0,0 +1,172 @@ +// ©AngelaMos | 2026 +// export.go + +package export + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/rank" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const ( + dateLayout = "2006-01-02" + noDate = "----------" + scoreRounder = 10000.0 + digestTitle = "# Nadezhda Digest" +) + +type CVEEntry struct { + ID string `json:"id"` + CVSS *float64 `json:"cvss,omitempty"` + EPSS *float64 `json:"epss,omitempty"` + KEV bool `json:"kev"` +} + +type Entry struct { + Rank int `json:"rank"` + Score float64 `json:"score"` + Headline string `json:"headline"` + URL string `json:"url"` + Outlets []string `json:"outlets"` + FirstSeen string `json:"first_seen"` + LastSeen string `json:"last_seen"` + CVEs []CVEEntry `json:"cves,omitempty"` +} + +type Digest struct { + Count int `json:"count"` + Entries []Entry `json:"entries"` +} + +func Build(scored []rank.Scored, top int) Digest { + limit := len(scored) + if top > 0 && top < limit { + limit = top + } + entries := make([]Entry, 0, limit) + for i := 0; i < limit; i++ { + s := scored[i] + h := headline(s.Cluster.Articles) + entries = append(entries, Entry{ + Rank: i + 1, + Score: roundScore(s.Score), + Headline: h.Title, + URL: h.CanonicalURL, + Outlets: outlets(s.Cluster.Articles), + FirstSeen: formatDate(s.Cluster.FirstSeen), + LastSeen: formatDate(s.Cluster.LastSeen), + CVEs: cveEntries(s.Cluster.CVEs), + }) + } + return Digest{Count: len(entries), Entries: entries} +} + +func JSON(scored []rank.Scored, top int) (string, error) { + b, err := json.MarshalIndent(Build(scored, top), "", " ") + if err != nil { + return "", fmt.Errorf("export json: %w", err) + } + return string(b), nil +} + +func Markdown(scored []rank.Scored, top int) string { + d := Build(scored, top) + var b strings.Builder + b.WriteString(digestTitle + "\n\n") + for _, e := range d.Entries { + fmt.Fprintf(&b, "## %d. %s\n", e.Rank, oneLine(e.Headline)) + fmt.Fprintf(&b, "score %.2f | %d outlet(s): %s\n", e.Score, len(e.Outlets), strings.Join(e.Outlets, ", ")) + for _, c := range e.CVEs { + fmt.Fprintf(&b, "- %s\n", cveLine(c)) + } + fmt.Fprintf(&b, "%s\n\n", e.URL) + } + return b.String() +} + +func headline(articles []store.DigestArticle) store.DigestArticle { + var best store.DigestArticle + for i, a := range articles { + if i == 0 || a.PublishedAt > best.PublishedAt || + (a.PublishedAt == best.PublishedAt && a.ID < best.ID) { + best = a + } + } + return best +} + +func outlets(articles []store.DigestArticle) []string { + seen := make(map[string]struct{}, len(articles)) + var out []string + for _, a := range articles { + if _, ok := seen[a.SourceName]; ok { + continue + } + seen[a.SourceName] = struct{}{} + out = append(out, a.SourceName) + } + sort.Strings(out) + return out +} + +func cveEntries(cves []store.DigestCVE) []CVEEntry { + out := make([]CVEEntry, 0, len(cves)) + for _, v := range cves { + out = append(out, CVEEntry{ID: v.ID, CVSS: v.CVSSScore, EPSS: v.EPSS, KEV: v.IsKEV}) + } + sort.SliceStable(out, func(a, b int) bool { + if out[a].KEV != out[b].KEV { + return out[a].KEV + } + if cvssOf(out[a]) != cvssOf(out[b]) { + return cvssOf(out[a]) > cvssOf(out[b]) + } + return out[a].ID < out[b].ID + }) + return out +} + +func cvssOf(c CVEEntry) float64 { + if c.CVSS == nil { + return -1 + } + return *c.CVSS +} + +func cveLine(c CVEEntry) string { + var parts []string + if c.KEV { + parts = append(parts, "KEV") + } + if c.CVSS != nil { + parts = append(parts, fmt.Sprintf("CVSS %.1f", *c.CVSS)) + } + if c.EPSS != nil { + parts = append(parts, fmt.Sprintf("EPSS %.2f", *c.EPSS)) + } + if len(parts) == 0 { + return c.ID + } + return fmt.Sprintf("%s (%s)", c.ID, strings.Join(parts, ", ")) +} + +func oneLine(s string) string { + return strings.Join(strings.Fields(s), " ") +} + +func roundScore(v float64) float64 { + return float64(int64(v*scoreRounder+0.5)) / scoreRounder +} + +func formatDate(unix int64) string { + if unix == 0 { + return noDate + } + return time.Unix(unix, 0).UTC().Format(dateLayout) +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/export/export_test.go b/PROJECTS/intermediate/security-news-scraper/internal/export/export_test.go new file mode 100644 index 00000000..88c499fc --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/export/export_test.go @@ -0,0 +1,104 @@ +// ©AngelaMos | 2026 +// export_test.go + +package export + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/CarterPerez-dev/nadezhda/internal/rank" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +func f(v float64) *float64 { return &v } + +func sample() []rank.Scored { + return []rank.Scored{ + { + Score: 0.9412, + Cluster: store.DigestCluster{ + Size: 3, FirstSeen: 1751600000, LastSeen: 1751700000, + Articles: []store.DigestArticle{ + {ID: 1, SourceName: "thehackernews", Title: "Old framing", CanonicalURL: "https://thn/1", PublishedAt: 100}, + {ID: 2, SourceName: "bleepingcomputer", Title: "Freshest framing", CanonicalURL: "https://bc/2", PublishedAt: 300}, + {ID: 3, SourceName: "bleepingcomputer", Title: "Mid framing", CanonicalURL: "https://bc/3", PublishedAt: 200}, + }, + CVEs: []store.DigestCVE{ + {ID: "CVE-2026-2", CVSSScore: f(7.5), IsKEV: false}, + {ID: "CVE-2026-1", CVSSScore: f(9.8), EPSS: f(0.91), IsKEV: true}, + }, + }, + }, + { + Score: 0.10, + Cluster: store.DigestCluster{ + Size: 1, FirstSeen: 1751000000, LastSeen: 1751000000, + Articles: []store.DigestArticle{{ID: 9, SourceName: "krebs", Title: "Quiet story", CanonicalURL: "https://k/9", PublishedAt: 50}}, + }, + }, + } +} + +func TestBuildHeadlineAndOutlets(t *testing.T) { + d := Build(sample(), 0) + if d.Count != 2 { + t.Fatalf("count = %d, want 2", d.Count) + } + e := d.Entries[0] + if e.Rank != 1 || e.Score != 0.9412 { + t.Errorf("rank/score = %d/%v", e.Rank, e.Score) + } + if e.Headline != "Freshest framing" { + t.Errorf("headline = %q, want the freshest member", e.Headline) + } + if e.URL != "https://bc/2" { + t.Errorf("url = %q, want the freshest member's", e.URL) + } + if len(e.Outlets) != 2 || e.Outlets[0] != "bleepingcomputer" || e.Outlets[1] != "thehackernews" { + t.Errorf("outlets = %v, want distinct + sorted", e.Outlets) + } + if e.CVEs[0].ID != "CVE-2026-1" { + t.Errorf("first cve = %q, want the KEV one first", e.CVEs[0].ID) + } +} + +func TestBuildTopLimit(t *testing.T) { + d := Build(sample(), 1) + if d.Count != 1 { + t.Errorf("top=1 should yield 1 entry, got %d", d.Count) + } +} + +func TestJSONValidAndStable(t *testing.T) { + out, err := JSON(sample(), 0) + if err != nil { + t.Fatal(err) + } + var back Digest + if err := json.Unmarshal([]byte(out), &back); err != nil { + t.Fatalf("output is not valid JSON: %v", err) + } + if back.Entries[0].Headline != "Freshest framing" { + t.Errorf("round-trip headline = %q", back.Entries[0].Headline) + } + if !back.Entries[0].CVEs[0].KEV { + t.Error("first cve should be the KEV one") + } +} + +func TestMarkdownContainsSignals(t *testing.T) { + md := Markdown(sample(), 0) + for _, want := range []string{ + "# Nadezhda Digest", + "## 1. Freshest framing", + "2 outlet(s): bleepingcomputer, thehackernews", + "CVE-2026-1 (KEV, CVSS 9.8, EPSS 0.91)", + "https://bc/2", + } { + if !strings.Contains(md, want) { + t.Errorf("markdown missing %q\n---\n%s", want, md) + } + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/rank/rank.go b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank.go new file mode 100644 index 00000000..04f2bf33 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank.go @@ -0,0 +1,159 @@ +// ©AngelaMos | 2026 +// rank.go + +package rank + +import ( + "math" + "sort" + "strings" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const ( + cvssMax = 10.0 + minVelocityWindowHours = 1.0 + secondsPerHour = 3600.0 +) + +type Signals struct { + AgeHours float64 + MaxCVSS float64 + KEV bool + MaxEPSS float64 + ClusterSize int + ClusterAgeHours float64 + SourceWeight float64 + KeywordMatch bool +} + +func Score(s Signals, cfg config.Rank) float64 { + w := cfg.Weights + return w.Recency*recency(s.AgeHours, cfg.HalfLifeHours) + + w.CVSS*clamp01(s.MaxCVSS/cvssMax) + + w.KEV*boolScore(s.KEV) + + w.EPSS*clamp01(s.MaxEPSS) + + w.Velocity*velocity(s.ClusterSize, s.ClusterAgeHours, cfg.VelocityNorm) + + w.Source*clamp01(s.SourceWeight) + + w.Keyword*boolScore(s.KeywordMatch) +} + +func recency(ageHours float64, halfLifeHours int) float64 { + if ageHours < 0 { + ageHours = 0 + } + if halfLifeHours < 1 { + halfLifeHours = 1 + } + return math.Exp(-math.Ln2 * ageHours / float64(halfLifeHours)) +} + +func velocity(size int, ageHours, norm float64) float64 { + if size <= 1 || norm <= 0 { + return 0 + } + if ageHours < minVelocityWindowHours { + ageHours = minVelocityWindowHours + } + return clamp01((float64(size) / ageHours) / norm) +} + +func boolScore(b bool) float64 { + if b { + return 1 + } + return 0 +} + +func clamp01(v float64) float64 { + if v < 0 { + return 0 + } + if v > 1 { + return 1 + } + return v +} + +type Scored struct { + Cluster store.DigestCluster + Score float64 +} + +func Rank(clusters []store.DigestCluster, cfg config.Rank, watchlist []string, now time.Time) []Scored { + out := make([]Scored, len(clusters)) + for i, c := range clusters { + out[i] = Scored{Cluster: c, Score: Score(signalsFor(c, watchlist, now), cfg)} + } + sort.SliceStable(out, func(a, b int) bool { + if out[a].Score != out[b].Score { + return out[a].Score > out[b].Score + } + return out[a].Cluster.LastSeen > out[b].Cluster.LastSeen + }) + return out +} + +func signalsFor(c store.DigestCluster, watchlist []string, now time.Time) Signals { + s := Signals{ + AgeHours: hoursSince(c.LastSeen, now), + ClusterSize: c.Size, + ClusterAgeHours: float64(c.LastSeen-c.FirstSeen) / secondsPerHour, + SourceWeight: maxSourceWeight(c.Articles), + KeywordMatch: matchesWatchlist(c, watchlist), + } + for _, v := range c.CVEs { + if v.CVSSScore != nil && *v.CVSSScore > s.MaxCVSS { + s.MaxCVSS = *v.CVSSScore + } + if v.EPSS != nil && *v.EPSS > s.MaxEPSS { + s.MaxEPSS = *v.EPSS + } + if v.IsKEV { + s.KEV = true + } + } + return s +} + +func hoursSince(unix int64, now time.Time) float64 { + return float64(now.Unix()-unix) / secondsPerHour +} + +func maxSourceWeight(articles []store.DigestArticle) float64 { + var max float64 + for _, a := range articles { + if a.SourceWeight > max { + max = a.SourceWeight + } + } + return max +} + +func matchesWatchlist(c store.DigestCluster, watchlist []string) bool { + if len(watchlist) == 0 { + return false + } + var sb strings.Builder + for _, a := range c.Articles { + sb.WriteString(strings.ToLower(a.Title)) + sb.WriteByte(' ') + } + for _, v := range c.CVEs { + sb.WriteString(strings.ToLower(v.ID)) + sb.WriteByte(' ') + } + hay := sb.String() + for _, term := range watchlist { + if term == "" { + continue + } + if strings.Contains(hay, strings.ToLower(term)) { + return true + } + } + return false +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/rank/rank_test.go b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank_test.go new file mode 100644 index 00000000..1748bc13 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/rank/rank_test.go @@ -0,0 +1,106 @@ +// ©AngelaMos | 2026 +// rank_test.go + +package rank + +import ( + "math" + "testing" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +func f(v float64) *float64 { return &v } + +func TestScoreGoldenOrder(t *testing.T) { + cfg := config.Default().Rank + + a := Signals{ + AgeHours: 2, MaxCVSS: 9.8, KEV: true, MaxEPSS: 0.97, + ClusterSize: 5, ClusterAgeHours: 6, SourceWeight: 1.0, KeywordMatch: true, + } + b := Signals{ + AgeHours: 120, MaxCVSS: 4.3, KEV: false, MaxEPSS: 0.02, + ClusterSize: 1, ClusterAgeHours: 0, SourceWeight: 0.6, KeywordMatch: false, + } + + sa, sb := Score(a, cfg), Score(b, cfg) + if !(sa > sb) { + t.Fatalf("expected A (%.4f) to rank strictly above B (%.4f)", sa, sb) + } + if sa < 0.95 { + t.Errorf("A score = %.4f, want ~0.98", sa) + } + if sb > 0.20 { + t.Errorf("B score = %.4f, want ~0.14", sb) + } +} + +func TestRecencyDecay(t *testing.T) { + if got := recency(0, 48); math.Abs(got-1.0) > 1e-9 { + t.Errorf("recency(0) = %v, want 1.0", got) + } + if got := recency(48, 48); math.Abs(got-0.5) > 1e-9 { + t.Errorf("recency(one half-life) = %v, want 0.5", got) + } + if got := recency(-5, 48); got != 1.0 { + t.Errorf("negative age should clamp to 1.0, got %v", got) + } +} + +func TestVelocity(t *testing.T) { + if v := velocity(1, 0, 0.5); v != 0 { + t.Errorf("size-1 velocity = %v, want 0", v) + } + if v := velocity(6, 12, 0.5); v != 1.0 { + t.Errorf("size-6 in 12h should saturate, got %v", v) + } + if v := velocity(3, 0, 0.5); v != 1.0 { + t.Errorf("burst (age 1e-9 { + t.Errorf("slow cluster velocity = %v, want 0.04", v) + } +} + +func TestRankOrdersByScore(t *testing.T) { + now := time.Unix(1_000_000, 0) + hot := store.DigestCluster{ + ClusterID: 1, Size: 4, FirstSeen: now.Unix() - 3600, LastSeen: now.Unix() - 60, + Articles: []store.DigestArticle{{SourceName: "a", SourceWeight: 1.0, PublishedAt: now.Unix() - 60}}, + CVEs: []store.DigestCVE{{ID: "CVE-2026-1", CVSSScore: f(9.9), EPSS: f(0.95), IsKEV: true}}, + } + cold := store.DigestCluster{ + ClusterID: 2, Size: 1, FirstSeen: now.Unix() - 500*3600, LastSeen: now.Unix() - 500*3600, + Articles: []store.DigestArticle{{SourceName: "b", SourceWeight: 0.5, PublishedAt: now.Unix() - 500*3600}}, + } + + scored := Rank([]store.DigestCluster{cold, hot}, config.Default().Rank, nil, now) + if scored[0].Cluster.ClusterID != 1 { + t.Errorf("hot KEV cluster should rank first, got cluster %d", scored[0].Cluster.ClusterID) + } + if !(scored[0].Score > scored[1].Score) { + t.Errorf("scores not descending: %.4f then %.4f", scored[0].Score, scored[1].Score) + } +} + +func TestKeywordMatch(t *testing.T) { + c := store.DigestCluster{ + Articles: []store.DigestArticle{{Title: "Fortinet FortiOS flaw exploited"}}, + CVEs: []store.DigestCVE{{ID: "CVE-2026-1"}}, + } + if !matchesWatchlist(c, []string{"fortinet"}) { + t.Error("case-insensitive watchlist term should match title") + } + if !matchesWatchlist(c, []string{"CVE-2026-1"}) { + t.Error("watchlist should match against CVE ids") + } + if matchesWatchlist(c, []string{"cisco"}) { + t.Error("non-matching term should not match") + } + if matchesWatchlist(c, nil) { + t.Error("empty watchlist should not match") + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/digest_test.go b/PROJECTS/intermediate/security-news-scraper/internal/store/digest_test.go new file mode 100644 index 00000000..b6230eba --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/digest_test.go @@ -0,0 +1,110 @@ +// ©AngelaMos | 2026 +// digest_test.go + +package store + +import "testing" + +func TestDigestClustersAggregates(t *testing.T) { + s := openTemp(t) + + src1, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1.0, Enabled: true}) + src2, _ := s.UpsertSource(SourceInput{Name: "b", URL: "https://b/f", Type: "rss", Weight: 0.8, Enabled: true}) + + a1, _ := s.InsertArticle(Article{SourceID: src1, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "Story one", PublishedAt: 1000}) + a2, _ := s.InsertArticle(Article{SourceID: src2, CanonicalURL: "https://b/1", ContentHash: "c2", TitleHash: "t2", Title: "Story two", PublishedAt: 2000}) + + if err := s.UpsertCVEStub("CVE-2026-1"); err != nil { + t.Fatal(err) + } + score := 9.8 + if err := s.UpdateCVEEnrichment(CVE{ID: "CVE-2026-1", CVSSScore: &score, IsKEV: true, EnrichedAt: 1, EnrichStatus: EnrichStatusOK}); err != nil { + t.Fatal(err) + } + if err := s.LinkArticleCVE(a1, "CVE-2026-1"); err != nil { + t.Fatal(err) + } + if err := s.LinkArticleCVE(a2, "CVE-2026-1"); err != nil { + t.Fatal(err) + } + + if err := s.ReplaceClusters([]ClusterRow{ + {Key: "1", Members: []int64{a1, a2}, FirstSeen: 1000, LastSeen: 2000}, + }); err != nil { + t.Fatal(err) + } + + clusters, err := s.DigestClusters(0) + if err != nil { + t.Fatal(err) + } + if len(clusters) != 1 { + t.Fatalf("clusters = %d, want 1", len(clusters)) + } + c := clusters[0] + if len(c.Articles) != 2 { + t.Errorf("articles = %d, want 2", len(c.Articles)) + } + if len(c.CVEs) != 1 { + t.Fatalf("cves = %d, want 1 (deduped across both articles)", len(c.CVEs)) + } + if !c.CVEs[0].IsKEV || c.CVEs[0].CVSSScore == nil || *c.CVEs[0].CVSSScore != 9.8 { + t.Errorf("cve signals not aggregated: %+v", c.CVEs[0]) + } +} + +func TestDigestClustersAttachToCorrectCluster(t *testing.T) { + s := openTemp(t) + src, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1, Enabled: true}) + + a1, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "one"}) + a2, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/2", ContentHash: "c2", TitleHash: "t2", Title: "two"}) + + for _, id := range []string{"CVE-2026-1", "CVE-2026-2"} { + if err := s.UpsertCVEStub(id); err != nil { + t.Fatal(err) + } + } + _ = s.LinkArticleCVE(a1, "CVE-2026-1") + _ = s.LinkArticleCVE(a2, "CVE-2026-2") + + if err := s.ReplaceClusters([]ClusterRow{ + {Key: "1", Members: []int64{a1}, FirstSeen: 1000, LastSeen: 1000}, + {Key: "2", Members: []int64{a2}, FirstSeen: 2000, LastSeen: 2000}, + }); err != nil { + t.Fatal(err) + } + + clusters, err := s.DigestClusters(0) + if err != nil { + t.Fatal(err) + } + if len(clusters) != 2 { + t.Fatalf("clusters = %d, want 2", len(clusters)) + } + byArticle := map[int64]DigestCluster{} + for _, c := range clusters { + if len(c.Articles) != 1 || len(c.CVEs) != 1 { + t.Fatalf("cluster %d: articles=%d cves=%d, want 1/1", c.ClusterID, len(c.Articles), len(c.CVEs)) + } + byArticle[c.Articles[0].ID] = c + } + if byArticle[a1].CVEs[0].ID != "CVE-2026-1" || byArticle[a2].CVEs[0].ID != "CVE-2026-2" { + t.Error("CVEs attached to the wrong cluster") + } +} + +func TestDigestClustersSinceFilter(t *testing.T) { + s := openTemp(t) + src, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1, Enabled: true}) + a1, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "old"}) + if err := s.ReplaceClusters([]ClusterRow{{Key: "1", Members: []int64{a1}, FirstSeen: 500, LastSeen: 500}}); err != nil { + t.Fatal(err) + } + if got, _ := s.DigestClusters(1000); len(got) != 0 { + t.Errorf("cluster with last_seen 500 should be excluded by since=1000, got %d", len(got)) + } + if got, _ := s.DigestClusters(0); len(got) != 1 { + t.Errorf("since=0 should include it, got %d", len(got)) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go index b65e681e..e28bb698 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go @@ -295,6 +295,122 @@ func (s *Store) ArticlesForCVE(id string) ([]ArticleSummary, error) { return scanArticleSummaries(rows) } +type DigestArticle struct { + ID int64 + SourceName string + SourceWeight float64 + Title string + CanonicalURL string + PublishedAt int64 +} + +type DigestCVE struct { + ID string + CVSSScore *float64 + EPSS *float64 + IsKEV bool +} + +type DigestCluster struct { + ClusterID int64 + Key string + Size int + FirstSeen int64 + LastSeen int64 + Articles []DigestArticle + CVEs []DigestCVE +} + +func (s *Store) DigestClusters(since int64) ([]DigestCluster, error) { + byID, order, err := s.digestClusterRows(since) + if err != nil { + return nil, err + } + if err := s.digestAttachArticles(byID); err != nil { + return nil, err + } + if err := s.digestAttachCVEs(byID); err != nil { + return nil, err + } + out := make([]DigestCluster, 0, len(order)) + for _, id := range order { + out = append(out, *byID[id]) + } + return out, nil +} + +func (s *Store) digestClusterRows(since int64) (map[int64]*DigestCluster, []int64, error) { + rows, err := s.db.Query(` + SELECT id, cluster_key, size, first_seen, last_seen + FROM clusters WHERE last_seen >= ? ORDER BY id`, since) + if err != nil { + return nil, nil, fmt.Errorf("digest clusters: %w", err) + } + defer rows.Close() + byID := make(map[int64]*DigestCluster) + var order []int64 + for rows.Next() { + var dc DigestCluster + if err := rows.Scan(&dc.ClusterID, &dc.Key, &dc.Size, &dc.FirstSeen, &dc.LastSeen); err != nil { + return nil, nil, fmt.Errorf("digest clusters: scan: %w", err) + } + clone := dc + byID[dc.ClusterID] = &clone + order = append(order, dc.ClusterID) + } + return byID, order, rows.Err() +} + +func (s *Store) digestAttachArticles(byID map[int64]*DigestCluster) error { + rows, err := s.db.Query(` + SELECT cm.cluster_id, a.id, s.name, s.weight, a.title, a.canonical_url, a.published_at + FROM cluster_members cm + JOIN articles a ON a.id = cm.article_id + JOIN sources s ON s.id = a.source_id + ORDER BY cm.cluster_id, a.id`) + if err != nil { + return fmt.Errorf("digest articles: %w", err) + } + defer rows.Close() + for rows.Next() { + var clusterID int64 + var a DigestArticle + if err := rows.Scan(&clusterID, &a.ID, &a.SourceName, &a.SourceWeight, &a.Title, &a.CanonicalURL, &a.PublishedAt); err != nil { + return fmt.Errorf("digest articles: scan: %w", err) + } + if dc, ok := byID[clusterID]; ok { + dc.Articles = append(dc.Articles, a) + } + } + return rows.Err() +} + +func (s *Store) digestAttachCVEs(byID map[int64]*DigestCluster) error { + rows, err := s.db.Query(` + SELECT DISTINCT cm.cluster_id, c.id, c.cvss_score, c.epss, c.is_kev + FROM cluster_members cm + JOIN article_cves ac ON ac.article_id = cm.article_id + JOIN cves c ON c.id = ac.cve_id + ORDER BY cm.cluster_id, c.id`) + if err != nil { + return fmt.Errorf("digest cves: %w", err) + } + defer rows.Close() + for rows.Next() { + var clusterID int64 + var v DigestCVE + var isKEV int + if err := rows.Scan(&clusterID, &v.ID, &v.CVSSScore, &v.EPSS, &isKEV); err != nil { + return fmt.Errorf("digest cves: scan: %w", err) + } + v.IsKEV = isKEV != 0 + if dc, ok := byID[clusterID]; ok { + dc.CVEs = append(dc.CVEs, v) + } + } + return rows.Err() +} + func (s *Store) UpsertCVEStub(id string) error { _, err := s.db.Exec(`INSERT INTO cves (id) VALUES (?) ON CONFLICT(id) DO NOTHING`, id) if err != nil { From 26651547e104492b38ce11094eae1ba7d61820cf Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Mon, 6 Jul 2026 01:43:56 -0400 Subject: [PATCH 06/15] feat(nadezhda): M5 bubbletea TUI browser (cybercore) + open-in-browser Interactive terminal UI over the ranked story clusters. Rave Maximal cybercore theming: cyan-to-magenta spectrum score bars, severity-reactive rows, hot KEV chips, cool-ramp outlet dots, and a scrollable detail dossier with per-outlet links and full CVE cards (CVSS meter, KEV / ransomware, EPSS, CWE, vector, description). Reuses store.DigestClusters and rank.Rank verbatim and preloads full store.CVE records for detail. - internal/tui: model/update/view state machine with an injected Loader and opener, so Update is fully unit-testable without a terminal or DB (nav / clamp / open / back / quit / resize / empty-store / too-small). - o opens the selected story in the default browser, keyboard-only, cross-platform, and refuses non-http urls. - tui command replaces the M5 stub; supports --since. - deps: charmbracelet bubbletea v1.3.10 + bubbles v1.0.0 + lipgloss v1.1.0. --- .../cmd/nadezhda/stubs.go | 1 - .../security-news-scraper/cmd/nadezhda/tui.go | 75 ++++ .../intermediate/security-news-scraper/go.mod | 20 ++ .../intermediate/security-news-scraper/go.sum | 43 +++ .../internal/tui/browser.go | 31 ++ .../internal/tui/detail.go | 202 +++++++++++ .../internal/tui/format.go | 197 +++++++++++ .../internal/tui/format_test.go | 173 ++++++++++ .../internal/tui/keys.go | 54 +++ .../internal/tui/list.go | 222 ++++++++++++ .../internal/tui/model.go | 253 ++++++++++++++ .../internal/tui/model_test.go | 322 ++++++++++++++++++ .../internal/tui/theme.go | 151 ++++++++ .../internal/tui/view.go | 159 +++++++++ 14 files changed, 1902 insertions(+), 1 deletion(-) create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/tui.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/tui/browser.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/tui/detail.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/tui/format.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/tui/format_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/tui/keys.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/tui/list.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/tui/model.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/tui/model_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/tui/theme.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/tui/view.go diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go index e12b52fe..5dbe4177 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go @@ -21,7 +21,6 @@ func init() { short string milestone string }{ - {"tui", "Browse aggregated news in an interactive terminal UI", "milestone M5"}, {"ideate", "Generate content angles from ranked clusters via an AI provider", "milestone M6"}, {"watch", "Run as a daemon, re-ingesting on an interval", "milestone M7"}, } diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/tui.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/tui.go new file mode 100644 index 00000000..093b87ac --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/tui.go @@ -0,0 +1,75 @@ +// ©AngelaMos | 2026 +// tui.go + +package main + +import ( + "fmt" + "time" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/rank" + "github.com/CarterPerez-dev/nadezhda/internal/store" + "github.com/CarterPerez-dev/nadezhda/internal/tui" +) + +var tuiSince string + +var tuiCmd = &cobra.Command{ + Use: "tui", + Short: "Browse aggregated news in an interactive terminal UI", + RunE: runTUI, +} + +func init() { + tuiCmd.Flags().StringVar(&tuiSince, "since", "", "only clusters active within this window (e.g. 24h, 168h)") + rootCmd.AddCommand(tuiCmd) +} + +func runTUI(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + + now := time.Now() + var since int64 + if tuiSince != "" { + d, err := time.ParseDuration(tuiSince) + if err != nil { + return fmt.Errorf("invalid --since %q: %w", tuiSince, err) + } + since = now.Add(-d).Unix() + } + + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + + loader := func() (tui.Data, error) { + clusters, err := st.DigestClusters(since) + if err != nil { + return tui.Data{}, err + } + scored := rank.Rank(clusters, cfg.Rank, cfg.Watchlist, now) + detail := make(map[string]store.CVE) + for _, s := range scored { + for _, v := range s.Cluster.CVEs { + if _, ok := detail[v.ID]; ok { + continue + } + full, err := st.GetCVE(v.ID) + if err != nil { + continue + } + detail[v.ID] = full + } + } + return tui.Data{Scored: scored, CVEDetail: detail}, nil + } + + return tui.Run(loader) +} diff --git a/PROJECTS/intermediate/security-news-scraper/go.mod b/PROJECTS/intermediate/security-news-scraper/go.mod index 7a74c19b..382c7a78 100644 --- a/PROJECTS/intermediate/security-news-scraper/go.mod +++ b/PROJECTS/intermediate/security-news-scraper/go.mod @@ -4,6 +4,9 @@ go 1.25.0 require ( github.com/PuerkitoBio/goquery v1.12.0 + github.com/charmbracelet/bubbles v1.0.0 + github.com/charmbracelet/bubbletea v1.3.10 + github.com/charmbracelet/lipgloss v1.1.0 github.com/mmcdole/gofeed v1.3.0 github.com/spf13/cobra v1.10.2 github.com/temoto/robotstxt v1.1.2 @@ -15,17 +18,34 @@ require ( require ( github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/charmbracelet/colorprofile v0.4.1 // indirect + github.com/charmbracelet/x/ansi v0.11.6 // indirect + github.com/charmbracelet/x/cellbuf v0.0.15 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect + github.com/clipperhouse/displaywidth v0.9.0 // indirect + github.com/clipperhouse/stringish v0.1.1 // indirect + github.com/clipperhouse/uax29/v2 v2.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.19 // indirect github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/termenv v0.16.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.9 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/sys v0.44.0 // indirect golang.org/x/text v0.35.0 // indirect diff --git a/PROJECTS/intermediate/security-news-scraper/go.sum b/PROJECTS/intermediate/security-news-scraper/go.sum index d61e7395..7cf2a882 100644 --- a/PROJECTS/intermediate/security-news-scraper/go.sum +++ b/PROJECTS/intermediate/security-news-scraper/go.sum @@ -2,12 +2,36 @@ github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc= +github.com/charmbracelet/bubbles v1.0.0/go.mod h1:9d/Zd5GdnauMI5ivUIVisuEm3ave1XwXtD1ckyV6r3E= +github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= +github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= +github.com/charmbracelet/colorprofile v0.4.1 h1:a1lO03qTrSIRaK8c3JRxJDZOvhvIeSco3ej+ngLk1kk= +github.com/charmbracelet/colorprofile v0.4.1/go.mod h1:U1d9Dljmdf9DLegaJ0nGZNJvoXAhayhmidOdcBwAvKk= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/x/ansi v0.11.6 h1:GhV21SiDz/45W9AnV2R61xZMRri5NlLnl6CVF7ihZW8= +github.com/charmbracelet/x/ansi v0.11.6/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ= +github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= +github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q= +github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= +github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+y9OLhMtAtA= +github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA= +github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= +github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= +github.com/clipperhouse/uax29/v2 v2.5.0 h1:x7T0T4eTHDONxFJsL94uKNKPHrclyFI0lm7+w94cO8U= +github.com/clipperhouse/uax29/v2 v2.5.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= @@ -20,8 +44,14 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= +github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= +github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk= @@ -31,12 +61,20 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= @@ -48,6 +86,8 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -56,6 +96,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -86,6 +128,7 @@ golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/browser.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/browser.go new file mode 100644 index 00000000..5de543cf --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/browser.go @@ -0,0 +1,31 @@ +// ©AngelaMos | 2026 +// browser.go + +package tui + +import ( + "fmt" + "net/url" + "os/exec" + "runtime" +) + +func openURL(target string) error { + u, err := url.Parse(target) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") { + return fmt.Errorf("refusing to open non-http url: %q", target) + } + name, args := openerCommand(target) + return exec.Command(name, args...).Start() +} + +func openerCommand(target string) (string, []string) { + switch runtime.GOOS { + case "windows": + return "rundll32", []string{"url.dll,FileProtocolHandler", target} + case "darwin": + return "open", []string{target} + default: + return "xdg-open", []string{target} + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/detail.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/detail.go new file mode 100644 index 00000000..854bd490 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/detail.go @@ -0,0 +1,202 @@ +// ©AngelaMos | 2026 +// detail.go + +package tui + +import ( + "fmt" + "sort" + "strings" + + "github.com/charmbracelet/lipgloss" + + "github.com/CarterPerez-dev/nadezhda/internal/rank" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const ( + cvssMaxScore = 10.0 + nvdBaseURL = "https://nvd.nist.gov/vuln/detail/" + detailIndent = 3 + metaDivider = " · " +) + +func (m Model) renderDetailBody() string { + s := m.selected() + c := s.Cluster + w := m.bodyWidth() + t := m.theme + indent := strings.Repeat(" ", detailIndent) + var b strings.Builder + + b.WriteString(t.HeadlineSel.Width(w).Render(headlineOf(c))) + b.WriteString("\n") + b.WriteString(m.detailMeta(s)) + b.WriteString("\n\n") + + b.WriteString(m.sectionHeader("OUTLETS", w)) + b.WriteString("\n") + for _, a := range sortedArticles(c) { + b.WriteString(m.renderArticle(a, w, indent)) + } + + cves := m.sortedDetailCVEs(c) + if len(cves) > 0 { + b.WriteString(m.sectionHeader("VULNERABILITIES", w)) + b.WriteString("\n") + for _, v := range cves { + b.WriteString(m.renderCVE(v, w, indent)) + } + } + return strings.TrimRight(b.String(), "\n") +} + +func (m Model) detailMeta(s rank.Scored) string { + t := m.theme + c := s.Cluster + parts := []string{ + t.spectrumBar(s.Score, colScoreBarW) + " " + t.Meta.Bold(true).Render(fmt.Sprintf("%.2f", s.Score)), + t.Muted.Render(fmt.Sprintf("%d outlets", outletCount(c))), + } + if mx := clusterMaxCVSS(c); mx != nil { + b := cvssBand(mx) + parts = append(parts, t.Muted.Render("CVSS ")+t.bandFG(b).Bold(true).Render(cvssString(mx))) + } + if me := clusterMaxEPSS(c); me != nil { + b := epssBand(me) + parts = append(parts, t.Muted.Render("EPSS ")+t.bandFG(b).Bold(true).Render(epssString(me))) + } + parts = append(parts, + t.Muted.Render("first "+relativeAge(c.FirstSeen, m.now)), + t.Muted.Render("last "+relativeAge(c.LastSeen, m.now)), + ) + return strings.Join(parts, t.Dim.Render(metaDivider)) +} + +func (m Model) sectionHeader(title string, w int) string { + t := m.theme + head := t.fg(colorViolet).Bold(true).Render(glyphSelected + " " + title + " ") + n := w - lipgloss.Width(head) + if n < 0 { + n = 0 + } + return head + t.Dim.Render(strings.Repeat(ruleBody, n)) +} + +func (m Model) renderArticle(a store.DigestArticle, w int, indent string) string { + t := m.theme + head := t.fg(colorCyan).Bold(true).Render(a.SourceName) + + t.Dim.Render(metaDivider) + t.Muted.Render(relativeAge(a.PublishedAt, m.now)) + title := m.wrapIndent(a.Title, w, t.Text) + url := indent + t.Link.Render(truncate(a.CanonicalURL, w-detailIndent)) + return head + "\n" + title + "\n" + url + "\n\n" +} + +func (m Model) renderCVE(v store.CVE, w int, indent string) string { + t := m.theme + bnd := cvssBand(v.CVSSScore) + var b strings.Builder + + id := t.fg(colorCyan).Bold(true).Render(v.ID) + if v.IsKEV { + id += " " + t.chip("KEV", colorMagenta) + } + if v.KEVRansomware { + id += " " + t.chip("RANSOMWARE", colorMagenta) + } + id += " " + t.bandFG(bnd).Bold(true).Render(bandLabel(bnd)) + b.WriteString(id + "\n") + + if v.CVSSScore != nil { + line := indent + t.Label.Render("CVSS ") + + t.bandBar(bnd, *v.CVSSScore/cvssMaxScore, colScoreBarW) + " " + + t.bandFG(bnd).Bold(true).Render(cvssString(v.CVSSScore)) + if v.CVSSSeverity != "" { + line += " " + t.Muted.Render(v.CVSSSeverity) + } + if v.CVSSVersion != "" { + line += t.Dim.Render(" (v" + v.CVSSVersion + ")") + } + b.WriteString(line + "\n") + } + + if v.EPSS != nil { + eb := epssBand(v.EPSS) + line := indent + t.Label.Render("EPSS ") + t.bandFG(eb).Bold(true).Render(epssString(v.EPSS)) + if v.EPSSPercentile != nil { + line += t.Dim.Render(" (percentile ") + t.Muted.Render(epssString(v.EPSSPercentile)) + t.Dim.Render(")") + } + b.WriteString(line + "\n") + } + + var meta []string + if v.CWE != "" { + meta = append(meta, t.Label.Render("CWE ")+t.Muted.Render(v.CWE)) + } + if v.CVSSVector != "" { + meta = append(meta, t.Dim.Render(v.CVSSVector)) + } + if len(meta) > 0 { + b.WriteString(indent + strings.Join(meta, t.Dim.Render(metaDivider)) + "\n") + } + + if strings.TrimSpace(v.Description) != "" { + b.WriteString(m.wrapIndent(v.Description, w, t.Muted) + "\n") + } + b.WriteString(indent + t.Link.Render(nvdBaseURL+v.ID) + "\n\n") + return b.String() +} + +func (m Model) wrapIndent(s string, w int, style lipgloss.Style) string { + width := w - detailIndent + if width < 1 { + width = 1 + } + wrapped := style.Width(width).Render(strings.TrimSpace(s)) + pad := strings.Repeat(" ", detailIndent) + lines := strings.Split(wrapped, "\n") + for i := range lines { + lines[i] = pad + lines[i] + } + return strings.Join(lines, "\n") +} + +func sortedArticles(c store.DigestCluster) []store.DigestArticle { + out := make([]store.DigestArticle, len(c.Articles)) + copy(out, c.Articles) + sort.SliceStable(out, func(i, j int) bool { + if out[i].PublishedAt != out[j].PublishedAt { + return out[i].PublishedAt > out[j].PublishedAt + } + return out[i].ID < out[j].ID + }) + return out +} + +func (m Model) sortedDetailCVEs(c store.DigestCluster) []store.CVE { + out := make([]store.CVE, 0, len(c.CVEs)) + for _, dc := range c.CVEs { + if full, ok := m.cveDetail[dc.ID]; ok { + out = append(out, full) + continue + } + out = append(out, store.CVE{ID: dc.ID, CVSSScore: dc.CVSSScore, EPSS: dc.EPSS, IsKEV: dc.IsKEV}) + } + sort.SliceStable(out, func(i, j int) bool { + if out[i].IsKEV != out[j].IsKEV { + return out[i].IsKEV + } + if ci, cj := scoreOf(out[i].CVSSScore), scoreOf(out[j].CVSSScore); ci != cj { + return ci > cj + } + return out[i].ID < out[j].ID + }) + return out +} + +func scoreOf(p *float64) float64 { + if p == nil { + return -1 + } + return *p +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/format.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/format.go new file mode 100644 index 00000000..790147a3 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/format.go @@ -0,0 +1,197 @@ +// ©AngelaMos | 2026 +// format.go + +package tui + +import ( + "fmt" + "strings" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +type band int + +const ( + bandNone band = iota + bandLow + bandMedium + bandHigh + bandCritical +) + +const ( + cvssCritical = 9.0 + cvssHigh = 7.0 + cvssMedium = 4.0 + + epssHot = 0.5 + epssWarm = 0.1 + + secsPerMinute = 60 + secsPerHour = 3600 + secsPerDay = 86400 + + ellipsis = "…" + emptyMarker = "───" + naMarker = "—" +) + +func clamp01(v float64) float64 { + if v < 0 { + return 0 + } + if v > 1 { + return 1 + } + return v +} + +func cvssBand(score *float64) band { + if score == nil { + return bandNone + } + switch v := *score; { + case v >= cvssCritical: + return bandCritical + case v >= cvssHigh: + return bandHigh + case v >= cvssMedium: + return bandMedium + case v > 0: + return bandLow + default: + return bandNone + } +} + +func bandLabel(b band) string { + switch b { + case bandCritical: + return "CRITICAL" + case bandHigh: + return "HIGH" + case bandMedium: + return "MEDIUM" + case bandLow: + return "LOW" + default: + return naMarker + } +} + +func clusterHasKEV(c store.DigestCluster) bool { + for _, v := range c.CVEs { + if v.IsKEV { + return true + } + } + return false +} + +func clusterMaxCVSS(c store.DigestCluster) *float64 { + var best *float64 + for _, v := range c.CVEs { + if v.CVSSScore == nil { + continue + } + if best == nil || *v.CVSSScore > *best { + best = v.CVSSScore + } + } + return best +} + +func clusterMaxEPSS(c store.DigestCluster) *float64 { + var best *float64 + for _, v := range c.CVEs { + if v.EPSS == nil { + continue + } + if best == nil || *v.EPSS > *best { + best = v.EPSS + } + } + return best +} + +func clusterBand(c store.DigestCluster) band { + if clusterHasKEV(c) { + return bandCritical + } + return cvssBand(clusterMaxCVSS(c)) +} + +func epssBand(p *float64) band { + if p == nil { + return bandNone + } + switch v := *p; { + case v >= epssHot: + return bandCritical + case v >= epssWarm: + return bandHigh + case v > 0: + return bandMedium + default: + return bandNone + } +} + +func cvssString(score *float64) string { + if score == nil { + return naMarker + } + return fmt.Sprintf("%.1f", *score) +} + +func epssString(p *float64) string { + if p == nil { + return naMarker + } + return fmt.Sprintf("%.1f%%", clamp01(*p)*100) +} + +func relativeAge(unix int64, now time.Time) string { + if unix <= 0 { + return naMarker + } + secs := now.Unix() - unix + if secs < 0 { + secs = 0 + } + switch { + case secs < secsPerMinute: + return "just now" + case secs < secsPerHour: + return fmt.Sprintf("%dm ago", secs/secsPerMinute) + case secs < secsPerDay: + return fmt.Sprintf("%dh ago", secs/secsPerHour) + default: + return fmt.Sprintf("%dd ago", secs/secsPerDay) + } +} + +func truncate(s string, max int) string { + s = strings.Join(strings.Fields(s), " ") + r := []rune(s) + if max <= 0 { + return "" + } + if len(r) <= max { + return s + } + if max == 1 { + return ellipsis + } + return string(r[:max-1]) + ellipsis +} + +func padRight(s string, width int) string { + w := len([]rune(s)) + if w >= width { + return s + } + return s + strings.Repeat(" ", width-w) +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/format_test.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/format_test.go new file mode 100644 index 00000000..c96d4ca0 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/format_test.go @@ -0,0 +1,173 @@ +// ©AngelaMos | 2026 +// format_test.go + +package tui + +import ( + "testing" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +func ptr(v float64) *float64 { return &v } + +func TestCVSSBand(t *testing.T) { + cases := []struct { + in *float64 + want band + }{ + {nil, bandNone}, + {ptr(0), bandNone}, + {ptr(3.9), bandLow}, + {ptr(4.0), bandMedium}, + {ptr(6.9), bandMedium}, + {ptr(7.0), bandHigh}, + {ptr(8.9), bandHigh}, + {ptr(9.0), bandCritical}, + {ptr(10.0), bandCritical}, + } + for _, c := range cases { + if got := cvssBand(c.in); got != c.want { + t.Errorf("cvssBand(%v) = %v, want %v", c.in, got, c.want) + } + } +} + +func TestEPSSBand(t *testing.T) { + cases := []struct { + in *float64 + want band + }{ + {nil, bandNone}, + {ptr(0), bandNone}, + {ptr(0.05), bandMedium}, + {ptr(0.1), bandHigh}, + {ptr(0.49), bandHigh}, + {ptr(0.5), bandCritical}, + {ptr(0.97), bandCritical}, + } + for _, c := range cases { + if got := epssBand(c.in); got != c.want { + t.Errorf("epssBand(%v) = %v, want %v", c.in, got, c.want) + } + } +} + +func TestBandLabel(t *testing.T) { + cases := map[band]string{ + bandCritical: "CRITICAL", + bandHigh: "HIGH", + bandMedium: "MEDIUM", + bandLow: "LOW", + bandNone: naMarker, + } + for b, want := range cases { + if got := bandLabel(b); got != want { + t.Errorf("bandLabel(%v) = %q, want %q", b, got, want) + } + } +} + +func TestCVSSString(t *testing.T) { + if got := cvssString(nil); got != naMarker { + t.Errorf("cvssString(nil) = %q, want %q", got, naMarker) + } + if got := cvssString(ptr(9.8)); got != "9.8" { + t.Errorf("cvssString(9.8) = %q, want 9.8", got) + } + if got := cvssString(ptr(10)); got != "10.0" { + t.Errorf("cvssString(10) = %q, want 10.0", got) + } +} + +func TestEPSSString(t *testing.T) { + if got := epssString(nil); got != naMarker { + t.Errorf("epssString(nil) = %q, want %q", got, naMarker) + } + if got := epssString(ptr(0.5)); got != "50.0%" { + t.Errorf("epssString(0.5) = %q, want 50.0%%", got) + } + if got := epssString(ptr(0.001)); got != "0.1%" { + t.Errorf("epssString(0.001) = %q, want 0.1%%", got) + } +} + +func TestRelativeAge(t *testing.T) { + now := time.Unix(1_000_000, 0) + cases := []struct { + unix int64 + want string + }{ + {0, naMarker}, + {now.Unix() + 100, "just now"}, + {now.Unix() - 30, "just now"}, + {now.Unix() - 120, "2m ago"}, + {now.Unix() - 7200, "2h ago"}, + {now.Unix() - 172800, "2d ago"}, + } + for _, c := range cases { + if got := relativeAge(c.unix, now); got != c.want { + t.Errorf("relativeAge(%d) = %q, want %q", c.unix, got, c.want) + } + } +} + +func TestTruncate(t *testing.T) { + cases := []struct { + in string + max int + want string + }{ + {"hello world", 20, "hello world"}, + {"hello world", 5, "hell" + ellipsis}, + {" multi space ", 20, "multi space"}, + {"abc", 1, ellipsis}, + {"abc", 0, ""}, + } + for _, c := range cases { + if got := truncate(c.in, c.max); got != c.want { + t.Errorf("truncate(%q, %d) = %q, want %q", c.in, c.max, got, c.want) + } + } +} + +func TestClusterSignals(t *testing.T) { + c := store.DigestCluster{ + CVEs: []store.DigestCVE{ + {ID: "CVE-A", CVSSScore: ptr(7.5), EPSS: ptr(0.2)}, + {ID: "CVE-B", CVSSScore: ptr(9.8), EPSS: ptr(0.9), IsKEV: true}, + {ID: "CVE-C"}, + }, + } + if !clusterHasKEV(c) { + t.Error("clusterHasKEV = false, want true") + } + if got := clusterMaxCVSS(c); got == nil || *got != 9.8 { + t.Errorf("clusterMaxCVSS = %v, want 9.8", got) + } + if got := clusterMaxEPSS(c); got == nil || *got != 0.9 { + t.Errorf("clusterMaxEPSS = %v, want 0.9", got) + } + if got := clusterBand(c); got != bandCritical { + t.Errorf("clusterBand = %v, want bandCritical (KEV)", got) + } + + noKev := store.DigestCluster{CVEs: []store.DigestCVE{{ID: "CVE-D", CVSSScore: ptr(7.5)}}} + if got := clusterBand(noKev); got != bandHigh { + t.Errorf("clusterBand(no kev, 7.5) = %v, want bandHigh", got) + } + empty := store.DigestCluster{} + if got := clusterBand(empty); got != bandNone { + t.Errorf("clusterBand(empty) = %v, want bandNone", got) + } +} + +func TestOutletColor(t *testing.T) { + cases := map[int]string{1: colorDim, 2: colorBlue, 3: colorCyan, 4: colorViolet, 9: colorViolet} + for n, want := range cases { + if got := outletColor(n); got != want { + t.Errorf("outletColor(%d) = %q, want %q", n, got, want) + } + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/keys.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/keys.go new file mode 100644 index 00000000..11995599 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/keys.go @@ -0,0 +1,54 @@ +// ©AngelaMos | 2026 +// keys.go + +package tui + +import "github.com/charmbracelet/bubbles/key" + +type keyMap struct { + Up key.Binding + Down key.Binding + Top key.Binding + Bottom key.Binding + Open key.Binding + Browser key.Binding + Back key.Binding + Quit key.Binding +} + +func defaultKeyMap() keyMap { + return keyMap{ + Up: key.NewBinding( + key.WithKeys("up", "k"), + key.WithHelp("↑/k", "up"), + ), + Down: key.NewBinding( + key.WithKeys("down", "j"), + key.WithHelp("↓/j", "down"), + ), + Top: key.NewBinding( + key.WithKeys("g", "home"), + key.WithHelp("g", "top"), + ), + Bottom: key.NewBinding( + key.WithKeys("G", "end"), + key.WithHelp("G", "bottom"), + ), + Open: key.NewBinding( + key.WithKeys("enter"), + key.WithHelp("⏎", "detail"), + ), + Browser: key.NewBinding( + key.WithKeys("o"), + key.WithHelp("o", "open"), + ), + Back: key.NewBinding( + key.WithKeys("esc", "backspace"), + key.WithHelp("esc", "back"), + ), + Quit: key.NewBinding( + key.WithKeys("q", "ctrl+c"), + key.WithHelp("q", "quit"), + ), + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/list.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/list.go new file mode 100644 index 00000000..56474138 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/list.go @@ -0,0 +1,222 @@ +// ©AngelaMos | 2026 +// list.go + +package tui + +import ( + "fmt" + "strings" + + "github.com/charmbracelet/lipgloss" + + "github.com/CarterPerez-dev/nadezhda/internal/rank" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const ( + colSelBarW = 1 + colRankW = 2 + colScoreBarW = 10 + colScoreNumW = 4 + colOutletW = 6 + colCVEW = 5 + colGaps = 6 + minHeadlineW = 12 + + glyphSelected = "◆" + glyphHot = "▐" + glyphOutlet = "◉" +) + +func (m Model) listView() string { + header := m.listChrome() + footer := m.listFooter() + capacity := m.height - lipgloss.Height(header) - lipgloss.Height(footer) + if capacity < 1 { + capacity = 1 + } + body := m.listBody(capacity) + return lipgloss.JoinVertical(lipgloss.Left, header, body, footer) +} + +func (m Model) listChrome() string { + right := m.theme.Meta.Render(fmt.Sprintf("%d stories", len(m.scored))) + return lipgloss.JoinVertical(lipgloss.Left, + m.spread(m.wordmark(), right), + m.theme.rule(m.width), + m.listColHeader(), + ) +} + +func (m Model) listColHeader() string { + t := m.theme.ColHead + left := strings.Join([]string{ + " ", + fmt.Sprintf("%*s", colRankW, "#"), + padRight("STORY", m.headlineWidth()), + padRight("SCORE", colScoreBarW), + fmt.Sprintf("%*s", colScoreNumW, ""), + padRight("OUT", colOutletW), + }, " ") + return m.spread(t.Render(left), t.Render("CVE")) +} + +func (m Model) listFooter() string { + keys := m.keyHints(m.keys.Up, m.keys.Down, m.keys.Open, m.keys.Browser, m.keys.Quit) + right := m.severityLegend() + if s := m.statusText(); s != "" { + right = s + } + return lipgloss.JoinVertical(lipgloss.Left, + m.theme.rule(m.width), + m.spread(keys, right), + ) +} + +func (m Model) listBody(capacity int) string { + if len(m.scored) == 0 { + return m.emptyBody(capacity) + } + first, last := windowRange(m.cursor, capacity, len(m.scored)) + lines := make([]string, 0, capacity) + for i := first; i < last; i++ { + lines = append(lines, m.renderRow(i, m.scored[i], i == m.cursor)) + } + for len(lines) < capacity { + lines = append(lines, "") + } + return strings.Join(lines, "\n") +} + +func (m Model) emptyBody(capacity int) string { + t := m.theme + msg := t.Muted.Render("no stories in the store yet — run ") + + t.KeyGlyph.Render("nadezhda scrape") + + t.Muted.Render(" then ") + + t.KeyGlyph.Render("nadezhda enrich") + return lipgloss.Place(m.width, capacity, lipgloss.Center, lipgloss.Center, msg) +} + +func (m Model) renderRow(i int, s rank.Scored, selected bool) string { + t := m.theme + c := s.Cluster + b := clusterBand(c) + kev := clusterHasKEV(c) + + barStyle := t.Dim + barGlyph := " " + switch { + case selected: + barGlyph, barStyle = glyphSelected, t.SelBar + case kev || b == bandCritical: + barGlyph, barStyle = glyphHot, t.bandFG(bandCritical) + case b != bandNone: + barGlyph, barStyle = glyphHot, t.bandFG(b) + } + + rankStyle := t.Rank + headStyle := t.Headline + if selected { + rankStyle, headStyle = t.RankSel, t.HeadlineSel + } + + hlW := m.headlineWidth() + left := strings.Join([]string{ + barStyle.Render(barGlyph), + rankStyle.Render(fmt.Sprintf("%*d", colRankW, i+1)), + headStyle.Render(padRight(truncate(headlineOf(c), hlW), hlW)), + t.spectrumBar(s.Score, colScoreBarW), + t.Meta.Render(fmt.Sprintf("%*s", colScoreNumW, fmt.Sprintf("%.2f", s.Score))), + m.renderOutlets(c), + }, " ") + return m.spread(left, m.renderCVEChip(c, b, kev)) +} + +func windowRange(cursor, capacity, total int) (int, int) { + if capacity < 1 { + capacity = 1 + } + first := 0 + if cursor >= capacity { + first = cursor - capacity + 1 + } + last := first + capacity + if last > total { + last = total + } + return first, last +} + +func (m Model) renderOutlets(c store.DigestCluster) string { + n := outletCount(c) + dots := n + if dots > colOutletW { + dots = colOutletW + } + styled := m.theme.fg(outletColor(n)).Render(strings.Repeat(glyphOutlet, dots)) + if pad := colOutletW - dots; pad > 0 { + styled += strings.Repeat(" ", pad) + } + return styled +} + +func (m Model) renderCVEChip(c store.DigestCluster, b band, kev bool) string { + t := m.theme + if kev { + return t.chip("KEV", colorMagenta) + } + if max := clusterMaxCVSS(c); max != nil { + return t.bandFG(b).Bold(true).Render(cvssString(max)) + } + if len(c.CVEs) > 0 { + return t.Dim.Render("cve") + } + return t.Dim.Render(emptyMarker) +} + +func (m Model) headlineWidth() int { + fixed := colSelBarW + colRankW + colScoreBarW + colScoreNumW + colOutletW + colCVEW + colGaps + if w := m.width - fixed; w > minHeadlineW { + return w + } + return minHeadlineW +} + +func headlineArticle(c store.DigestCluster) store.DigestArticle { + var best store.DigestArticle + for i, a := range c.Articles { + if i == 0 || a.PublishedAt > best.PublishedAt || + (a.PublishedAt == best.PublishedAt && a.ID < best.ID) { + best = a + } + } + return best +} + +func headlineOf(c store.DigestCluster) string { + if a := headlineArticle(c); strings.TrimSpace(a.Title) != "" { + return a.Title + } + return "(untitled cluster)" +} + +func outletCount(c store.DigestCluster) int { + seen := make(map[string]struct{}, len(c.Articles)) + for _, a := range c.Articles { + seen[a.SourceName] = struct{}{} + } + return len(seen) +} + +func outletColor(n int) string { + switch { + case n >= 4: + return colorViolet + case n == 3: + return colorCyan + case n == 2: + return colorBlue + default: + return colorDim + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/model.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/model.go new file mode 100644 index 00000000..0f1884ec --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/model.go @@ -0,0 +1,253 @@ +// ©AngelaMos | 2026 +// model.go + +package tui + +import ( + "time" + + "github.com/charmbracelet/bubbles/key" + "github.com/charmbracelet/bubbles/spinner" + "github.com/charmbracelet/bubbles/viewport" + tea "github.com/charmbracelet/bubbletea" + + "github.com/CarterPerez-dev/nadezhda/internal/rank" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +type viewState int + +const ( + stateLoading viewState = iota + stateList + stateDetail + stateError +) + +const ( + defaultWidth = 100 + defaultHeight = 32 + detailChrome = 4 + spinnerFPS = time.Second / 10 +) + +var raveSpinner = spinner.Spinner{ + Frames: []string{"◇", "◈", "◆", "◈", "◇", "·"}, + FPS: spinnerFPS, +} + +type Data struct { + Scored []rank.Scored + CVEDetail map[string]store.CVE +} + +type Loader func() (Data, error) + +type dataMsg struct{ data Data } + +type errMsg struct{ err error } + +type openedMsg struct { + url string + err error +} + +type Model struct { + state viewState + loader Loader + now time.Time + theme Theme + keys keyMap + spinner spinner.Model + viewport viewport.Model + + width int + height int + + scored []rank.Scored + cveDetail map[string]store.CVE + + cursor int + err error + + opener func(string) error + status string + statusErr bool +} + +func New(loader Loader, now time.Time) Model { + th := NewTheme() + sp := spinner.New(spinner.WithSpinner(raveSpinner), spinner.WithStyle(th.Spinner)) + m := Model{ + state: stateLoading, + loader: loader, + now: now, + theme: th, + keys: defaultKeyMap(), + spinner: sp, + viewport: viewport.New(defaultWidth, defaultHeight-detailChrome), + width: defaultWidth, + height: defaultHeight, + cveDetail: map[string]store.CVE{}, + opener: openURL, + } + return m +} + +func (m Model) Init() tea.Cmd { + return tea.Batch(m.spinner.Tick, m.load()) +} + +func (m Model) load() tea.Cmd { + loader := m.loader + return func() tea.Msg { + data, err := loader() + if err != nil { + return errMsg{err} + } + return dataMsg{data} + } +} + +func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + m.applySize(msg.Width, msg.Height) + return m, nil + case dataMsg: + m.scored = msg.data.Scored + m.cveDetail = msg.data.CVEDetail + m.state = stateList + return m, nil + case errMsg: + m.state = stateError + m.err = msg.err + return m, nil + case openedMsg: + if msg.err != nil { + m.status, m.statusErr = "open failed: "+msg.err.Error(), true + } else { + m.status, m.statusErr = "opened in browser", false + } + return m, nil + case spinner.TickMsg: + if m.state != stateLoading { + return m, nil + } + var cmd tea.Cmd + m.spinner, cmd = m.spinner.Update(msg) + return m, cmd + case tea.KeyMsg: + return m.handleKey(msg) + } + if m.state == stateDetail { + var cmd tea.Cmd + m.viewport, cmd = m.viewport.Update(msg) + return m, cmd + } + return m, nil +} + +func (m Model) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { + m.status = "" + if key.Matches(msg, m.keys.Quit) { + return m, tea.Quit + } + if key.Matches(msg, m.keys.Browser) && (m.state == stateList || m.state == stateDetail) { + return m, m.openSelected() + } + switch m.state { + case stateList: + return m.handleListKey(msg) + case stateDetail: + return m.handleDetailKey(msg) + default: + return m, nil + } +} + +func (m Model) openSelected() tea.Cmd { + target := headlineArticle(m.selected().Cluster).CanonicalURL + if target == "" { + return nil + } + open := m.opener + return func() tea.Msg { + return openedMsg{url: target, err: open(target)} + } +} + +func (m Model) handleListKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { + switch { + case key.Matches(msg, m.keys.Up): + if m.cursor > 0 { + m.cursor-- + } + case key.Matches(msg, m.keys.Down): + if m.cursor < len(m.scored)-1 { + m.cursor++ + } + case key.Matches(msg, m.keys.Top): + m.cursor = 0 + case key.Matches(msg, m.keys.Bottom): + if len(m.scored) > 0 { + m.cursor = len(m.scored) - 1 + } + case key.Matches(msg, m.keys.Open): + if len(m.scored) > 0 { + m.state = stateDetail + m.viewport.SetContent(m.renderDetailBody()) + m.viewport.GotoTop() + } + } + return m, nil +} + +func (m Model) handleDetailKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { + if key.Matches(msg, m.keys.Back) { + m.state = stateList + return m, nil + } + var cmd tea.Cmd + m.viewport, cmd = m.viewport.Update(msg) + return m, cmd +} + +func (m *Model) applySize(w, h int) { + m.width = w + m.height = h + m.viewport.Width = m.bodyWidth() + m.viewport.Height = m.detailBodyHeight() + if m.state == stateDetail { + m.viewport.SetContent(m.renderDetailBody()) + } +} + +func (m Model) bodyWidth() int { + if m.width > 1 { + return m.width + } + return 1 +} + +func (m Model) detailBodyHeight() int { + if h := m.height - detailChrome; h > 1 { + return h + } + return 1 +} + +func (m Model) selected() rank.Scored { + if m.cursor < 0 || m.cursor >= len(m.scored) { + return rank.Scored{} + } + return m.scored[m.cursor] +} + +func Run(loader Loader) error { + m := New(loader, time.Now()) + _, err := tea.NewProgram(m, tea.WithAltScreen()).Run() + return err +} + +var _ tea.Model = Model{} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/model_test.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/model_test.go new file mode 100644 index 00000000..e8c8fd27 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/model_test.go @@ -0,0 +1,322 @@ +// ©AngelaMos | 2026 +// model_test.go + +package tui + +import ( + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/charmbracelet/bubbles/spinner" + tea "github.com/charmbracelet/bubbletea" + + "github.com/CarterPerez-dev/nadezhda/internal/rank" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +func testNow() time.Time { return time.Unix(1_720_000_000, 0) } + +func runeKey(r rune) tea.KeyMsg { return tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{r}} } + +func sampleData() Data { + now := testNow().Unix() + c1 := store.DigestCluster{ + ClusterID: 1, + Key: "log4shell", + Size: 2, + FirstSeen: now - 3600, + LastSeen: now - 600, + Articles: []store.DigestArticle{ + {ID: 1, SourceName: "krebs", SourceWeight: 1.0, Title: "Log4Shell RCE exploited in the wild", CanonicalURL: "https://krebsonsecurity.com/log4shell", PublishedAt: now - 600}, + {ID: 2, SourceName: "theregister", SourceWeight: 0.9, Title: "Log4j flaw under active attack", CanonicalURL: "https://theregister.com/log4j", PublishedAt: now - 3600}, + }, + CVEs: []store.DigestCVE{ + {ID: "CVE-2021-44228", CVSSScore: ptr(10.0), EPSS: ptr(0.97), IsKEV: true}, + }, + } + c2 := store.DigestCluster{ + ClusterID: 2, + Key: "policy", + Size: 1, + FirstSeen: now - 7200, + LastSeen: now - 7200, + Articles: []store.DigestArticle{ + {ID: 3, SourceName: "darkreading", SourceWeight: 0.7, Title: "New disclosure guidelines published", CanonicalURL: "https://darkreading.com/policy", PublishedAt: now - 7200}, + }, + } + scored := []rank.Scored{{Cluster: c1, Score: 0.94}, {Cluster: c2, Score: 0.42}} + detail := map[string]store.CVE{ + "CVE-2021-44228": { + ID: "CVE-2021-44228", + Description: "Apache Log4j2 JNDI features do not protect against attacker controlled LDAP endpoints.", + CVSSScore: ptr(10.0), + CVSSVersion: "3.1", + CVSSSeverity: "CRITICAL", + CVSSVector: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", + CWE: "CWE-502", + IsKEV: true, + EPSS: ptr(0.97), + }, + } + return Data{Scored: scored, CVEDetail: detail} +} + +func toModel(t *testing.T, tm tea.Model) Model { + t.Helper() + m, ok := tm.(Model) + if !ok { + t.Fatalf("Update returned %T, want tui.Model", tm) + } + return m +} + +func step(t *testing.T, m Model, msg tea.Msg) Model { + t.Helper() + tm, _ := m.Update(msg) + return toModel(t, tm) +} + +func loadedModel(t *testing.T) Model { + t.Helper() + return step(t, New(nil, testNow()), dataMsg{sampleData()}) +} + +func TestInitialStateIsLoading(t *testing.T) { + if m := New(nil, testNow()); m.state != stateLoading { + t.Fatalf("initial state = %v, want stateLoading", m.state) + } +} + +func TestDataMsgTransitionsToList(t *testing.T) { + m := loadedModel(t) + if m.state != stateList { + t.Fatalf("state = %v, want stateList", m.state) + } + if len(m.scored) != 2 { + t.Fatalf("len(scored) = %d, want 2", len(m.scored)) + } +} + +func TestErrMsgTransitionsToError(t *testing.T) { + m := step(t, New(nil, testNow()), errMsg{errors.New("wire down")}) + if m.state != stateError { + t.Fatalf("state = %v, want stateError", m.state) + } + if m.err == nil { + t.Fatal("err = nil, want non-nil") + } +} + +func TestListNavigationClamps(t *testing.T) { + m := loadedModel(t) + m = step(t, m, runeKey('j')) + if m.cursor != 1 { + t.Fatalf("after down, cursor = %d, want 1", m.cursor) + } + m = step(t, m, runeKey('j')) + if m.cursor != 1 { + t.Fatalf("after down past end, cursor = %d, want 1", m.cursor) + } + m = step(t, m, runeKey('k')) + if m.cursor != 0 { + t.Fatalf("after up, cursor = %d, want 0", m.cursor) + } + m = step(t, m, runeKey('k')) + if m.cursor != 0 { + t.Fatalf("after up past start, cursor = %d, want 0", m.cursor) + } +} + +func TestListTopBottom(t *testing.T) { + m := loadedModel(t) + m = step(t, m, runeKey('G')) + if m.cursor != 1 { + t.Fatalf("after G, cursor = %d, want 1", m.cursor) + } + m = step(t, m, runeKey('g')) + if m.cursor != 0 { + t.Fatalf("after g, cursor = %d, want 0", m.cursor) + } +} + +func TestOpenAndBack(t *testing.T) { + m := loadedModel(t) + m = step(t, m, tea.KeyMsg{Type: tea.KeyEnter}) + if m.state != stateDetail { + t.Fatalf("after enter, state = %v, want stateDetail", m.state) + } + m = step(t, m, tea.KeyMsg{Type: tea.KeyEsc}) + if m.state != stateList { + t.Fatalf("after esc, state = %v, want stateList", m.state) + } +} + +func TestQuitReturnsQuitCmd(t *testing.T) { + m := loadedModel(t) + _, cmd := m.Update(runeKey('q')) + if cmd == nil { + t.Fatal("quit returned nil cmd") + } + if reflect.TypeOf(cmd()) != reflect.TypeOf(tea.Quit()) { + t.Fatalf("quit cmd yielded %T, want tea.QuitMsg", cmd()) + } +} + +func TestWindowSizeSizesViewport(t *testing.T) { + m := loadedModel(t) + m = step(t, m, tea.WindowSizeMsg{Width: 120, Height: 40}) + if m.width != 120 || m.height != 40 { + t.Fatalf("size = %dx%d, want 120x40", m.width, m.height) + } + if m.viewport.Width != 120 { + t.Fatalf("viewport.Width = %d, want 120", m.viewport.Width) + } + if m.viewport.Height != 40-detailChrome { + t.Fatalf("viewport.Height = %d, want %d", m.viewport.Height, 40-detailChrome) + } +} + +func TestSpinnerTickIgnoredOutsideLoading(t *testing.T) { + m := loadedModel(t) + m = step(t, m, spinner.TickMsg{}) + if m.state != stateList { + t.Fatalf("spinner tick changed state to %v, want stateList", m.state) + } +} + +func TestViewsRenderNonEmpty(t *testing.T) { + loading := New(nil, testNow()) + if strings.TrimSpace(loading.View()) == "" { + t.Error("loading view is empty") + } + + list := loadedModel(t) + lv := list.View() + if !strings.Contains(lv, "NADEZHDA") { + t.Error("list view missing brand") + } + if !strings.Contains(lv, "Log4Shell") { + t.Error("list view missing headline") + } + + detail := step(t, list, tea.KeyMsg{Type: tea.KeyEnter}) + dv := detail.View() + if !strings.Contains(dv, "CVE-2021-44228") { + t.Error("detail view missing CVE id") + } + if !strings.Contains(dv, "CRITICAL") { + t.Error("detail view missing severity label") + } + + errv := step(t, New(nil, testNow()), errMsg{errors.New("boom")}) + if strings.TrimSpace(errv.View()) == "" { + t.Error("error view is empty") + } +} + +func TestEmptyStoreRendersHint(t *testing.T) { + m := step(t, New(nil, testNow()), dataMsg{Data{}}) + if m.state != stateList { + t.Fatalf("state = %v, want stateList", m.state) + } + if !strings.Contains(m.View(), "scrape") { + t.Error("empty list view missing scrape hint") + } +} + +func TestTooSmallTerminal(t *testing.T) { + m := step(t, loadedModel(t), tea.WindowSizeMsg{Width: 20, Height: 5}) + if !strings.Contains(m.View(), "too small") { + t.Error("tiny terminal did not render the too-small notice") + } +} + +func TestWindowRange(t *testing.T) { + cases := []struct { + cursor, capacity, total int + first, last int + }{ + {0, 5, 10, 0, 5}, + {4, 5, 10, 0, 5}, + {5, 5, 10, 1, 6}, + {9, 5, 10, 5, 10}, + {0, 5, 3, 0, 3}, + {2, 5, 3, 0, 3}, + {0, 0, 10, 0, 1}, + {7, 3, 10, 5, 8}, + } + for _, c := range cases { + first, last := windowRange(c.cursor, c.capacity, c.total) + if first != c.first || last != c.last { + t.Errorf("windowRange(%d, %d, %d) = (%d, %d), want (%d, %d)", + c.cursor, c.capacity, c.total, first, last, c.first, c.last) + } + if c.cursor < c.total && (c.cursor < first || c.cursor >= last) { + t.Errorf("windowRange(%d, %d, %d) window [%d,%d) excludes cursor", + c.cursor, c.capacity, c.total, first, last) + } + } +} + +func TestHeadlineFreshest(t *testing.T) { + fresh := store.DigestCluster{Articles: []store.DigestArticle{ + {ID: 1, Title: "older", PublishedAt: 100}, + {ID: 2, Title: "newest", PublishedAt: 300}, + {ID: 3, Title: "mid", PublishedAt: 200}, + }} + if got := headlineOf(fresh); got != "newest" { + t.Errorf("headlineOf(fresh) = %q, want newest", got) + } + tie := store.DigestCluster{Articles: []store.DigestArticle{ + {ID: 5, Title: "high-id", PublishedAt: 300}, + {ID: 2, Title: "low-id", PublishedAt: 300}, + }} + if got := headlineOf(tie); got != "low-id" { + t.Errorf("headlineOf(tie) = %q, want low-id (lowest id on equal time)", got) + } + if got := headlineOf(store.DigestCluster{}); got != "(untitled cluster)" { + t.Errorf("headlineOf(empty) = %q, want (untitled cluster)", got) + } +} + +func TestOpenInBrowser(t *testing.T) { + m := loadedModel(t) + var opened string + m.opener = func(u string) error { opened = u; return nil } + + tm, cmd := m.Update(runeKey('o')) + m = toModel(t, tm) + if cmd == nil { + t.Fatal("pressing o produced no command") + } + msg := cmd() + if opened != "https://krebsonsecurity.com/log4shell" { + t.Fatalf("opened %q, want the selected cluster's headline url", opened) + } + om, ok := msg.(openedMsg) + if !ok { + t.Fatalf("open cmd returned %T, want openedMsg", msg) + } + m = step(t, m, om) + if m.status == "" || m.statusErr { + t.Errorf("after open: status=%q err=%v, want a success status", m.status, m.statusErr) + } + if !strings.Contains(m.View(), m.status) { + t.Error("open status not shown in the view") + } + m = step(t, m, runeKey('j')) + if m.status != "" { + t.Error("status should clear on the next keypress") + } +} + +func TestOpenURLRejectsNonHTTP(t *testing.T) { + for _, bad := range []string{"file:///etc/passwd", "javascript:alert(1)", "", "ftp://x/y"} { + if err := openURL(bad); err == nil { + t.Errorf("openURL(%q) = nil, want refusal", bad) + } + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/theme.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/theme.go new file mode 100644 index 00000000..3e3fd795 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/theme.go @@ -0,0 +1,151 @@ +// ©AngelaMos | 2026 +// theme.go + +package tui + +import ( + "math" + "strings" + + "github.com/charmbracelet/lipgloss" +) + +const ( + colorVoid = "#0B0620" + colorText = "#E7E0FF" + colorMuted = "#9A8AD0" + colorDim = "#574B7D" + + colorMagenta = "#FF2E97" + colorAmber = "#FF8A3D" + colorYellow = "#FFE04D" + colorCyan = "#00E5FF" + colorViolet = "#A65CFF" + colorBlue = "#2D6BFF" +) + +const ( + barFull = "█" + barEmpty = "░" + ruleLead = "▓▒░" + ruleBody = "─" +) + +func bandColor(b band) string { + switch b { + case bandCritical: + return colorMagenta + case bandHigh: + return colorAmber + case bandMedium: + return colorYellow + case bandLow: + return colorCyan + default: + return colorDim + } +} + +type Theme struct { + Brand lipgloss.Style + BrandMark lipgloss.Style + Meta lipgloss.Style + ColHead lipgloss.Style + Text lipgloss.Style + Muted lipgloss.Style + Dim lipgloss.Style + Rank lipgloss.Style + RankSel lipgloss.Style + Headline lipgloss.Style + HeadlineSel lipgloss.Style + SelBar lipgloss.Style + KeyGlyph lipgloss.Style + KeyDesc lipgloss.Style + Spinner lipgloss.Style + PanelTitle lipgloss.Style + Label lipgloss.Style + Link lipgloss.Style +} + +func NewTheme() Theme { + base := lipgloss.NewStyle().Foreground(lipgloss.Color(colorText)) + return Theme{ + Brand: lipgloss.NewStyle().Foreground(lipgloss.Color(colorViolet)).Bold(true), + BrandMark: lipgloss.NewStyle().Foreground(lipgloss.Color(colorMagenta)).Bold(true), + Meta: lipgloss.NewStyle().Foreground(lipgloss.Color(colorCyan)), + ColHead: lipgloss.NewStyle().Foreground(lipgloss.Color(colorMuted)).Bold(true), + Text: base, + Muted: lipgloss.NewStyle().Foreground(lipgloss.Color(colorMuted)), + Dim: lipgloss.NewStyle().Foreground(lipgloss.Color(colorDim)), + Rank: lipgloss.NewStyle().Foreground(lipgloss.Color(colorMuted)), + RankSel: lipgloss.NewStyle().Foreground(lipgloss.Color(colorMagenta)).Bold(true), + Headline: base, + HeadlineSel: lipgloss.NewStyle().Foreground(lipgloss.Color(colorText)).Bold(true), + SelBar: lipgloss.NewStyle().Foreground(lipgloss.Color(colorMagenta)).Bold(true), + KeyGlyph: lipgloss.NewStyle().Foreground(lipgloss.Color(colorCyan)).Bold(true), + KeyDesc: lipgloss.NewStyle().Foreground(lipgloss.Color(colorMuted)), + Spinner: lipgloss.NewStyle().Foreground(lipgloss.Color(colorMagenta)).Bold(true), + PanelTitle: lipgloss.NewStyle().Foreground(lipgloss.Color(colorViolet)).Bold(true), + Label: lipgloss.NewStyle().Foreground(lipgloss.Color(colorBlue)).Bold(true), + Link: lipgloss.NewStyle().Foreground(lipgloss.Color(colorCyan)).Underline(true), + } +} + +func (t Theme) fg(color string) lipgloss.Style { + return lipgloss.NewStyle().Foreground(lipgloss.Color(color)) +} + +func (t Theme) bandFG(b band) lipgloss.Style { + return t.fg(bandColor(b)) +} + +func (t Theme) chip(text, color string) string { + return lipgloss.NewStyle(). + Foreground(lipgloss.Color(colorVoid)). + Background(lipgloss.Color(color)). + Bold(true). + Render(" " + text + " ") +} + +func (t Theme) spectrumBar(frac float64, width int) string { + if width <= 0 { + return "" + } + filled := int(math.Round(clamp01(frac) * float64(width))) + stops := []string{colorCyan, colorBlue, colorViolet, colorMagenta} + var b strings.Builder + for i := 0; i < width; i++ { + if i < filled { + b.WriteString(t.fg(stops[i*len(stops)/width]).Render(barFull)) + } else { + b.WriteString(t.Dim.Render(barEmpty)) + } + } + return b.String() +} + +func (t Theme) bandBar(b band, frac float64, width int) string { + if width <= 0 { + return "" + } + filled := int(math.Round(clamp01(frac) * float64(width))) + color := bandColor(b) + var sb strings.Builder + for i := 0; i < width; i++ { + if i < filled { + sb.WriteString(t.fg(color).Render(barFull)) + } else { + sb.WriteString(t.Dim.Render(barEmpty)) + } + } + return sb.String() +} + +func (t Theme) rule(width int) string { + lead := len([]rune(ruleLead)) + 1 + n := width - lead + if n < 0 { + return t.fg(colorViolet).Render(ruleLead) + } + return t.fg(colorViolet).Render(ruleLead) + " " + t.Dim.Render(strings.Repeat(ruleBody, n)) +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/view.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/view.go new file mode 100644 index 00000000..15a3278f --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/view.go @@ -0,0 +1,159 @@ +// ©AngelaMos | 2026 +// view.go + +package tui + +import ( + "fmt" + "strings" + + "github.com/charmbracelet/bubbles/key" + "github.com/charmbracelet/lipgloss" +) + +const ( + brandName = "NADEZHDA" + tagline = "THREAT.WIRE" + + markOpen = "◤ " + markClose = " ◢" + markMid = "═◆═" + + errWrapMax = 80 + errWrapMin = 20 + errWrapMargin = 6 + + minRenderWidth = 44 + minRenderHeight = 10 +) + +func (m Model) View() string { + if m.width < minRenderWidth || m.height < minRenderHeight { + return m.tooSmallView() + } + switch m.state { + case stateLoading: + return m.loadingView() + case stateError: + return m.errorView() + case stateDetail: + return m.detailView() + default: + return m.listView() + } +} + +func (m Model) tooSmallView() string { + w, h := m.width, m.height + if w < 1 { + w = 1 + } + if h < 1 { + h = 1 + } + msg := m.theme.Muted.Render(fmt.Sprintf("terminal too small — need at least %dx%d", minRenderWidth, minRenderHeight)) + return lipgloss.Place(w, h, lipgloss.Center, lipgloss.Center, msg) +} + +func (m Model) wordmark() string { + t := m.theme + return t.BrandMark.Render(markOpen) + t.Brand.Render(brandName) + t.BrandMark.Render(markClose) + + " " + t.fg(colorCyan).Render(markMid) + " " + t.fg(colorViolet).Render(tagline) + " " + t.fg(colorCyan).Render(markMid) +} + +func (m Model) spread(left, right string) string { + gap := m.width - lipgloss.Width(left) - lipgloss.Width(right) + if gap < 1 { + gap = 1 + } + return left + strings.Repeat(" ", gap) + right +} + +func (m Model) keyHints(bindings ...key.Binding) string { + t := m.theme + parts := make([]string, 0, len(bindings)) + for _, b := range bindings { + h := b.Help() + parts = append(parts, t.KeyGlyph.Render(h.Key)+" "+t.KeyDesc.Render(h.Desc)) + } + return strings.Join(parts, t.Dim.Render(" · ")) +} + +func (m Model) severityLegend() string { + t := m.theme + item := func(color, label string) string { + return t.fg(color).Render("■") + " " + t.KeyDesc.Render(label) + } + return strings.Join([]string{ + item(colorMagenta, "KEV/CRIT"), + item(colorAmber, "HIGH"), + item(colorYellow, "MED"), + item(colorCyan, "LOW"), + }, " ") +} + +func (m Model) loadingView() string { + t := m.theme + brand := t.BrandMark.Render(markOpen) + t.Brand.Render(brandName) + t.BrandMark.Render(markClose) + tag := t.fg(colorViolet).Render(tagline) + line := m.spinner.View() + " " + t.Meta.Render("assembling threat wire") + " " + m.spinner.View() + block := lipgloss.JoinVertical(lipgloss.Center, brand, tag, "", line) + return lipgloss.Place(m.width, m.height, lipgloss.Center, lipgloss.Center, block) +} + +func (m Model) errorView() string { + t := m.theme + title := t.fg(colorMagenta).Bold(true).Render("▚ WIRE FAULT ▞") + block := lipgloss.JoinVertical(lipgloss.Center, + title, + "", + m.errString(), + "", + t.KeyDesc.Render("press ")+t.KeyGlyph.Render("q")+t.KeyDesc.Render(" to quit"), + ) + return lipgloss.Place(m.width, m.height, lipgloss.Center, lipgloss.Center, block) +} + +func (m Model) errString() string { + if m.err == nil { + return m.theme.Text.Render("unknown fault") + } + w := m.width - errWrapMargin + if w > errWrapMax { + w = errWrapMax + } + if w < errWrapMin { + w = errWrapMin + } + return m.theme.Text.Width(w).Align(lipgloss.Center).Render(m.err.Error()) +} + +func (m Model) detailView() string { + t := m.theme + scroll := fmt.Sprintf("%3.0f%%", m.viewport.ScrollPercent()*100) + left := t.PanelTitle.Render(markOpen+"DOSSIER"+markClose) + " " + + t.Muted.Render(fmt.Sprintf("story %d of %d", m.cursor+1, len(m.scored))) + head := m.spread(left, t.Meta.Render(scroll)) + foot := m.spread( + m.keyHints(m.keys.Back, m.keys.Browser, m.keys.Down, m.keys.Up, m.keys.Quit), + m.statusText(), + ) + return lipgloss.JoinVertical(lipgloss.Left, + head, + t.rule(m.width), + m.viewport.View(), + t.rule(m.width), + foot, + ) +} + +func (m Model) statusText() string { + if m.status == "" { + return "" + } + color := colorCyan + if m.statusErr { + color = colorMagenta + } + return m.theme.fg(color).Render(m.status) +} From 41ffd49699d36eecdb46f72933f619f6c5e1ee6e Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Mon, 6 Jul 2026 01:44:53 -0400 Subject: [PATCH 07/15] feat(nadezhda): keyless CVE enrichment, folded into scrape, news-first ranking This is a security-news tool; CVE enrichment is optional garnish, never a setup tax. Drop the NVD API key from the default path entirely. - internal/cve/cvelist: keyless CVE Program cvelistV5 client (raw GitHub, carries CISA-ADP / vulnrichment CVSS). Parses cna and adp metrics with version precedence v4.0 > v3.1 > v3.0 > v2.0, plus CWE and description; 404 maps to not-found; v2 severity is capped at HIGH. Introduces a neutral CoreResult and a CVESource interface (NVDResult renamed to CoreResult; NVDClient still satisfies it). - enrich: Clients.Core is the CVESource. buildCoreSource picks the keyless cvelist client by default and NVD only when NVD_API_KEY is set (booster). - scrape auto-enriches referenced CVEs best-effort after clustering (time-bounded, non-fatal, --no-enrich to skip); enrich kept as an optional manual refresh. Default flow is scrape -> tui. - ranking default weights rebalanced news-first (recency / velocity / source / keyword 70 percent, cvss / kev / epss 30 percent; was 55 percent CVE). - KAT fixture testdata/cvelist/CVE-2021-44228.json. --- .../cmd/nadezhda/enrich.go | 42 +- .../cmd/nadezhda/scrape.go | 35 +- .../internal/config/config.go | 14 +- .../internal/cve/cvelist.go | 249 ++++++ .../internal/cve/cvelist_test.go | 132 +++ .../security-news-scraper/internal/cve/nvd.go | 26 +- .../internal/enrich/enrich.go | 24 +- .../internal/enrich/enrich_test.go | 4 +- .../testdata/cvelist/CVE-2021-44228.json | 796 ++++++++++++++++++ 9 files changed, 1267 insertions(+), 55 deletions(-) create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/cve/cvelist.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/cve/cvelist_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/cvelist/CVE-2021-44228.json diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/enrich.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/enrich.go index 0a35040d..7b041561 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/enrich.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/enrich.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/cobra" + "github.com/CarterPerez-dev/nadezhda/internal/config" "github.com/CarterPerez-dev/nadezhda/internal/cve" "github.com/CarterPerez-dev/nadezhda/internal/enrich" "github.com/CarterPerez-dev/nadezhda/internal/store" @@ -22,7 +23,8 @@ const nvdAPIKeyEnv = "NVD_API_KEY" var enrichCmd = &cobra.Command{ Use: "enrich", - Short: "Enrich extracted CVEs with NVD, CISA KEV, and EPSS intelligence", + Short: "Refresh CVE intelligence (CVE List / NVD, CISA KEV, EPSS) for extracted CVEs", + Long: "Refresh CVE intelligence for extracted CVEs. Keyless by default (CVE Program, CISA KEV, EPSS); uses NVD only when NVD_API_KEY is set. scrape already runs this automatically, so this command is for manually refreshing.", RunE: runEnrich, } @@ -41,27 +43,39 @@ func runEnrich(cmd *cobra.Command, args []string) error { } defer st.Close() - httpClient := &http.Client{Timeout: time.Duration(cfg.Fetch.TimeoutSeconds) * time.Second} - apiKey := os.Getenv(nvdAPIKeyEnv) - if apiKey == "" { - apiKey = cfg.Enrich.NVDAPIKey - } - clients := enrich.Clients{ - NVD: cve.NewNVDClient(httpClient, cve.NVDEndpoint, apiKey), - KEV: cve.NewKEVClient(httpClient, cve.KEVEndpoint), - EPSS: cve.NewEPSSClient(httpClient, cve.EPSSEndpoint), - } - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() - stats, err := enrich.Run(ctx, st, clients, time.Now(), cfg.Enrich.CacheTTLHours, cfg.Enrich.NegativeTTLHours) + stats, err := enrich.Run(ctx, st, buildEnrichClients(cfg), time.Now(), cfg.Enrich.CacheTTLHours, cfg.Enrich.NegativeTTLHours) if err != nil { return err } fmt.Fprintf(cmd.OutOrStdout(), - "enriched %d/%d CVEs (%d not in NVD, %d KEV, %d errors)\n", + "enriched %d/%d CVEs (%d not found, %d KEV, %d errors)\n", stats.Enriched, stats.Total, stats.NotFound, stats.KEVHits, stats.Errors) return nil } + +func buildEnrichClients(cfg config.Config) enrich.Clients { + httpClient := &http.Client{Timeout: time.Duration(cfg.Fetch.TimeoutSeconds) * time.Second} + return enrich.Clients{ + Core: buildCoreSource(httpClient, nvdAPIKey(cfg)), + KEV: cve.NewKEVClient(httpClient, cve.KEVEndpoint), + EPSS: cve.NewEPSSClient(httpClient, cve.EPSSEndpoint), + } +} + +func buildCoreSource(httpClient *http.Client, apiKey string) cve.CVESource { + if apiKey != "" { + return cve.NewNVDClient(httpClient, cve.NVDEndpoint, apiKey) + } + return cve.NewCVEListClient(httpClient, cve.CVEListEndpoint) +} + +func nvdAPIKey(cfg config.Config) string { + if k := os.Getenv(nvdAPIKeyEnv); k != "" { + return k + } + return cfg.Enrich.NVDAPIKey +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go index 5bcbf4d4..fbb1c0f8 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go @@ -13,6 +13,8 @@ import ( "github.com/spf13/cobra" "github.com/CarterPerez-dev/nadezhda/internal/cluster" + "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/enrich" "github.com/CarterPerez-dev/nadezhda/internal/fetch" "github.com/CarterPerez-dev/nadezhda/internal/ingest" "github.com/CarterPerez-dev/nadezhda/internal/source" @@ -21,6 +23,8 @@ import ( const secondsPerHour = 3600 +const enrichBudget = 5 * time.Minute + const ( statusNotModified = "304" statusError = "error" @@ -28,7 +32,10 @@ const ( dash = "-" ) -var scrapeSource string +var ( + scrapeSource string + scrapeNoEnrich bool +) var scrapeCmd = &cobra.Command{ Use: "scrape", @@ -38,6 +45,7 @@ var scrapeCmd = &cobra.Command{ func init() { scrapeCmd.Flags().StringVar(&scrapeSource, "source", "", "ingest only this source by name") + scrapeCmd.Flags().BoolVar(&scrapeNoEnrich, "no-enrich", false, "skip the keyless CVE enrichment pass") rootCmd.AddCommand(scrapeCmd) } @@ -88,9 +96,34 @@ func runScrape(cmd *cobra.Command, args []string) error { } fmt.Fprintf(cmd.OutOrStdout(), "%d clusters (%d multi-source, largest %d)\n", stats.Total, stats.MultiSource, stats.LargestSize) + + if !scrapeNoEnrich { + enrichAfterScrape(ctx, cmd, cfg, st) + } return nil } +func enrichAfterScrape(ctx context.Context, cmd *cobra.Command, cfg config.Config, st *store.Store) { + out := cmd.OutOrStdout() + ectx, cancel := context.WithTimeout(ctx, enrichBudget) + defer cancel() + + stats, err := enrich.Run(ectx, st, buildEnrichClients(cfg), time.Now(), cfg.Enrich.CacheTTLHours, cfg.Enrich.NegativeTTLHours) + if err != nil { + if done := stats.Enriched + stats.NotFound; done > 0 { + fmt.Fprintf(out, "enriched %d/%d CVEs before stopping: %v\n", done, stats.Total, err) + } else { + fmt.Fprintf(out, "enrich skipped: %v (news is unaffected)\n", err) + } + return + } + if stats.Total == 0 { + return + } + fmt.Fprintf(out, "enriched %d/%d CVEs (%d KEV, %d not found)\n", + stats.Enriched, stats.Total, stats.KEVHits, stats.NotFound) +} + func selectTargets(srcs []source.Source, only string) ([]source.Source, error) { if only != "" { for _, s := range srcs { diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go index 3a2c925d..030567f7 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go @@ -33,13 +33,13 @@ const ( defaultHalfLifeHours = 48 defaultVelocityNorm = 0.5 - defaultWeightRecency = 0.20 - defaultWeightCVSS = 0.15 - defaultWeightKEV = 0.25 - defaultWeightEPSS = 0.15 - defaultWeightVelocity = 0.15 - defaultWeightSource = 0.05 - defaultWeightKeyword = 0.05 + defaultWeightRecency = 0.30 + defaultWeightVelocity = 0.20 + defaultWeightKEV = 0.12 + defaultWeightCVSS = 0.10 + defaultWeightSource = 0.10 + defaultWeightKeyword = 0.10 + defaultWeightEPSS = 0.08 defaultAIProvider = "qwen" defaultQwenBaseURL = "http://localhost:11434/v1" diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cve/cvelist.go b/PROJECTS/intermediate/security-news-scraper/internal/cve/cvelist.go new file mode 100644 index 00000000..2d4401a9 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/cve/cvelist.go @@ -0,0 +1,249 @@ +// ©AngelaMos | 2026 +// cvelist.go + +package cve + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "golang.org/x/time/rate" +) + +const ( + CVEListEndpoint = "https://raw.githubusercontent.com/CVEProject/cvelistV5/main/cves" + + cvelistRate = 15 + cvelistMaxRetries = 2 + cvelistBackoff = time.Second + cvelistBucketTail = 3 + cveIDParts = 3 + langEnPrefix = "en" + + cvssCriticalMin = 9.0 + cvssHighMin = 7.0 + cvssMediumMin = 4.0 + + cvssVersion2Prefix = "2" + + sevCritical = "CRITICAL" + sevHigh = "HIGH" + sevMedium = "MEDIUM" + sevLow = "LOW" + sevNone = "NONE" +) + +type CoreResult struct { + Found bool + Description string + CVSSScore *float64 + CVSSVersion string + CVSSSeverity string + CVSSVector string + CWE string + Published string + Modified string +} + +type CVESource interface { + Fetch(ctx context.Context, cveID string) (CoreResult, error) +} + +type CVEListClient struct { + http *http.Client + baseURL string + limiter *rate.Limiter + backoffBase time.Duration +} + +func NewCVEListClient(client *http.Client, baseURL string) *CVEListClient { + return &CVEListClient{ + http: client, + baseURL: strings.TrimRight(baseURL, "/"), + limiter: rate.NewLimiter(rate.Limit(cvelistRate), 1), + backoffBase: cvelistBackoff, + } +} + +func (c *CVEListClient) Fetch(ctx context.Context, cveID string) (CoreResult, error) { + endpoint, err := c.recordURL(cveID) + if err != nil { + return CoreResult{}, err + } + + var rec cvelistRecord + for attempt := 0; attempt <= cvelistMaxRetries; attempt++ { + if attempt > 0 { + if err := sleep(ctx, c.backoffBase*time.Duration(1<<(attempt-1))); err != nil { + return CoreResult{}, err + } + } + if err := c.limiter.Wait(ctx); err != nil { + return CoreResult{}, err + } + err := getJSON(ctx, c.http, endpoint, nil, &rec) + if err == nil { + return rec.toResult(), nil + } + var se *statusError + if errors.As(err, &se) && se.code == http.StatusNotFound { + return CoreResult{Found: false}, nil + } + if !retriable(err) || attempt == cvelistMaxRetries { + return CoreResult{}, err + } + } + return CoreResult{Found: false}, nil +} + +func (c *CVEListClient) recordURL(cveID string) (string, error) { + parts := strings.Split(cveID, "-") + if len(parts) != cveIDParts || parts[0] != "CVE" { + return "", fmt.Errorf("cve: malformed id %q", cveID) + } + num := parts[2] + if len(num) <= cvelistBucketTail { + return "", fmt.Errorf("cve: short id %q", cveID) + } + bucket := num[:len(num)-cvelistBucketTail] + "xxx" + return fmt.Sprintf("%s/%s/%s/%s.json", c.baseURL, parts[1], bucket, cveID), nil +} + +type cvelistRecord struct { + CVEMetadata struct { + DatePublished string `json:"datePublished"` + DateUpdated string `json:"dateUpdated"` + } `json:"cveMetadata"` + Containers struct { + CNA cvelistContainer `json:"cna"` + ADP []cvelistContainer `json:"adp"` + } `json:"containers"` +} + +type cvelistContainer struct { + Descriptions []cvelistLangValue `json:"descriptions"` + ProblemTypes []struct { + Descriptions []struct { + Lang string `json:"lang"` + CweID string `json:"cweId"` + } `json:"descriptions"` + } `json:"problemTypes"` + Metrics []cvelistMetric `json:"metrics"` +} + +type cvelistLangValue struct { + Lang string `json:"lang"` + Value string `json:"value"` +} + +type cvelistMetric struct { + V40 *cvelistCVSS `json:"cvssV4_0"` + V31 *cvelistCVSS `json:"cvssV3_1"` + V30 *cvelistCVSS `json:"cvssV3_0"` + V2 *cvelistCVSS `json:"cvssV2_0"` +} + +type cvelistCVSS struct { + Version string `json:"version"` + BaseScore float64 `json:"baseScore"` + BaseSeverity string `json:"baseSeverity"` + VectorString string `json:"vectorString"` +} + +func (rec cvelistRecord) containers() []cvelistContainer { + out := make([]cvelistContainer, 0, len(rec.Containers.ADP)+1) + out = append(out, rec.Containers.CNA) + out = append(out, rec.Containers.ADP...) + return out +} + +func (rec cvelistRecord) toResult() CoreResult { + res := CoreResult{ + Found: true, + Description: rec.description(), + CWE: rec.cwe(), + Published: rec.CVEMetadata.DatePublished, + Modified: rec.CVEMetadata.DateUpdated, + } + if cv := rec.cvss(); cv != nil { + score := cv.BaseScore + res.CVSSScore = &score + res.CVSSVersion = cv.Version + res.CVSSVector = cv.VectorString + res.CVSSSeverity = cv.BaseSeverity + if res.CVSSSeverity == "" { + res.CVSSSeverity = severityFromScore(score, cv.Version) + } + } + return res +} + +func (rec cvelistRecord) cvss() *cvelistCVSS { + tiers := []func(cvelistMetric) *cvelistCVSS{ + func(m cvelistMetric) *cvelistCVSS { return m.V40 }, + func(m cvelistMetric) *cvelistCVSS { return m.V31 }, + func(m cvelistMetric) *cvelistCVSS { return m.V30 }, + func(m cvelistMetric) *cvelistCVSS { return m.V2 }, + } + containers := rec.containers() + for _, pick := range tiers { + for _, c := range containers { + for _, m := range c.Metrics { + if cv := pick(m); cv != nil { + return cv + } + } + } + } + return nil +} + +func (rec cvelistRecord) cwe() string { + for _, c := range rec.containers() { + for _, pt := range c.ProblemTypes { + for _, d := range pt.Descriptions { + if d.CweID != "" { + return d.CweID + } + } + } + } + return "" +} + +func (rec cvelistRecord) description() string { + for _, c := range rec.containers() { + for _, d := range c.Descriptions { + if strings.HasPrefix(d.Lang, langEnPrefix) && strings.TrimSpace(d.Value) != "" { + return d.Value + } + } + } + for _, c := range rec.containers() { + for _, d := range c.Descriptions { + if strings.TrimSpace(d.Value) != "" { + return d.Value + } + } + } + return "" +} + +func severityFromScore(score float64, version string) string { + switch { + case score <= 0: + return sevNone + case score >= cvssCriticalMin && !strings.HasPrefix(version, cvssVersion2Prefix): + return sevCritical + case score >= cvssHighMin: + return sevHigh + case score >= cvssMediumMin: + return sevMedium + default: + return sevLow + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cve/cvelist_test.go b/PROJECTS/intermediate/security-news-scraper/internal/cve/cvelist_test.go new file mode 100644 index 00000000..6d9b4e03 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/cve/cvelist_test.go @@ -0,0 +1,132 @@ +// ©AngelaMos | 2026 +// cvelist_test.go + +package cve + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" +) + +func TestCVEListParsesLog4Shell(t *testing.T) { + body, err := os.ReadFile("../../testdata/cvelist/CVE-2021-44228.json") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(body) + })) + defer srv.Close() + + client := NewCVEListClient(srv.Client(), srv.URL) + res, err := client.Fetch(context.Background(), "CVE-2021-44228") + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if !res.Found { + t.Fatal("Found = false, want true") + } + if res.CVSSScore == nil || *res.CVSSScore != 10.0 { + t.Errorf("CVSSScore = %v, want 10.0", res.CVSSScore) + } + if res.CVSSVersion != "3.1" || res.CVSSSeverity != "CRITICAL" { + t.Errorf("cvss = %q/%q, want 3.1/CRITICAL", res.CVSSVersion, res.CVSSSeverity) + } + if res.CVSSVector != "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H" { + t.Errorf("vector = %q", res.CVSSVector) + } + if res.CWE != "CWE-502" { + t.Errorf("CWE = %q, want CWE-502 (first problemType)", res.CWE) + } + if !strings.Contains(res.Description, "Log4j") { + t.Errorf("description missing Log4j: %q", res.Description) + } + if res.Published == "" || res.Modified == "" { + t.Errorf("published/modified empty: %q / %q", res.Published, res.Modified) + } +} + +func TestCVEListNotFound(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + client := NewCVEListClient(srv.Client(), srv.URL) + res, err := client.Fetch(context.Background(), "CVE-2099-99999") + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if res.Found { + t.Error("Found = true on 404, want false") + } +} + +func TestCVEListRecordURL(t *testing.T) { + client := NewCVEListClient(http.DefaultClient, "https://example.com/cves") + cases := map[string]string{ + "CVE-2021-44228": "https://example.com/cves/2021/44xxx/CVE-2021-44228.json", + "CVE-2021-1234": "https://example.com/cves/2021/1xxx/CVE-2021-1234.json", + "CVE-2023-12345": "https://example.com/cves/2023/12xxx/CVE-2023-12345.json", + } + for id, want := range cases { + got, err := client.recordURL(id) + if err != nil { + t.Errorf("recordURL(%q): %v", id, err) + continue + } + if got != want { + t.Errorf("recordURL(%q) = %q, want %q", id, got, want) + } + } + if _, err := client.recordURL("not-a-cve"); err == nil { + t.Error("expected error for malformed id") + } + if _, err := client.recordURL("CVE-2021-12"); err == nil { + t.Error("expected error for a too-short cve number") + } +} + +func serveJSON(t *testing.T, body string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return srv +} + +func TestCVEListCNAMetrics(t *testing.T) { + srv := serveJSON(t, `{"cveMetadata":{"datePublished":"2020-01-01T00:00:00Z","dateUpdated":"2020-02-01T00:00:00Z"},"containers":{"cna":{"descriptions":[{"lang":"en","value":"cna container path"}],"problemTypes":[{"descriptions":[{"lang":"en","cweId":"CWE-79"}]}],"metrics":[{"cvssV3_1":{"version":"3.1","baseScore":7.5,"baseSeverity":"HIGH","vectorString":"CVSS:3.1/AV:N/AC:L"}}]}}}`) + res, err := NewCVEListClient(srv.Client(), srv.URL).Fetch(context.Background(), "CVE-2020-0001") + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if res.CVSSScore == nil || *res.CVSSScore != 7.5 || res.CVSSVersion != "3.1" || res.CVSSSeverity != "HIGH" { + t.Errorf("cna metrics = %v/%q/%q, want 7.5/3.1/HIGH", res.CVSSScore, res.CVSSVersion, res.CVSSSeverity) + } + if res.CWE != "CWE-79" { + t.Errorf("CWE = %q, want CWE-79", res.CWE) + } +} + +func TestCVEListV2SeverityNotCritical(t *testing.T) { + srv := serveJSON(t, `{"cveMetadata":{"datePublished":"2005-01-01T00:00:00Z","dateUpdated":"2005-01-01T00:00:00Z"},"containers":{"cna":{"descriptions":[{"lang":"en","value":"v2 only"}],"metrics":[{"cvssV2_0":{"version":"2.0","baseScore":9.0,"vectorString":"AV:N/AC:L/Au:N/C:C/I:C/A:C"}}]}}}`) + res, err := NewCVEListClient(srv.Client(), srv.URL).Fetch(context.Background(), "CVE-2005-0001") + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if res.CVSSVersion != "2.0" || res.CVSSScore == nil || *res.CVSSScore != 9.0 { + t.Errorf("v2 = %q/%v, want 2.0/9.0", res.CVSSVersion, res.CVSSScore) + } + if res.CVSSSeverity != "HIGH" { + t.Errorf("v2 severity = %q, want HIGH (CVSS v2 has no CRITICAL tier)", res.CVSSSeverity) + } +} + +var _ CVESource = (*CVEListClient)(nil) +var _ CVESource = (*NVDClient)(nil) diff --git a/PROJECTS/intermediate/security-news-scraper/internal/cve/nvd.go b/PROJECTS/intermediate/security-news-scraper/internal/cve/nvd.go index 6e2be481..74f063ee 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/cve/nvd.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/cve/nvd.go @@ -24,18 +24,6 @@ const ( langEnglish = "en" ) -type NVDResult struct { - Found bool - Description string - CVSSScore *float64 - CVSSVersion string - CVSSSeverity string - CVSSVector string - CWE string - Published string - Modified string -} - type NVDClient struct { http *http.Client baseURL string @@ -58,7 +46,7 @@ func NewNVDClient(client *http.Client, baseURL, apiKey string) *NVDClient { } } -func (c *NVDClient) Fetch(ctx context.Context, cveID string) (NVDResult, error) { +func (c *NVDClient) Fetch(ctx context.Context, cveID string) (CoreResult, error) { endpoint := c.baseURL + "?" + url.Values{"cveId": {cveID}}.Encode() header := http.Header{} if c.apiKey != "" { @@ -69,23 +57,23 @@ func (c *NVDClient) Fetch(ctx context.Context, cveID string) (NVDResult, error) for attempt := 0; attempt <= nvdMaxRetries; attempt++ { if attempt > 0 { if err := sleep(ctx, c.backoffBase*time.Duration(1<<(attempt-1))); err != nil { - return NVDResult{}, err + return CoreResult{}, err } } if err := c.limiter.Wait(ctx); err != nil { - return NVDResult{}, err + return CoreResult{}, err } err := getJSON(ctx, c.http, endpoint, header, &env) if err == nil { break } if !retriable(err) || attempt == nvdMaxRetries { - return NVDResult{}, err + return CoreResult{}, err } } if env.TotalResults == 0 || len(env.Vulnerabilities) == 0 { - return NVDResult{Found: false}, nil + return CoreResult{Found: false}, nil } return env.Vulnerabilities[0].CVE.toResult(), nil } @@ -140,8 +128,8 @@ type nvdCVSSData struct { BaseSeverity string `json:"baseSeverity"` } -func (v nvdCVE) toResult() NVDResult { - res := NVDResult{ +func (v nvdCVE) toResult() CoreResult { + res := CoreResult{ Found: true, Description: english(v.Descriptions), Published: v.Published, diff --git a/PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich.go b/PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich.go index 584ab983..51bd23de 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich.go @@ -14,7 +14,7 @@ import ( const secondsPerHour = 3600 type Clients struct { - NVD *cve.NVDClient + Core cve.CVESource KEV *cve.KEVClient EPSS *cve.EPSSClient } @@ -51,7 +51,7 @@ func Run(ctx context.Context, st *store.Store, clients Clients, now time.Time, p stats := Stats{Total: len(ids)} for _, id := range ids { - nvdRes, err := clients.NVD.Fetch(ctx, id) + coreRes, err := clients.Core.Fetch(ctx, id) if err != nil { if ctx.Err() != nil { return stats, ctx.Err() @@ -61,16 +61,16 @@ func Run(ctx context.Context, st *store.Store, clients Clients, now time.Time, p } rec := store.CVE{ID: id, EnrichedAt: now.Unix()} - if nvdRes.Found { + if coreRes.Found { rec.EnrichStatus = store.EnrichStatusOK - rec.Description = nvdRes.Description - rec.CVSSScore = nvdRes.CVSSScore - rec.CVSSVersion = nvdRes.CVSSVersion - rec.CVSSSeverity = nvdRes.CVSSSeverity - rec.CVSSVector = nvdRes.CVSSVector - rec.CWE = nvdRes.CWE - rec.NVDPublished = nvdRes.Published - rec.NVDModified = nvdRes.Modified + rec.Description = coreRes.Description + rec.CVSSScore = coreRes.CVSSScore + rec.CVSSVersion = coreRes.CVSSVersion + rec.CVSSSeverity = coreRes.CVSSSeverity + rec.CVSSVector = coreRes.CVSSVector + rec.CWE = coreRes.CWE + rec.NVDPublished = coreRes.Published + rec.NVDModified = coreRes.Modified } else { rec.EnrichStatus = store.EnrichStatusNotFound } @@ -92,7 +92,7 @@ func Run(ctx context.Context, st *store.Store, clients Clients, now time.Time, p stats.Errors++ continue } - if nvdRes.Found { + if coreRes.Found { stats.Enriched++ } else { stats.NotFound++ diff --git a/PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich_test.go b/PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich_test.go index 6a1bdd06..03745e9c 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich_test.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/enrich/enrich_test.go @@ -50,7 +50,7 @@ func TestRunEnrichesLog4Shell(t *testing.T) { } clients := Clients{ - NVD: cve.NewNVDClient(nvd.Client(), nvd.URL, ""), + Core: cve.NewNVDClient(nvd.Client(), nvd.URL, ""), KEV: cve.NewKEVClient(kev.Client(), kev.URL), EPSS: cve.NewEPSSClient(epss.Client(), epss.URL), } @@ -97,7 +97,7 @@ func TestRunSkipsFreshAndMarksNotFound(t *testing.T) { t.Fatal(err) } clients := Clients{ - NVD: cve.NewNVDClient(nvd.Client(), nvd.URL, ""), + Core: cve.NewNVDClient(nvd.Client(), nvd.URL, ""), KEV: cve.NewKEVClient(kev.Client(), kev.URL), EPSS: cve.NewEPSSClient(epss.Client(), epss.URL), } diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/cvelist/CVE-2021-44228.json b/PROJECTS/intermediate/security-news-scraper/testdata/cvelist/CVE-2021-44228.json new file mode 100644 index 00000000..be61eac0 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/cvelist/CVE-2021-44228.json @@ -0,0 +1,796 @@ +{ + "dataType": "CVE_RECORD", + "dataVersion": "5.1", + "cveMetadata": { + "state": "PUBLISHED", + "cveId": "CVE-2021-44228", + "assignerOrgId": "f0158376-9dc2-43b6-827c-5f631a4d8d09", + "assignerShortName": "apache", + "dateUpdated": "2025-10-21T23:25:23.121Z", + "dateReserved": "2021-11-26T00:00:00.000Z", + "datePublished": "2021-12-10T00:00:00.000Z" + }, + "containers": { + "cna": { + "title": "Apache Log4j2 JNDI features do not protect against attacker controlled LDAP and other JNDI related endpoints", + "providerMetadata": { + "orgId": "f0158376-9dc2-43b6-827c-5f631a4d8d09", + "shortName": "apache", + "dateUpdated": "2023-04-03T00:00:00.000Z" + }, + "descriptions": [ + { + "lang": "en", + "value": "Apache Log4j2 2.0-beta9 through 2.15.0 (excluding security releases 2.12.2, 2.12.3, and 2.3.1) JNDI features used in configuration, log messages, and parameters do not protect against attacker controlled LDAP and other JNDI related endpoints. An attacker who can control log messages or log message parameters can execute arbitrary code loaded from LDAP servers when message lookup substitution is enabled. From log4j 2.15.0, this behavior has been disabled by default. From version 2.16.0 (along with 2.12.2, 2.12.3, and 2.3.1), this functionality has been completely removed. Note that this vulnerability is specific to log4j-core and does not affect log4net, log4cxx, or other Apache Logging Services projects." + } + ], + "affected": [ + { + "vendor": "Apache Software Foundation", + "product": "Apache Log4j2", + "versions": [ + { + "version": "2.0-beta9", + "status": "affected", + "lessThan": "log4j-core*", + "versionType": "custom", + "changes": [ + { + "at": "2.3.1", + "status": "unaffected" + }, + { + "at": "2.4", + "status": "affected" + }, + { + "at": "2.12.2", + "status": "unaffected" + }, + { + "at": "2.13.0", + "status": "affected" + }, + { + "at": "2.15.0", + "status": "unaffected" + } + ] + } + ] + } + ], + "references": [ + { + "url": "https://logging.apache.org/log4j/2.x/security.html" + }, + { + "name": "[oss-security] 20211210 CVE-2021-44228: Apache Log4j2 JNDI features do not protect against attacker controlled LDAP and other JNDI related endpoints", + "tags": [ + "mailing-list" + ], + "url": "http://www.openwall.com/lists/oss-security/2021/12/10/1" + }, + { + "name": "[oss-security] 20211210 Re: CVE-2021-44228: Apache Log4j2 JNDI features do not protect against attacker controlled LDAP and other JNDI related endpoints", + "tags": [ + "mailing-list" + ], + "url": "http://www.openwall.com/lists/oss-security/2021/12/10/2" + }, + { + "name": "20211210 Vulnerability in Apache Log4j Library Affecting Cisco Products: December 2021", + "tags": [ + "vendor-advisory" + ], + "url": "https://tools.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd" + }, + { + "name": "[oss-security] 20211210 Re: CVE-2021-44228: Apache Log4j2 JNDI features do not protect against attacker controlled LDAP and other JNDI related endpoints", + "tags": [ + "mailing-list" + ], + "url": "http://www.openwall.com/lists/oss-security/2021/12/10/3" + }, + { + "url": "https://security.netapp.com/advisory/ntap-20211210-0007/" + }, + { + "url": "http://packetstormsecurity.com/files/165225/Apache-Log4j2-2.14.1-Remote-Code-Execution.html" + }, + { + "url": "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032" + }, + { + "url": "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html" + }, + { + "name": "DSA-5020", + "tags": [ + "vendor-advisory" + ], + "url": "https://www.debian.org/security/2021/dsa-5020" + }, + { + "name": "[debian-lts-announce] 20211212 [SECURITY] [DLA 2842-1] apache-log4j2 security update", + "tags": [ + "mailing-list" + ], + "url": "https://lists.debian.org/debian-lts-announce/2021/12/msg00007.html" + }, + { + "name": "FEDORA-2021-f0f501d01f", + "tags": [ + "vendor-advisory" + ], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/VU57UJDCFIASIO35GC55JMKSRXJMCDFM/" + }, + { + "name": "Microsoft’s Response to CVE-2021-44228 Apache Log4j 2", + "tags": [ + "vendor-advisory" + ], + "url": "https://msrc-blog.microsoft.com/2021/12/11/microsofts-response-to-cve-2021-44228-apache-log4j2/" + }, + { + "name": "[oss-security] 20211213 Re: CVE-2021-4104: Deserialization of untrusted data in JMSAppender in Apache Log4j 1.2", + "tags": [ + "mailing-list" + ], + "url": "http://www.openwall.com/lists/oss-security/2021/12/13/2" + }, + { + "name": "[oss-security] 20211213 CVE-2021-4104: Deserialization of untrusted data in JMSAppender in Apache Log4j 1.2", + "tags": [ + "mailing-list" + ], + "url": "http://www.openwall.com/lists/oss-security/2021/12/13/1" + }, + { + "name": "[oss-security] 20211214 CVE-2021-45046: Apache Log4j2 Thread Context Message Pattern and Context Lookup Pattern vulnerable to a denial of service attack", + "tags": [ + "mailing-list" + ], + "url": "http://www.openwall.com/lists/oss-security/2021/12/14/4" + }, + { + "name": "20211210 A Vulnerability in Apache Log4j Library Affecting Cisco Products: December 2021", + "tags": [ + "vendor-advisory" + ], + "url": "https://tools.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd" + }, + { + "name": "VU#930724", + "tags": [ + "third-party-advisory" + ], + "url": "https://www.kb.cert.org/vuls/id/930724" + }, + { + "url": "https://twitter.com/kurtseifried/status/1469345530182455296" + }, + { + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf" + }, + { + "url": "http://packetstormsecurity.com/files/165260/VMware-Security-Advisory-2021-0028.html" + }, + { + "url": "http://packetstormsecurity.com/files/165270/Apache-Log4j2-2.14.1-Remote-Code-Execution.html" + }, + { + "url": "http://packetstormsecurity.com/files/165261/Apache-Log4j2-2.14.1-Information-Disclosure.html" + }, + { + "url": "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html" + }, + { + "name": "20211210 Vulnerabilities in Apache Log4j Library Affecting Cisco Products: December 2021", + "tags": [ + "vendor-advisory" + ], + "url": "https://tools.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd" + }, + { + "name": "[oss-security] 20211215 Re: CVE-2021-45046: Apache Log4j2 Thread Context Message Pattern and Context Lookup Pattern vulnerable to a denial of service attack", + "tags": [ + "mailing-list" + ], + "url": "http://www.openwall.com/lists/oss-security/2021/12/15/3" + }, + { + "url": "http://packetstormsecurity.com/files/165282/Log4j-Payload-Generator.html" + }, + { + "url": "http://packetstormsecurity.com/files/165281/Log4j2-Log4Shell-Regexes.html" + }, + { + "url": "http://packetstormsecurity.com/files/165307/Log4j-Remote-Code-Execution-Word-Bypassing.html" + }, + { + "url": "http://packetstormsecurity.com/files/165311/log4j-scan-Extensive-Scanner.html" + }, + { + "url": "http://packetstormsecurity.com/files/165306/L4sh-Log4j-Remote-Code-Execution.html" + }, + { + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf" + }, + { + "name": "FEDORA-2021-66d6c484f3", + "tags": [ + "vendor-advisory" + ], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/M5CSVUNV4HWZZXGOKNSK6L7RPM7BOKIB/" + }, + { + "url": "http://packetstormsecurity.com/files/165371/VMware-Security-Advisory-2021-0028.4.html" + }, + { + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf" + }, + { + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf" + }, + { + "url": "https://www.oracle.com/security-alerts/cpujan2022.html" + }, + { + "url": "http://packetstormsecurity.com/files/165532/Log4Shell-HTTP-Header-Injection.html" + }, + { + "url": "https://github.com/cisagov/log4j-affected-db/blob/develop/SOFTWARE-LIST.md" + }, + { + "url": "http://packetstormsecurity.com/files/165642/VMware-vCenter-Server-Unauthenticated-Log4Shell-JNDI-Injection-Remote-Code-Execution.html" + }, + { + "url": "http://packetstormsecurity.com/files/165673/UniFi-Network-Application-Unauthenticated-Log4Shell-Remote-Code-Execution.html" + }, + { + "name": "20220314 APPLE-SA-2022-03-14-7 Xcode 13.3", + "tags": [ + "mailing-list" + ], + "url": "http://seclists.org/fulldisclosure/2022/Mar/23" + }, + { + "url": "https://www.bentley.com/en/common-vulnerability-exposure/be-2022-0001" + }, + { + "url": "https://github.com/cisagov/log4j-affected-db" + }, + { + "url": "https://support.apple.com/kb/HT213189" + }, + { + "url": "https://www.oracle.com/security-alerts/cpuapr2022.html" + }, + { + "url": "https://github.com/nu11secur1ty/CVE-mitre/tree/main/CVE-2021-44228" + }, + { + "url": "https://www.nu11secur1ty.com/2021/12/cve-2021-44228.html" + }, + { + "name": "20220721 Open-Xchange Security Advisory 2022-07-21", + "tags": [ + "mailing-list" + ], + "url": "http://seclists.org/fulldisclosure/2022/Jul/11" + }, + { + "url": "http://packetstormsecurity.com/files/167794/Open-Xchange-App-Suite-7.10.x-Cross-Site-Scripting-Command-Injection.html" + }, + { + "url": "http://packetstormsecurity.com/files/167917/MobileIron-Log4Shell-Remote-Command-Execution.html" + }, + { + "name": "20221208 Intel Data Center Manager <= 5.1 Local Privileges Escalation", + "tags": [ + "mailing-list" + ], + "url": "http://seclists.org/fulldisclosure/2022/Dec/2" + }, + { + "url": "http://packetstormsecurity.com/files/171626/AD-Manager-Plus-7122-Remote-Code-Execution.html" + } + ], + "credits": [ + { + "lang": "en", + "value": "This issue was discovered by Chen Zhaojun of Alibaba Cloud Security Team." + } + ], + "metrics": [ + { + "other": { + "type": "unknown", + "content": { + "other": "critical" + } + } + } + ], + "problemTypes": [ + { + "descriptions": [ + { + "type": "CWE", + "lang": "en", + "description": "CWE-502 Deserialization of Untrusted Data", + "cweId": "CWE-502" + } + ] + }, + { + "descriptions": [ + { + "type": "CWE", + "lang": "en", + "description": "CWE-400 Uncontrolled Resource Consumption", + "cweId": "CWE-400" + } + ] + }, + { + "descriptions": [ + { + "type": "CWE", + "lang": "en", + "description": "CWE-20 Improper Input Validation", + "cweId": "CWE-20" + } + ] + } + ], + "x_generator": { + "engine": "Vulnogram 0.0.9" + }, + "source": { + "discovery": "UNKNOWN" + } + }, + "adp": [ + { + "providerMetadata": { + "orgId": "af854a3a-2127-422b-91ae-364da2661108", + "shortName": "CVE", + "dateUpdated": "2024-08-04T04:17:24.696Z" + }, + "title": "CVE Program Container", + "references": [ + { + "url": "https://logging.apache.org/log4j/2.x/security.html", + "tags": [ + "x_transferred" + ] + }, + { + "name": "[oss-security] 20211210 CVE-2021-44228: Apache Log4j2 JNDI features do not protect against attacker controlled LDAP and other JNDI related endpoints", + "tags": [ + "mailing-list", + "x_transferred" + ], + "url": "http://www.openwall.com/lists/oss-security/2021/12/10/1" + }, + { + "name": "[oss-security] 20211210 Re: CVE-2021-44228: Apache Log4j2 JNDI features do not protect against attacker controlled LDAP and other JNDI related endpoints", + "tags": [ + "mailing-list", + "x_transferred" + ], + "url": "http://www.openwall.com/lists/oss-security/2021/12/10/2" + }, + { + "name": "20211210 Vulnerability in Apache Log4j Library Affecting Cisco Products: December 2021", + "tags": [ + "vendor-advisory", + "x_transferred" + ], + "url": "https://tools.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd" + }, + { + "name": "[oss-security] 20211210 Re: CVE-2021-44228: Apache Log4j2 JNDI features do not protect against attacker controlled LDAP and other JNDI related endpoints", + "tags": [ + "mailing-list", + "x_transferred" + ], + "url": "http://www.openwall.com/lists/oss-security/2021/12/10/3" + }, + { + "url": "https://security.netapp.com/advisory/ntap-20211210-0007/", + "tags": [ + "x_transferred" + ] + }, + { + "url": "http://packetstormsecurity.com/files/165225/Apache-Log4j2-2.14.1-Remote-Code-Execution.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032", + "tags": [ + "x_transferred" + ] + }, + { + "url": "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html", + "tags": [ + "x_transferred" + ] + }, + { + "name": "DSA-5020", + "tags": [ + "vendor-advisory", + "x_transferred" + ], + "url": "https://www.debian.org/security/2021/dsa-5020" + }, + { + "name": "[debian-lts-announce] 20211212 [SECURITY] [DLA 2842-1] apache-log4j2 security update", + "tags": [ + "mailing-list", + "x_transferred" + ], + "url": "https://lists.debian.org/debian-lts-announce/2021/12/msg00007.html" + }, + { + "name": "FEDORA-2021-f0f501d01f", + "tags": [ + "vendor-advisory", + "x_transferred" + ], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/VU57UJDCFIASIO35GC55JMKSRXJMCDFM/" + }, + { + "name": "Microsoft’s Response to CVE-2021-44228 Apache Log4j 2", + "tags": [ + "vendor-advisory", + "x_transferred" + ], + "url": "https://msrc-blog.microsoft.com/2021/12/11/microsofts-response-to-cve-2021-44228-apache-log4j2/" + }, + { + "name": "[oss-security] 20211213 Re: CVE-2021-4104: Deserialization of untrusted data in JMSAppender in Apache Log4j 1.2", + "tags": [ + "mailing-list", + "x_transferred" + ], + "url": "http://www.openwall.com/lists/oss-security/2021/12/13/2" + }, + { + "name": "[oss-security] 20211213 CVE-2021-4104: Deserialization of untrusted data in JMSAppender in Apache Log4j 1.2", + "tags": [ + "mailing-list", + "x_transferred" + ], + "url": "http://www.openwall.com/lists/oss-security/2021/12/13/1" + }, + { + "name": "[oss-security] 20211214 CVE-2021-45046: Apache Log4j2 Thread Context Message Pattern and Context Lookup Pattern vulnerable to a denial of service attack", + "tags": [ + "mailing-list", + "x_transferred" + ], + "url": "http://www.openwall.com/lists/oss-security/2021/12/14/4" + }, + { + "name": "20211210 A Vulnerability in Apache Log4j Library Affecting Cisco Products: December 2021", + "tags": [ + "vendor-advisory", + "x_transferred" + ], + "url": "https://tools.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd" + }, + { + "name": "VU#930724", + "tags": [ + "third-party-advisory", + "x_transferred" + ], + "url": "https://www.kb.cert.org/vuls/id/930724" + }, + { + "url": "https://twitter.com/kurtseifried/status/1469345530182455296", + "tags": [ + "x_transferred" + ] + }, + { + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf", + "tags": [ + "x_transferred" + ] + }, + { + "url": "http://packetstormsecurity.com/files/165260/VMware-Security-Advisory-2021-0028.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "http://packetstormsecurity.com/files/165270/Apache-Log4j2-2.14.1-Remote-Code-Execution.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "http://packetstormsecurity.com/files/165261/Apache-Log4j2-2.14.1-Information-Disclosure.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html", + "tags": [ + "x_transferred" + ] + }, + { + "name": "20211210 Vulnerabilities in Apache Log4j Library Affecting Cisco Products: December 2021", + "tags": [ + "vendor-advisory", + "x_transferred" + ], + "url": "https://tools.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd" + }, + { + "name": "[oss-security] 20211215 Re: CVE-2021-45046: Apache Log4j2 Thread Context Message Pattern and Context Lookup Pattern vulnerable to a denial of service attack", + "tags": [ + "mailing-list", + "x_transferred" + ], + "url": "http://www.openwall.com/lists/oss-security/2021/12/15/3" + }, + { + "url": "http://packetstormsecurity.com/files/165282/Log4j-Payload-Generator.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "http://packetstormsecurity.com/files/165281/Log4j2-Log4Shell-Regexes.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "http://packetstormsecurity.com/files/165307/Log4j-Remote-Code-Execution-Word-Bypassing.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "http://packetstormsecurity.com/files/165311/log4j-scan-Extensive-Scanner.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "http://packetstormsecurity.com/files/165306/L4sh-Log4j-Remote-Code-Execution.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf", + "tags": [ + "x_transferred" + ] + }, + { + "name": "FEDORA-2021-66d6c484f3", + "tags": [ + "vendor-advisory", + "x_transferred" + ], + "url": "https://lists.fedoraproject.org/archives/list/package-announce%40lists.fedoraproject.org/message/M5CSVUNV4HWZZXGOKNSK6L7RPM7BOKIB/" + }, + { + "url": "http://packetstormsecurity.com/files/165371/VMware-Security-Advisory-2021-0028.4.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf", + "tags": [ + "x_transferred" + ] + }, + { + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf", + "tags": [ + "x_transferred" + ] + }, + { + "url": "https://www.oracle.com/security-alerts/cpujan2022.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "http://packetstormsecurity.com/files/165532/Log4Shell-HTTP-Header-Injection.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "https://github.com/cisagov/log4j-affected-db/blob/develop/SOFTWARE-LIST.md", + "tags": [ + "x_transferred" + ] + }, + { + "url": "http://packetstormsecurity.com/files/165642/VMware-vCenter-Server-Unauthenticated-Log4Shell-JNDI-Injection-Remote-Code-Execution.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "http://packetstormsecurity.com/files/165673/UniFi-Network-Application-Unauthenticated-Log4Shell-Remote-Code-Execution.html", + "tags": [ + "x_transferred" + ] + }, + { + "name": "20220314 APPLE-SA-2022-03-14-7 Xcode 13.3", + "tags": [ + "mailing-list", + "x_transferred" + ], + "url": "http://seclists.org/fulldisclosure/2022/Mar/23" + }, + { + "url": "https://www.bentley.com/en/common-vulnerability-exposure/be-2022-0001", + "tags": [ + "x_transferred" + ] + }, + { + "url": "https://github.com/cisagov/log4j-affected-db", + "tags": [ + "x_transferred" + ] + }, + { + "url": "https://support.apple.com/kb/HT213189", + "tags": [ + "x_transferred" + ] + }, + { + "url": "https://www.oracle.com/security-alerts/cpuapr2022.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "https://github.com/nu11secur1ty/CVE-mitre/tree/main/CVE-2021-44228", + "tags": [ + "x_transferred" + ] + }, + { + "url": "https://www.nu11secur1ty.com/2021/12/cve-2021-44228.html", + "tags": [ + "x_transferred" + ] + }, + { + "name": "20220721 Open-Xchange Security Advisory 2022-07-21", + "tags": [ + "mailing-list", + "x_transferred" + ], + "url": "http://seclists.org/fulldisclosure/2022/Jul/11" + }, + { + "url": "http://packetstormsecurity.com/files/167794/Open-Xchange-App-Suite-7.10.x-Cross-Site-Scripting-Command-Injection.html", + "tags": [ + "x_transferred" + ] + }, + { + "url": "http://packetstormsecurity.com/files/167917/MobileIron-Log4Shell-Remote-Command-Execution.html", + "tags": [ + "x_transferred" + ] + }, + { + "name": "20221208 Intel Data Center Manager <= 5.1 Local Privileges Escalation", + "tags": [ + "mailing-list", + "x_transferred" + ], + "url": "http://seclists.org/fulldisclosure/2022/Dec/2" + }, + { + "url": "http://packetstormsecurity.com/files/171626/AD-Manager-Plus-7122-Remote-Code-Execution.html", + "tags": [ + "x_transferred" + ] + } + ] + }, + { + "metrics": [ + { + "cvssV3_1": { + "scope": "CHANGED", + "version": "3.1", + "baseScore": 10, + "attackVector": "NETWORK", + "baseSeverity": "CRITICAL", + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", + "integrityImpact": "HIGH", + "userInteraction": "NONE", + "attackComplexity": "LOW", + "availabilityImpact": "HIGH", + "privilegesRequired": "NONE", + "confidentialityImpact": "HIGH" + } + }, + { + "other": { + "type": "ssvc", + "content": { + "id": "CVE-2021-44228", + "role": "CISA Coordinator", + "options": [ + { + "Exploitation": "active" + }, + { + "Automatable": "yes" + }, + { + "Technical Impact": "total" + } + ], + "version": "2.0.3", + "timestamp": "2025-02-04T14:25:34.416117Z" + } + } + }, + { + "other": { + "type": "kev", + "content": { + "dateAdded": "2021-12-10", + "reference": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog?field_cve=CVE-2021-44228" + } + } + } + ], + "references": [ + { + "url": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog?field_cve=CVE-2021-44228", + "tags": [ + "government-resource" + ] + } + ], + "timeline": [ + { + "time": "2021-12-10T00:00:00.000Z", + "lang": "en", + "value": "CVE-2021-44228 added to CISA KEV" + } + ], + "title": "CISA ADP Vulnrichment", + "providerMetadata": { + "orgId": "134c704f-9b21-4f2e-91b3-4a467353bcc0", + "shortName": "CISA-ADP", + "dateUpdated": "2025-10-21T23:25:23.121Z" + } + } + ] + } +} From 0a86bb8593d13cbb6072098f137d4b8ddffc3097 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Mon, 6 Jul 2026 21:26:10 -0400 Subject: [PATCH 08/15] feat(nadezhda): AI setup wizard (nadezhda ai) + credential store + Ollama port fix Add a re-runnable interactive wizard that configures an AI provider for the ideation layer: Claude, OpenAI, Gemini, or a local Ollama. Keys persist to ~/.config/nadezhda/credentials (0600, atomic temp+rename) and auto-load on every run; a shell-exported key always wins over the file. A key-name allowlist (NADEZHDA_* / *_API_KEY) blocks env-name injection into spawned subprocesses. The Ollama path detects a running instance and auto-configures it, else writes an embedded compose file plus one docker command, else prints an install hint. Ollama's host port is remapped to a non-default 39847 (OLLAMA_HOST_PORT) to avoid colliding with an existing Ollama on 11434; the container-internal port stays 11434. Thread a signal-cancellable root context through cobra (ExecuteContext). Default the Anthropic model to claude-sonnet-4-6. --- .../security-news-scraper/cmd/nadezhda/ai.go | 22 +++ .../cmd/nadezhda/main.go | 10 +- .../cmd/nadezhda/root.go | 22 +++ .../intermediate/security-news-scraper/go.mod | 1 + .../intermediate/security-news-scraper/go.sum | 2 + .../internal/config/config.go | 4 +- .../internal/setup/credentials.go | 145 ++++++++++++++++ .../internal/setup/credentials_test.go | 144 ++++++++++++++++ .../internal/setup/ollama.compose.yml | 30 ++++ .../internal/setup/ollama.go | 67 ++++++++ .../internal/setup/ollama_test.go | 62 +++++++ .../internal/setup/paths.go | 33 ++++ .../internal/setup/wizard.go | 158 ++++++++++++++++++ .../internal/setup/wizard_test.go | 82 +++++++++ .../security-news-scraper/justfile | 10 ++ 15 files changed, 788 insertions(+), 4 deletions(-) create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/ai.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/setup/credentials.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/setup/credentials_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/setup/ollama.compose.yml create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/setup/ollama.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/setup/ollama_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/setup/paths.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/setup/wizard.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/setup/wizard_test.go diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/ai.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/ai.go new file mode 100644 index 00000000..62beefcb --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/ai.go @@ -0,0 +1,22 @@ +// ©AngelaMos | 2026 +// ai.go + +package main + +import ( + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/setup" +) + +var aiCmd = &cobra.Command{ + Use: "ai", + Short: "Set up an AI provider for ideation (interactive, re-runnable)", + RunE: func(cmd *cobra.Command, args []string) error { + return setup.Run(cmd.InOrStdin(), cmd.OutOrStdout()) + }, +} + +func init() { + rootCmd.AddCommand(aiCmd) +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go index a4c7e183..3c1d5bcc 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go @@ -3,10 +3,16 @@ package main -import "os" +import ( + "context" + "os" + "os/signal" +) func main() { - if err := rootCmd.Execute(); err != nil { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + if err := rootCmd.ExecuteContext(ctx); err != nil { os.Exit(1) } } diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/root.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/root.go index 50cd452d..25b7ea95 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/root.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/root.go @@ -4,9 +4,12 @@ package main import ( + "os" + "github.com/spf13/cobra" "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/setup" ) var ( @@ -27,6 +30,9 @@ func init() { } func loadConfig() (config.Config, error) { + if err := setup.Load(); err != nil { + return config.Config{}, err + } cfg, err := config.Load(flagConfig) if err != nil { return config.Config{}, err @@ -34,5 +40,21 @@ func loadConfig() (config.Config, error) { if flagDB != "" { cfg.DBPath = flagDB } + if p := os.Getenv(setup.EnvProvider); p != "" { + cfg.AI.Enabled = true + cfg.AI.Provider = p + } + if u := os.Getenv(setup.EnvQwenURL); u != "" { + cfg.AI.Qwen.BaseURL = u + } return cfg, nil } + +func isInteractive(cmd *cobra.Command) bool { + f, ok := cmd.InOrStdin().(*os.File) + if !ok { + return false + } + fi, err := f.Stat() + return err == nil && fi.Mode()&os.ModeCharDevice != 0 +} diff --git a/PROJECTS/intermediate/security-news-scraper/go.mod b/PROJECTS/intermediate/security-news-scraper/go.mod index 382c7a78..f51665b2 100644 --- a/PROJECTS/intermediate/security-news-scraper/go.mod +++ b/PROJECTS/intermediate/security-news-scraper/go.mod @@ -11,6 +11,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/temoto/robotstxt v1.1.2 golang.org/x/sync v0.21.0 + golang.org/x/term v0.41.0 golang.org/x/time v0.15.0 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.53.0 diff --git a/PROJECTS/intermediate/security-news-scraper/go.sum b/PROJECTS/intermediate/security-news-scraper/go.sum index 7cf2a882..5445ca88 100644 --- a/PROJECTS/intermediate/security-news-scraper/go.sum +++ b/PROJECTS/intermediate/security-news-scraper/go.sum @@ -149,6 +149,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go index 030567f7..b26b0f96 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go @@ -42,14 +42,14 @@ const ( defaultWeightEPSS = 0.08 defaultAIProvider = "qwen" - defaultQwenBaseURL = "http://localhost:11434/v1" + defaultQwenBaseURL = "http://localhost:39847/v1" defaultQwenModel = "qwen2.5:7b" defaultOpenAIURL = "https://api.openai.com/v1" defaultOpenAIModel = "gpt-4o-mini" defaultGeminiURL = "https://generativelanguage.googleapis.com/v1beta/openai/" defaultGeminiModel = "gemini-2.5-flash" defaultAnthropicURL = "https://api.anthropic.com/v1" - defaultClaudeModel = "claude-opus-4-8" + defaultClaudeModel = "claude-sonnet-4-6" ) var defaultTrackingParams = []string{ diff --git a/PROJECTS/intermediate/security-news-scraper/internal/setup/credentials.go b/PROJECTS/intermediate/security-news-scraper/internal/setup/credentials.go new file mode 100644 index 00000000..178e8030 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/setup/credentials.go @@ -0,0 +1,145 @@ +// ©AngelaMos | 2026 +// credentials.go + +package setup + +import ( + "bufio" + "fmt" + "os" + "sort" + "strings" +) + +const ( + EnvProvider = "NADEZHDA_AI_PROVIDER" + EnvQwenURL = "NADEZHDA_QWEN_BASE_URL" + + commentMark = "#" + kvSeparator = "=" + + credentialKeyPrefix = "NADEZHDA_" + credentialKeySuffix = "_API_KEY" + tmpSuffix = ".tmp" + + credentialsHeader = "# ©AngelaMos | 2026\n# nadezhda credentials — auto-managed by `nadezhda ai`; keep private, do not commit\n\n" +) + +func Load() error { + path, err := CredentialsPath() + if err != nil { + return err + } + kv, err := readCredentials(path) + if err != nil { + return err + } + for k, v := range kv { + if !allowedKey(k) { + continue + } + if _, set := os.LookupEnv(k); set { + continue + } + if err := os.Setenv(k, v); err != nil { + return fmt.Errorf("setup: set %s: %w", k, err) + } + } + return nil +} + +func allowedKey(k string) bool { + return strings.HasPrefix(k, credentialKeyPrefix) || strings.HasSuffix(k, credentialKeySuffix) +} + +func NonSecretEnviron() []string { + env := os.Environ() + out := make([]string, 0, len(env)) + for _, e := range env { + if k, _, ok := strings.Cut(e, kvSeparator); ok && allowedKey(k) { + continue + } + out = append(out, e) + } + return out +} + +func Save(updates map[string]string) error { + path, err := CredentialsPath() + if err != nil { + return err + } + dir, err := ConfigDir() + if err != nil { + return err + } + if err := os.MkdirAll(dir, dirPerm); err != nil { + return fmt.Errorf("setup: create config dir: %w", err) + } + kv, err := readCredentials(path) + if err != nil { + return err + } + for k, v := range updates { + kv[k] = v + } + return writeCredentials(path, kv) +} + +func readCredentials(path string) (map[string]string, error) { + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return map[string]string{}, nil + } + return nil, fmt.Errorf("setup: read credentials: %w", err) + } + defer f.Close() + + kv := map[string]string{} + sc := bufio.NewScanner(f) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, commentMark) { + continue + } + k, v, ok := strings.Cut(line, kvSeparator) + if !ok { + continue + } + key := strings.TrimSpace(k) + if key == "" { + continue + } + kv[key] = strings.TrimSpace(v) + } + return kv, sc.Err() +} + +func writeCredentials(path string, kv map[string]string) error { + keys := make([]string, 0, len(kv)) + for k := range kv { + keys = append(keys, k) + } + sort.Strings(keys) + + var b strings.Builder + b.WriteString(credentialsHeader) + for _, k := range keys { + fmt.Fprintf(&b, "%s=%s\n", k, kv[k]) + } + + tmp := path + tmpSuffix + if err := os.WriteFile(tmp, []byte(b.String()), filePerm); err != nil { + return fmt.Errorf("setup: write credentials: %w", err) + } + if err := os.Chmod(tmp, filePerm); err != nil { + os.Remove(tmp) + return fmt.Errorf("setup: secure credentials: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + os.Remove(tmp) + return fmt.Errorf("setup: replace credentials: %w", err) + } + return nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/setup/credentials_test.go b/PROJECTS/intermediate/security-news-scraper/internal/setup/credentials_test.go new file mode 100644 index 00000000..e58c4fd5 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/setup/credentials_test.go @@ -0,0 +1,144 @@ +// ©AngelaMos | 2026 +// credentials_test.go + +package setup + +import ( + "os" + "testing" +) + +func TestCredentialsRoundTripAndPerms(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + + if err := Save(map[string]string{"OPENAI_API_KEY": "sk-abc", EnvProvider: "openai"}); err != nil { + t.Fatalf("Save: %v", err) + } + if err := Save(map[string]string{"NVD_API_KEY": "nvd-1"}); err != nil { + t.Fatalf("Save merge: %v", err) + } + + path, _ := CredentialsPath() + kv, err := readCredentials(path) + if err != nil { + t.Fatalf("readCredentials: %v", err) + } + if kv["OPENAI_API_KEY"] != "sk-abc" || kv[EnvProvider] != "openai" || kv["NVD_API_KEY"] != "nvd-1" { + t.Errorf("merge lost data: %v", kv) + } + + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if perm := fi.Mode().Perm(); perm != 0o600 { + t.Errorf("credentials perm = %o, want 600", perm) + } +} + +func TestLoadEnvWins(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + if err := Save(map[string]string{"OPENAI_API_KEY": "fromfile", "NADEZHDA_TEST_ONLY": "x"}); err != nil { + t.Fatal(err) + } + t.Setenv("OPENAI_API_KEY", "fromenv") + defer os.Unsetenv("NADEZHDA_TEST_ONLY") + + if err := Load(); err != nil { + t.Fatalf("Load: %v", err) + } + if got := os.Getenv("OPENAI_API_KEY"); got != "fromenv" { + t.Errorf("OPENAI_API_KEY = %q, want fromenv (env must win over file)", got) + } + if got := os.Getenv("NADEZHDA_TEST_ONLY"); got != "x" { + t.Errorf("NADEZHDA_TEST_ONLY = %q, want x (unset var loads from file)", got) + } +} + +func TestReadCredentialsSkipsCommentsAndBlanks(t *testing.T) { + path := t.TempDir() + "/creds" + content := "# header\n\nOPENAI_API_KEY=sk-1\n # indented comment\nbadline_no_sep\nNVD_API_KEY = nvd-2 \n" + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + kv, err := readCredentials(path) + if err != nil { + t.Fatal(err) + } + if kv["OPENAI_API_KEY"] != "sk-1" || kv["NVD_API_KEY"] != "nvd-2" || len(kv) != 2 { + t.Errorf("parsed = %v, want exactly 2 clean entries", kv) + } +} + +func TestSaveCreatesDir0700AndFile0600(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + if err := Save(map[string]string{"OPENAI_API_KEY": "k"}); err != nil { + t.Fatal(err) + } + dir, _ := ConfigDir() + if di, _ := os.Stat(dir); di.Mode().Perm() != 0o700 { + t.Errorf("config dir perms = %o, want 700", di.Mode().Perm()) + } + path, _ := CredentialsPath() + if fi, _ := os.Stat(path); fi.Mode().Perm() != 0o600 { + t.Errorf("credentials perms = %o, want 600", fi.Mode().Perm()) + } +} + +func TestSaveTightensExistingLooseFile(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + dir, _ := ConfigDir() + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + path, _ := CredentialsPath() + if err := os.WriteFile(path, []byte("OPENAI_API_KEY=old\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := Save(map[string]string{"OPENAI_API_KEY": "new"}); err != nil { + t.Fatal(err) + } + if fi, _ := os.Stat(path); fi.Mode().Perm() != 0o600 { + t.Errorf("Save left perms %o on a pre-existing 0644 file, want 600", fi.Mode().Perm()) + } +} + +func TestLoadRespectsSetEmptyEnv(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + if err := Save(map[string]string{"OPENAI_API_KEY": "fromfile"}); err != nil { + t.Fatal(err) + } + t.Setenv("OPENAI_API_KEY", "") + if err := Load(); err != nil { + t.Fatal(err) + } + if got := os.Getenv("OPENAI_API_KEY"); got != "" { + t.Errorf("an explicitly-cleared env var was overridden from file: %q", got) + } +} + +func TestLoadBlocksDisallowedAndEmptyKeys(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + dir, _ := ConfigDir() + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + path, _ := CredentialsPath() + if err := os.WriteFile(path, []byte("=orphan\nLD_PRELOAD=/tmp/evil.so\nOPENAI_API_KEY=ok\n"), 0o600); err != nil { + t.Fatal(err) + } + os.Unsetenv("LD_PRELOAD") + os.Unsetenv("OPENAI_API_KEY") + defer os.Unsetenv("LD_PRELOAD") + defer os.Unsetenv("OPENAI_API_KEY") + + if err := Load(); err != nil { + t.Fatalf("Load must tolerate an empty-key line, got: %v", err) + } + if _, set := os.LookupEnv("LD_PRELOAD"); set { + t.Error("LD_PRELOAD injected from credentials file — allowlist bypass (RCE vector)") + } + if os.Getenv("OPENAI_API_KEY") != "ok" { + t.Error("allowlisted OPENAI_API_KEY not loaded") + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/setup/ollama.compose.yml b/PROJECTS/intermediate/security-news-scraper/internal/setup/ollama.compose.yml new file mode 100644 index 00000000..91d3f289 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/setup/ollama.compose.yml @@ -0,0 +1,30 @@ +# ©AngelaMos | 2026 +# ollama.compose.yml + +name: nadezhda-ollama + +services: + ollama: + image: ollama/ollama:latest + container_name: nadezhda-ollama + ports: + - "127.0.0.1:${OLLAMA_HOST_PORT:-39847}:11434" + volumes: + - nadezhda-ollama-models:/root/.ollama + entrypoint: ["/bin/sh", "-c"] + command: + - | + ollama serve & + until ollama list >/dev/null 2>&1; do sleep 1; done + ollama pull qwen2.5:7b + wait + healthcheck: + test: ["CMD-SHELL", "ollama list | grep -q qwen2.5"] + interval: 10s + timeout: 5s + retries: 60 + start_period: 30s + restart: unless-stopped + +volumes: + nadezhda-ollama-models: diff --git a/PROJECTS/intermediate/security-news-scraper/internal/setup/ollama.go b/PROJECTS/intermediate/security-news-scraper/internal/setup/ollama.go new file mode 100644 index 00000000..4be0a70d --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/setup/ollama.go @@ -0,0 +1,67 @@ +// ©AngelaMos | 2026 +// ollama.go + +package setup + +import ( + _ "embed" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" +) + +//go:embed ollama.compose.yml +var ComposeYAML []byte + +const ( + composeFileName = "ollama.compose.yml" + composePerm = 0o644 + versionPath = "/api/version" + apiV1Suffix = "/v1" +) + +var ollamaCandidates = []string{ + "http://localhost:39847", + "http://localhost:11434", +} + +func DetectOllama(client *http.Client) string { + for _, base := range ollamaCandidates { + resp, err := client.Get(base + versionPath) + if err != nil { + continue + } + resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return base + } + } + return "" +} + +func WriteCompose(dir string) (string, error) { + path := filepath.Join(dir, composeFileName) + if err := os.WriteFile(path, ComposeYAML, composePerm); err != nil { + return "", fmt.Errorf("setup: write compose: %w", err) + } + return path, nil +} + +func HasBinary(name string) bool { + _, err := exec.LookPath(name) + return err == nil +} + +func OllamaReachable(baseURL string) bool { + base := strings.TrimSuffix(baseURL, apiV1Suffix) + client := &http.Client{Timeout: detectTimeout} + resp, err := client.Get(base + versionPath) + if err != nil { + return false + } + resp.Body.Close() + return resp.StatusCode == http.StatusOK +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/setup/ollama_test.go b/PROJECTS/intermediate/security-news-scraper/internal/setup/ollama_test.go new file mode 100644 index 00000000..4b9d7f07 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/setup/ollama_test.go @@ -0,0 +1,62 @@ +// ©AngelaMos | 2026 +// ollama_test.go + +package setup + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestDetectOllama(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == versionPath { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + old := ollamaCandidates + defer func() { ollamaCandidates = old }() + client := &http.Client{Timeout: 2 * time.Second} + + ollamaCandidates = []string{srv.URL} + if got := DetectOllama(client); got != srv.URL { + t.Errorf("DetectOllama = %q, want %q", got, srv.URL) + } + + ollamaCandidates = []string{"http://127.0.0.1:1"} + if got := DetectOllama(client); got != "" { + t.Errorf("DetectOllama with no server = %q, want empty", got) + } +} + +func TestWriteComposeMatchesEmbed(t *testing.T) { + yaml := string(ComposeYAML) + if len(yaml) == 0 || !strings.Contains(yaml, "nadezhda-ollama") || !strings.Contains(yaml, "qwen2.5:7b") { + t.Fatal("embedded compose is empty or missing expected content") + } + + dir := t.TempDir() + path, err := WriteCompose(dir) + if err != nil { + t.Fatalf("WriteCompose: %v", err) + } + if path != filepath.Join(dir, composeFileName) { + t.Errorf("path = %q", path) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != yaml { + t.Error("written compose differs from embedded content") + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/setup/paths.go b/PROJECTS/intermediate/security-news-scraper/internal/setup/paths.go new file mode 100644 index 00000000..ecea50c6 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/setup/paths.go @@ -0,0 +1,33 @@ +// ©AngelaMos | 2026 +// paths.go + +package setup + +import ( + "fmt" + "os" + "path/filepath" +) + +const ( + appDir = "nadezhda" + credentialsFile = "credentials" + dirPerm = 0o700 + filePerm = 0o600 +) + +func ConfigDir() (string, error) { + base, err := os.UserConfigDir() + if err != nil { + return "", fmt.Errorf("setup: locate config dir: %w", err) + } + return filepath.Join(base, appDir), nil +} + +func CredentialsPath() (string, error) { + dir, err := ConfigDir() + if err != nil { + return "", err + } + return filepath.Join(dir, credentialsFile), nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/setup/wizard.go b/PROJECTS/intermediate/security-news-scraper/internal/setup/wizard.go new file mode 100644 index 00000000..f964bd67 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/setup/wizard.go @@ -0,0 +1,158 @@ +// ©AngelaMos | 2026 +// wizard.go + +package setup + +import ( + "fmt" + "io" + "net/http" + "os" + "strings" + "time" + + "golang.org/x/term" +) + +const ( + ProviderClaude = "anthropic" + ProviderOpenAI = "openai" + ProviderGemini = "gemini" + ProviderQwen = "qwen" + + envOpenAIKey = "OPENAI_API_KEY" + envGeminiKey = "GEMINI_API_KEY" + envAnthropicKey = "ANTHROPIC_API_KEY" + envNVDKey = "NVD_API_KEY" + + composeQwenURL = "http://localhost:39847/v1" + nativeQwenURL = "http://localhost:11434/v1" + + detectTimeout = 3 * time.Second + ollamaInstall = "https://ollama.com/download" +) + +type apiChoice struct { + provider string + envKey string + label string +} + +var apiProviders = map[string]apiChoice{ + "1": {ProviderClaude, envAnthropicKey, "Claude (Anthropic)"}, + "2": {ProviderOpenAI, envOpenAIKey, "OpenAI"}, + "3": {ProviderGemini, envGeminiKey, "Gemini (Google)"}, +} + +func Run(in io.Reader, out io.Writer) error { + fmt.Fprintln(out, "nadezhda AI setup") + fmt.Fprintln(out, " 1) Claude (Anthropic) 2) OpenAI 3) Gemini 4) Ollama (local qwen, free, no key)") + choice := prompt(in, out, "choose a provider [1-4]: ") + + switch choice { + case "1", "2", "3": + return setupAPIProvider(in, out, apiProviders[choice]) + case "4": + return setupOllama(out) + default: + return fmt.Errorf("setup: invalid choice %q (want 1-4)", choice) + } +} + +func setupAPIProvider(in io.Reader, out io.Writer, c apiChoice) error { + fmt.Fprintf(out, "\nConfiguring %s.\n", c.label) + key := strings.TrimSpace(readSecret(in, out, fmt.Sprintf("paste your %s API key: ", c.label))) + if key == "" { + return fmt.Errorf("setup: no API key entered") + } + updates := map[string]string{ + EnvProvider: c.provider, + c.envKey: key, + } + if nvd := strings.TrimSpace(readSecret(in, out, "optional NVD token for the CVE booster (enter to skip): ")); nvd != "" { + updates[envNVDKey] = nvd + } + if err := Save(updates); err != nil { + return err + } + path, _ := CredentialsPath() + fmt.Fprintf(out, "\nSaved to %s (0600). %s is ready — run: nadezhda ideate\n", path, c.label) + return nil +} + +func setupOllama(out io.Writer) error { + fmt.Fprintln(out, "\nConfiguring Ollama (local qwen2.5, no API key).") + client := &http.Client{Timeout: detectTimeout} + + if base := DetectOllama(client); base != "" { + fmt.Fprintf(out, "Found a running Ollama at %s — using it.\n", base) + return saveOllama(out, base+"/v1") + } + + fmt.Fprintln(out, "No running Ollama found.") + switch { + case HasBinary("docker"): + dir, _ := os.Getwd() + path, err := WriteCompose(dir) + if err != nil { + return err + } + fmt.Fprintf(out, "Wrote %s. Start it with:\n docker compose -f %s up -d\n", path, path) + fmt.Fprintln(out, "(first run pulls qwen2.5:7b, a few minutes), then re-run `nadezhda ai` and pick 4.") + return saveOllama(out, composeQwenURL) + case HasBinary("ollama"): + fmt.Fprintln(out, "Ollama is installed but not running. Start it and pull the model:") + fmt.Fprintln(out, " ollama serve & then: ollama pull qwen2.5:7b") + return saveOllama(out, nativeQwenURL) + default: + fmt.Fprintf(out, "Install Ollama (%s) or Docker, then re-run `nadezhda ai`.\n", ollamaInstall) + return saveOllama(out, composeQwenURL) + } +} + +func saveOllama(out io.Writer, baseURL string) error { + if err := Save(map[string]string{EnvProvider: ProviderQwen, EnvQwenURL: baseURL}); err != nil { + return err + } + path, _ := CredentialsPath() + fmt.Fprintf(out, "Saved qwen (%s) to %s. Run: nadezhda ideate\n", baseURL, path) + return nil +} + +func prompt(in io.Reader, out io.Writer, label string) string { + fmt.Fprint(out, label) + return readLine(in) +} + +func readSecret(in io.Reader, out io.Writer, label string) string { + fmt.Fprint(out, label) + if f, ok := in.(*os.File); ok && term.IsTerminal(int(f.Fd())) { + b, err := term.ReadPassword(int(f.Fd())) + fmt.Fprintln(out) + if err != nil { + return "" + } + return string(b) + } + return readLine(in) +} + +func readLine(in io.Reader) string { + var b strings.Builder + buf := make([]byte, 1) + for { + n, err := in.Read(buf) + if n > 0 { + if buf[0] == '\n' { + break + } + if buf[0] != '\r' { + b.WriteByte(buf[0]) + } + } + if err != nil { + break + } + } + return strings.TrimSpace(b.String()) +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/setup/wizard_test.go b/PROJECTS/intermediate/security-news-scraper/internal/setup/wizard_test.go new file mode 100644 index 00000000..61648c3f --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/setup/wizard_test.go @@ -0,0 +1,82 @@ +// ©AngelaMos | 2026 +// wizard_test.go + +package setup + +import ( + "bytes" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestWizardAPIProvider(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + var out bytes.Buffer + in := strings.NewReader("2\nsk-test-key\n\n") + + if err := Run(in, &out); err != nil { + t.Fatalf("Run: %v", err) + } + path, _ := CredentialsPath() + kv, _ := readCredentials(path) + if kv[EnvProvider] != ProviderOpenAI || kv["OPENAI_API_KEY"] != "sk-test-key" { + t.Errorf("saved = %v, want openai + key", kv) + } + if _, ok := kv["NVD_API_KEY"]; ok { + t.Error("NVD key should be absent when skipped") + } + if !strings.Contains(out.String(), "OpenAI is ready") { + t.Errorf("output missing confirmation:\n%s", out.String()) + } + if strings.Contains(out.String(), "sk-test-key") { + t.Error("wizard echoed the API key to its output") + } +} + +func TestWizardOllamaDetected(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + old := ollamaCandidates + ollamaCandidates = []string{srv.URL} + defer func() { ollamaCandidates = old }() + + var out bytes.Buffer + if err := Run(strings.NewReader("4\n"), &out); err != nil { + t.Fatalf("Run: %v", err) + } + path, _ := CredentialsPath() + kv, _ := readCredentials(path) + if kv[EnvProvider] != ProviderQwen || kv[EnvQwenURL] != srv.URL+"/v1" { + t.Errorf("saved = %v, want qwen + %s/v1", kv, srv.URL) + } + if !strings.Contains(out.String(), "Found a running Ollama") { + t.Errorf("output missing detect line:\n%s", out.String()) + } +} + +func TestWizardInvalidChoice(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + if err := Run(strings.NewReader("9\n"), &bytes.Buffer{}); err == nil { + t.Error("invalid choice should error") + } +} + +func TestReadLine(t *testing.T) { + cases := map[string]string{ + "hello\n": "hello", + "trim \r\n": "trim", + "noeol": "noeol", + "\n": "", + } + for in, want := range cases { + if got := readLine(strings.NewReader(in)); got != want { + t.Errorf("readLine(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/justfile b/PROJECTS/intermediate/security-news-scraper/justfile index 85d7f3c8..fa4b6927 100644 --- a/PROJECTS/intermediate/security-news-scraper/justfile +++ b/PROJECTS/intermediate/security-news-scraper/justfile @@ -29,5 +29,15 @@ lint: vet tidy: go mod tidy +ollama-up: + docker compose -f internal/setup/ollama.compose.yml up -d + @echo "ollama starting on 127.0.0.1:39847 (override with OLLAMA_HOST_PORT) — the first run pulls qwen2.5:7b (a few minutes). watch progress with: just ollama-logs" + +ollama-down: + docker compose -f internal/setup/ollama.compose.yml down + +ollama-logs: + docker compose -f internal/setup/ollama.compose.yml logs -f + clean: rm -f {{binary}} *.db *.db-wal *.db-shm From d57bcc15b6fd4773af0272da564081b83aa46314 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Mon, 6 Jul 2026 21:26:52 -0400 Subject: [PATCH 09/15] feat(nadezhda): M6 AI ideation layer (ideate + TUI i-key) Add an opt-in ideation layer that turns ranked story clusters into content angles. A Provider interface backs four models: qwen (default, local Ollama), OpenAI, and Gemini share one OpenAI-compatible client; an Anthropic Messages client is written against raw net/http (no vendor SDK, uniform + dep-light). Results persist in ai_notes with a UNIQUE(cluster_id, provider) index (migration 0003), so re-ideation upserts in place. The ideate command skips any (cluster, provider) that already has a note unless --force is passed; benign classifier refusals get their own counter and never fail the run. In the TUI, i ideates the selected cluster inline and renders an AI IDEAS section in the dossier; it is a no-op unless AI is enabled and only acts in the detail view. --- .../cmd/nadezhda/ideate.go | 181 ++++++++++++++++++ .../cmd/nadezhda/ideate_test.go | 133 +++++++++++++ .../cmd/nadezhda/stubs.go | 1 - .../security-news-scraper/cmd/nadezhda/tui.go | 46 ++++- .../internal/ai/anthropic.go | 84 ++++++++ .../internal/ai/client_test.go | 125 ++++++++++++ .../internal/ai/factory_test.go | 83 ++++++++ .../security-news-scraper/internal/ai/http.go | 65 +++++++ .../security-news-scraper/internal/ai/mock.go | 25 +++ .../internal/ai/mock_test.go | 36 ++++ .../internal/ai/openai.go | 78 ++++++++ .../internal/ai/prompt.go | 128 +++++++++++++ .../internal/ai/prompt_test.go | 87 +++++++++ .../internal/ai/provider.go | 126 ++++++++++++ .../internal/store/ai_notes.go | 92 +++++++++ .../internal/store/ai_notes_test.go | 127 ++++++++++++ .../0003_ai_notes_provider_unique.sql | 4 + .../internal/tui/browser.go | 6 +- .../internal/tui/detail.go | 11 ++ .../internal/tui/keys.go | 5 + .../internal/tui/list.go | 4 +- .../internal/tui/model.go | 69 ++++++- .../internal/tui/model_test.go | 95 ++++++++- .../internal/tui/view.go | 5 +- 24 files changed, 1598 insertions(+), 18 deletions(-) create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/ideate.go create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/ideate_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/ai/anthropic.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/ai/client_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/ai/factory_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/ai/http.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/ai/mock.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/ai/mock_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/ai/openai.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/ai/prompt.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/ai/prompt_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/ai/provider.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/ai_notes.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/ai_notes_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0003_ai_notes_provider_unique.sql diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/ideate.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/ideate.go new file mode 100644 index 00000000..3bd6d329 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/ideate.go @@ -0,0 +1,181 @@ +// ©AngelaMos | 2026 +// ideate.go + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "time" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/ai" + "github.com/CarterPerez-dev/nadezhda/internal/rank" + "github.com/CarterPerez-dev/nadezhda/internal/setup" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const defaultIdeateTop = 10 + +var ( + ideateTop int + ideateSince string + ideateProvider string + ideateForce bool +) + +var ideateCmd = &cobra.Command{ + Use: "ideate", + Short: "Generate content angles from ranked clusters via an AI provider (opt-in)", + RunE: runIdeate, +} + +func init() { + ideateCmd.Flags().IntVar(&ideateTop, "top", defaultIdeateTop, "ideate the top N ranked clusters") + ideateCmd.Flags().StringVar(&ideateSince, "since", "", "only clusters active within this window (e.g. 24h, 168h)") + ideateCmd.Flags().StringVar(&ideateProvider, "provider", "", "override the configured provider: qwen|openai|anthropic|gemini") + ideateCmd.Flags().BoolVar(&ideateForce, "force", false, "re-ideate clusters that already have a note for this provider") + rootCmd.AddCommand(ideateCmd) +} + +func runIdeate(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + + aiCfg := cfg.AI + if ideateProvider != "" { + aiCfg.Provider = ideateProvider + aiCfg.Enabled = true + } + if !aiCfg.Enabled { + if !isInteractive(cmd) { + return fmt.Errorf("AI is not set up — run `nadezhda ai` to configure a provider") + } + fmt.Fprintln(cmd.OutOrStdout(), "AI is not set up yet — let's fix that.") + if err := setup.Run(cmd.InOrStdin(), cmd.OutOrStdout()); err != nil { + return err + } + cfg, err = loadConfig() + if err != nil { + return err + } + aiCfg = cfg.AI + if !aiCfg.Enabled { + return fmt.Errorf("AI still not configured after setup") + } + if aiCfg.Provider == ai.ProviderQwen && !setup.OllamaReachable(aiCfg.Qwen.BaseURL) { + fmt.Fprintln(cmd.OutOrStdout(), "Ollama isn't reachable yet — finish the steps above, then run: nadezhda ideate") + return nil + } + } + provider, err := ai.Factory(aiCfg) + if err != nil { + return err + } + + now := time.Now() + var since int64 + if ideateSince != "" { + d, err := time.ParseDuration(ideateSince) + if err != nil { + return fmt.Errorf("invalid --since %q: %w", ideateSince, err) + } + since = now.Add(-d).Unix() + } + + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + + clusters, err := st.DigestClusters(since) + if err != nil { + return err + } + scored := rank.Rank(clusters, cfg.Rank, cfg.Watchlist, now) + if ideateTop > 0 && ideateTop < len(scored) { + scored = scored[:ideateTop] + } + + out := cmd.OutOrStdout() + ctx := cmd.Context() + var generated, skipped, refused, failed int + + for _, s := range scored { + cid := s.Cluster.ClusterID + if !ideateForce { + exists, err := st.AINoteExists(cid, provider.Name()) + if err != nil { + return err + } + if exists { + skipped++ + fmt.Fprintf(out, "skip cluster %d (already ideated by %s; use --force)\n", cid, provider.Name()) + continue + } + } + + res, err := provider.Generate(ctx, ai.RequestFromCluster(s.Cluster)) + if err != nil { + if errors.Is(err, ai.ErrRefused) { + refused++ + fmt.Fprintf(out, "refused cluster %d (provider declined); skipping\n", cid) + } else { + failed++ + fmt.Fprintf(out, "warn cluster %d: %v\n", cid, err) + } + continue + } + + angles, err := json.Marshal(res.Angles) + if err != nil { + return err + } + note := store.AINote{ + ClusterID: cid, + Provider: provider.Name(), + Summary: res.Summary, + Why: res.Why, + AnglesJSON: string(angles), + Format: res.Format, + CreatedAt: time.Now().Unix(), + } + if err := st.InsertAINote(note); err != nil { + return err + } + generated++ + printIdeation(out, s, res) + } + + fmt.Fprintf(out, "\nideated %d, skipped %d, refused %d, failed %d (provider: %s)\n", generated, skipped, refused, failed, provider.Name()) + if generated == 0 && failed > 0 { + return fmt.Errorf("all %d ideation attempts failed", failed) + } + return nil +} + +func printIdeation(out io.Writer, s rank.Scored, res ai.IdeationResult) { + fmt.Fprintf(out, "\n=== cluster %d score %.2f [%s] ===\n", s.Cluster.ClusterID, s.Score, res.Format) + fmt.Fprintf(out, "%s\n\n", clusterHeadline(s.Cluster)) + fmt.Fprintf(out, "summary: %s\n\n", res.Summary) + fmt.Fprintf(out, "why: %s\n\n", res.Why) + fmt.Fprintln(out, "angles:") + for i, a := range res.Angles { + fmt.Fprintf(out, " %d. %s\n", i+1, a) + } +} + +func clusterHeadline(c store.DigestCluster) string { + for _, a := range c.Articles { + if a.Title != "" { + return a.Title + } + } + return "(untitled cluster)" +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/ideate_test.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/ideate_test.go new file mode 100644 index 00000000..e860e078 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/ideate_test.go @@ -0,0 +1,133 @@ +// ©AngelaMos | 2026 +// ideate_test.go + +package main + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/setup" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +func seedCluster(t *testing.T, st *store.Store) { + t.Helper() + stmts := []string{ + `INSERT INTO sources (id, name, url, type, weight, enabled) VALUES (1, 'krebs', 'https://krebsonsecurity.com', 'rss', 1.0, 1)`, + `INSERT INTO articles (id, source_id, canonical_url, content_hash, title, published_at) VALUES (1, 1, 'https://krebsonsecurity.com/a', 'h1', 'Critical RCE exploited in the wild', 1000)`, + `INSERT INTO clusters (id, cluster_key, first_seen, last_seen, size) VALUES (1, 'k1', 900, 1000, 1)`, + `INSERT INTO cluster_members (cluster_id, article_id) VALUES (1, 1)`, + } + for _, s := range stmts { + if _, err := st.DB().Exec(s); err != nil { + t.Fatalf("seed %q: %v", s, err) + } + } +} + +func TestIdeateCommandEndToEnd(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + t.Setenv(setup.EnvProvider, "") + t.Setenv(setup.EnvQwenURL, "") + dir := t.TempDir() + dbPath := filepath.Join(dir, "test.db") + st, err := store.Open(dbPath) + if err != nil { + t.Fatalf("open store: %v", err) + } + seedCluster(t, st) + st.Close() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := json.Marshal(map[string]any{ + "choices": []map[string]any{{"message": map[string]any{ + "content": `{"summary":"a critical rce","why":"widely exploited","angles":["hook one","hook two","hook three"],"format":"newsletter"}`, + }}}, + }) + _, _ = w.Write(body) + })) + defer srv.Close() + + cfgPath := filepath.Join(dir, "config.yaml") + cfgYAML := "ai:\n enabled: true\n provider: qwen\n qwen:\n base_url: " + srv.URL + "\n model: qwen2.5:7b\n" + if err := os.WriteFile(cfgPath, []byte(cfgYAML), 0o644); err != nil { + t.Fatal(err) + } + + prevCfg, prevDB, prevTop, prevProv, prevForce, prevSince := flagConfig, flagDB, ideateTop, ideateProvider, ideateForce, ideateSince + defer func() { + flagConfig, flagDB, ideateTop, ideateProvider, ideateForce, ideateSince = prevCfg, prevDB, prevTop, prevProv, prevForce, prevSince + }() + flagConfig, flagDB, ideateTop, ideateProvider, ideateForce, ideateSince = cfgPath, dbPath, 1, "", false, "" + + cmd := &cobra.Command{} + cmd.SetContext(context.Background()) + var buf bytes.Buffer + cmd.SetOut(&buf) + + if err := runIdeate(cmd, nil); err != nil { + t.Fatalf("runIdeate: %v\noutput:\n%s", err, buf.String()) + } + out := buf.String() + for _, want := range []string{"ideated 1", "a critical rce", "hook one", "newsletter"} { + if !strings.Contains(out, want) { + t.Errorf("ideate output missing %q\n---\n%s", want, out) + } + } + + st2, err := store.Open(dbPath) + if err != nil { + t.Fatal(err) + } + ok, err := st2.AINoteExists(1, "qwen") + st2.Close() + if err != nil || !ok { + t.Fatalf("note not persisted: ok=%v err=%v", ok, err) + } + + buf.Reset() + if err := runIdeate(cmd, nil); err != nil { + t.Fatalf("second runIdeate: %v", err) + } + if !strings.Contains(buf.String(), "skip cluster 1") { + t.Errorf("second run should skip existing note:\n%s", buf.String()) + } + + buf.Reset() + ideateForce = true + if err := runIdeate(cmd, nil); err != nil { + t.Fatalf("force runIdeate: %v", err) + } + if !strings.Contains(buf.String(), "ideated 1") { + t.Errorf("--force should regenerate, not skip:\n%s", buf.String()) + } +} + +func TestIdeateDisabledErrors(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + t.Setenv(setup.EnvProvider, "") + t.Setenv(setup.EnvQwenURL, "") + prevCfg, prevProv := flagConfig, ideateProvider + defer func() { flagConfig, ideateProvider = prevCfg, prevProv }() + flagConfig, ideateProvider = "", "" + + cmd := &cobra.Command{} + cmd.SetContext(context.Background()) + cmd.SetOut(&bytes.Buffer{}) + cmd.SetIn(strings.NewReader("")) + + err := runIdeate(cmd, nil) + if err == nil || !strings.Contains(err.Error(), "not set up") { + t.Fatalf("expected a 'not set up' error, got %v", err) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go index 5dbe4177..9a0cee9e 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go @@ -21,7 +21,6 @@ func init() { short string milestone string }{ - {"ideate", "Generate content angles from ranked clusters via an AI provider", "milestone M6"}, {"watch", "Run as a daemon, re-ingesting on an interval", "milestone M7"}, } for _, s := range stubs { diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/tui.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/tui.go index 093b87ac..5135e0bd 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/tui.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/tui.go @@ -4,11 +4,13 @@ package main import ( + "encoding/json" "fmt" "time" "github.com/spf13/cobra" + "github.com/CarterPerez-dev/nadezhda/internal/ai" "github.com/CarterPerez-dev/nadezhda/internal/rank" "github.com/CarterPerez-dev/nadezhda/internal/store" "github.com/CarterPerez-dev/nadezhda/internal/tui" @@ -68,8 +70,48 @@ func runTUI(cmd *cobra.Command, args []string) error { detail[v.ID] = full } } - return tui.Data{Scored: scored, CVEDetail: detail}, nil + notes := map[int64]ai.IdeationResult{} + if persisted, err := st.LatestAINotes(); err == nil { + for cid, n := range persisted { + var angles []string + _ = json.Unmarshal([]byte(n.AnglesJSON), &angles) + notes[cid] = ai.IdeationResult{Summary: n.Summary, Why: n.Why, Angles: angles, Format: n.Format} + } + } + return tui.Data{Scored: scored, CVEDetail: detail, Notes: notes}, nil } - return tui.Run(loader) + var ideator tui.Ideator + if cfg.AI.Enabled { + provider, err := ai.Factory(cfg.AI) + if err != nil { + return err + } + ctx := cmd.Context() + ideator = func(c store.DigestCluster) (ai.IdeationResult, error) { + res, err := provider.Generate(ctx, ai.RequestFromCluster(c)) + if err != nil { + return ai.IdeationResult{}, err + } + angles, err := json.Marshal(res.Angles) + if err != nil { + return ai.IdeationResult{}, err + } + note := store.AINote{ + ClusterID: c.ClusterID, + Provider: provider.Name(), + Summary: res.Summary, + Why: res.Why, + AnglesJSON: string(angles), + Format: res.Format, + CreatedAt: time.Now().Unix(), + } + if err := st.InsertAINote(note); err != nil { + return ai.IdeationResult{}, fmt.Errorf("save note: %w", err) + } + return res, nil + } + } + + return tui.Run(loader, ideator) } diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ai/anthropic.go b/PROJECTS/intermediate/security-news-scraper/internal/ai/anthropic.go new file mode 100644 index 00000000..411e1250 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/ai/anthropic.go @@ -0,0 +1,84 @@ +// ©AngelaMos | 2026 +// anthropic.go + +package ai + +import ( + "context" + "net/http" + "strings" +) + +const ( + anthropicMessagesPath = "/messages" + anthropicVersion = "2023-06-01" + headerAPIKey = "x-api-key" + headerAnthropicVer = "anthropic-version" + stopReasonRefusal = "refusal" + contentTypeText = "text" +) + +type anthropicClient struct { + http *http.Client + baseURL string + model string + apiKey string +} + +func newAnthropic(client *http.Client, baseURL, model, apiKey string) *anthropicClient { + return &anthropicClient{ + http: client, + baseURL: strings.TrimRight(baseURL, "/"), + model: model, + apiKey: apiKey, + } +} + +func (c *anthropicClient) Name() string { return ProviderAnthropic } + +type anthropicMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type anthropicRequest struct { + Model string `json:"model"` + MaxTokens int `json:"max_tokens"` + System string `json:"system"` + Messages []anthropicMessage `json:"messages"` +} + +type anthropicResponse struct { + StopReason string `json:"stop_reason"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` +} + +func (c *anthropicClient) Generate(ctx context.Context, req IdeationRequest) (IdeationResult, error) { + system, user := buildPrompt(req) + body := anthropicRequest{ + Model: c.model, + MaxTokens: defaultMaxTokens, + System: system, + Messages: []anthropicMessage{{Role: roleUser, Content: user}}, + } + header := http.Header{} + header.Set(headerAPIKey, c.apiKey) + header.Set(headerAnthropicVer, anthropicVersion) + var resp anthropicResponse + if err := postJSON(ctx, c.http, c.baseURL+anthropicMessagesPath, header, body, &resp); err != nil { + return IdeationResult{}, err + } + if resp.StopReason == stopReasonRefusal { + return IdeationResult{}, ErrRefused + } + var text strings.Builder + for _, block := range resp.Content { + if block.Type == contentTypeText { + text.WriteString(block.Text) + } + } + return parseResult(text.String()) +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ai/client_test.go b/PROJECTS/intermediate/security-news-scraper/internal/ai/client_test.go new file mode 100644 index 00000000..0769a7e3 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/ai/client_test.go @@ -0,0 +1,125 @@ +// ©AngelaMos | 2026 +// client_test.go + +package ai + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" +) + +const ideationJSON = `{"summary":"s","why":"w","angles":["a","b","c"],"format":"video"}` + +func TestOpenAICompatGenerate(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer k" { + t.Errorf("Authorization = %q, want Bearer k", got) + } + body, _ := json.Marshal(map[string]any{ + "choices": []map[string]any{{"message": map[string]any{"role": "assistant", "content": ideationJSON}}}, + }) + _, _ = w.Write(body) + })) + defer srv.Close() + + c := newOpenAICompat("openai", srv.Client(), srv.URL, "gpt-4o-mini", "k") + res, err := c.Generate(context.Background(), IdeationRequest{Titles: []string{"t"}}) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if res.Summary != "s" || res.Format != FormatVideo || len(res.Angles) != 3 { + t.Errorf("got %+v", res) + } + if c.Name() != "openai" { + t.Errorf("Name = %q", c.Name()) + } +} + +func TestOpenAICompatNoKeyNoAuth(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "" { + t.Errorf("Authorization = %q, want empty (qwen has no key)", got) + } + body, _ := json.Marshal(map[string]any{ + "choices": []map[string]any{{"message": map[string]any{"content": ideationJSON}}}, + }) + _, _ = w.Write(body) + })) + defer srv.Close() + + c := newOpenAICompat("qwen", srv.Client(), srv.URL, "qwen2.5:7b", "") + if _, err := c.Generate(context.Background(), IdeationRequest{Titles: []string{"t"}}); err != nil { + t.Fatalf("Generate: %v", err) + } +} + +func TestOpenAICompatNoChoices(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"choices":[]}`)) + })) + defer srv.Close() + + c := newOpenAICompat("openai", srv.Client(), srv.URL, "m", "k") + if _, err := c.Generate(context.Background(), IdeationRequest{}); err == nil { + t.Error("expected error on empty choices") + } +} + +func TestOpenAICompatStatusError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"error":"bad key"}`)) + })) + defer srv.Close() + + c := newOpenAICompat("openai", srv.Client(), srv.URL, "m", "k") + if _, err := c.Generate(context.Background(), IdeationRequest{}); err == nil { + t.Error("expected error on 401") + } +} + +func TestAnthropicGenerate(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("x-api-key"); got != "k" { + t.Errorf("x-api-key = %q, want k", got) + } + if got := r.Header.Get("anthropic-version"); got != "2023-06-01" { + t.Errorf("anthropic-version = %q, want 2023-06-01", got) + } + body, _ := json.Marshal(map[string]any{ + "stop_reason": "end_turn", + "content": []map[string]any{{"type": "text", "text": ideationJSON}}, + }) + _, _ = w.Write(body) + })) + defer srv.Close() + + c := newAnthropic(srv.Client(), srv.URL, "claude-sonnet-4-6", "k") + res, err := c.Generate(context.Background(), IdeationRequest{Titles: []string{"t"}}) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if res.Summary != "s" || res.Format != FormatVideo { + t.Errorf("got %+v", res) + } + if c.Name() != ProviderAnthropic { + t.Errorf("Name = %q", c.Name()) + } +} + +func TestAnthropicRefusal(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"stop_reason":"refusal","content":[]}`)) + })) + defer srv.Close() + + c := newAnthropic(srv.Client(), srv.URL, "claude-sonnet-4-6", "k") + _, err := c.Generate(context.Background(), IdeationRequest{}) + if !errors.Is(err, ErrRefused) { + t.Errorf("err = %v, want ErrRefused", err) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ai/factory_test.go b/PROJECTS/intermediate/security-news-scraper/internal/ai/factory_test.go new file mode 100644 index 00000000..c2ad740a --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/ai/factory_test.go @@ -0,0 +1,83 @@ +// ©AngelaMos | 2026 +// factory_test.go + +package ai + +import ( + "testing" + + "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +func TestFactoryProviders(t *testing.T) { + base := config.Default().AI + + t.Setenv(envOpenAIKey, "") + t.Setenv(envGeminiKey, "") + t.Setenv(envAnthropicKey, "") + + base.Provider = ProviderQwen + p, err := Factory(base) + if err != nil || p.Name() != ProviderQwen { + t.Fatalf("qwen: p=%v err=%v", p, err) + } + + base.Provider = ProviderOpenAI + if _, err := Factory(base); err == nil { + t.Error("openai without key should error") + } + t.Setenv(envOpenAIKey, "k") + if p, err := Factory(base); err != nil || p.Name() != ProviderOpenAI { + t.Errorf("openai: p=%v err=%v", p, err) + } + + base.Provider = ProviderGemini + if _, err := Factory(base); err == nil { + t.Error("gemini without key should error") + } + t.Setenv(envGeminiKey, "k") + if p, err := Factory(base); err != nil || p.Name() != ProviderGemini { + t.Errorf("gemini: p=%v err=%v", p, err) + } + + base.Provider = ProviderAnthropic + if _, err := Factory(base); err == nil { + t.Error("anthropic without key should error") + } + t.Setenv(envAnthropicKey, "k") + if p, err := Factory(base); err != nil || p.Name() != ProviderAnthropic { + t.Errorf("anthropic: p=%v err=%v", p, err) + } + + base.Provider = "nope" + if _, err := Factory(base); err == nil { + t.Error("unknown provider should error") + } +} + +func TestRequestFromCluster(t *testing.T) { + cvss := 9.8 + epss := 0.5 + c := store.DigestCluster{ + Size: 3, + FirstSeen: 0, + LastSeen: 6 * secondsPerHour, + Articles: []store.DigestArticle{ + {Title: "A", SourceName: "krebs"}, + {Title: "A", SourceName: "krebs"}, + {Title: "B", SourceName: "register"}, + }, + CVEs: []store.DigestCVE{{ID: "CVE-2025-1", CVSSScore: &cvss, EPSS: &epss, IsKEV: true}}, + } + req := RequestFromCluster(c) + if len(req.Titles) != 2 || len(req.Sources) != 2 { + t.Errorf("dedup failed: titles=%v sources=%v", req.Titles, req.Sources) + } + if req.SpanHours != 6 || req.ClusterSize != 3 { + t.Errorf("span=%d size=%d", req.SpanHours, req.ClusterSize) + } + if len(req.CVEs) != 1 || !req.CVEs[0].KEV || req.CVEs[0].CVSS == nil { + t.Errorf("cve mapping: %+v", req.CVEs) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ai/http.go b/PROJECTS/intermediate/security-news-scraper/internal/ai/http.go new file mode 100644 index 00000000..9b83d02d --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/ai/http.go @@ -0,0 +1,65 @@ +// ©AngelaMos | 2026 +// http.go + +package ai + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" +) + +const ( + headerContentType = "Content-Type" + mimeJSON = "application/json" + errBodyLimit = 300 +) + +type statusError struct { + code int + url string + body string +} + +func (e *statusError) Error() string { + body := e.body + if len(body) > errBodyLimit { + body = strings.ToValidUTF8(body[:errBodyLimit], "") + "..." + } + return fmt.Sprintf("ai: POST %s: status %d: %s", e.url, e.code, body) +} + +func postJSON(ctx context.Context, client *http.Client, url string, header http.Header, reqBody, out any) error { + raw, err := json.Marshal(reqBody) + if err != nil { + return fmt.Errorf("ai: marshal request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(raw)) + if err != nil { + return fmt.Errorf("ai: build request %s: %w", url, err) + } + req.Header.Set(headerContentType, mimeJSON) + for key, values := range header { + req.Header[key] = values + } + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("ai: POST %s: %w", url, err) + } + defer resp.Body.Close() + data, err := io.ReadAll(io.LimitReader(resp.Body, maxJSONBytes)) + if err != nil { + return fmt.Errorf("ai: read %s: %w", url, err) + } + if resp.StatusCode != http.StatusOK { + return &statusError{code: resp.StatusCode, url: url, body: string(data)} + } + if err := json.Unmarshal(data, out); err != nil { + return fmt.Errorf("ai: decode %s: %w", url, err) + } + return nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ai/mock.go b/PROJECTS/intermediate/security-news-scraper/internal/ai/mock.go new file mode 100644 index 00000000..65f2ca8c --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/ai/mock.go @@ -0,0 +1,25 @@ +// ©AngelaMos | 2026 +// mock.go + +package ai + +import "context" + +type MockProvider struct { + ProviderName string + Result IdeationResult + Err error + Calls int +} + +func (m *MockProvider) Name() string { + if m.ProviderName == "" { + return "mock" + } + return m.ProviderName +} + +func (m *MockProvider) Generate(ctx context.Context, req IdeationRequest) (IdeationResult, error) { + m.Calls++ + return m.Result, m.Err +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ai/mock_test.go b/PROJECTS/intermediate/security-news-scraper/internal/ai/mock_test.go new file mode 100644 index 00000000..4a54329f --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/ai/mock_test.go @@ -0,0 +1,36 @@ +// ©AngelaMos | 2026 +// mock_test.go + +package ai + +import ( + "context" + "testing" +) + +func TestMockProvider(t *testing.T) { + want := IdeationResult{Summary: "s", Angles: []string{"a"}, Format: FormatBlog} + m := &MockProvider{ProviderName: "mock", Result: want} + if m.Name() != "mock" { + t.Errorf("Name = %q, want mock", m.Name()) + } + got, err := m.Generate(context.Background(), IdeationRequest{}) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if got.Summary != "s" || len(got.Angles) != 1 { + t.Errorf("result = %+v", got) + } + if m.Calls != 1 { + t.Errorf("Calls = %d, want 1", m.Calls) + } + + sentinel := context.Canceled + me := &MockProvider{Err: sentinel} + if me.Name() != "mock" { + t.Errorf("default Name = %q, want mock", me.Name()) + } + if _, err := me.Generate(context.Background(), IdeationRequest{}); err != sentinel { + t.Errorf("err = %v, want the injected sentinel", err) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ai/openai.go b/PROJECTS/intermediate/security-news-scraper/internal/ai/openai.go new file mode 100644 index 00000000..30fe0f8a --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/ai/openai.go @@ -0,0 +1,78 @@ +// ©AngelaMos | 2026 +// openai.go + +package ai + +import ( + "context" + "fmt" + "net/http" + "strings" +) + +const ( + openAIChatPath = "/chat/completions" + roleSystem = "system" + roleUser = "user" + headerAuth = "Authorization" + bearerPrefix = "Bearer " +) + +type openAICompat struct { + name string + http *http.Client + baseURL string + model string + apiKey string +} + +func newOpenAICompat(name string, client *http.Client, baseURL, model, apiKey string) *openAICompat { + return &openAICompat{ + name: name, + http: client, + baseURL: strings.TrimRight(baseURL, "/"), + model: model, + apiKey: apiKey, + } +} + +func (c *openAICompat) Name() string { return c.name } + +type openAIMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type openAIRequest struct { + Model string `json:"model"` + Messages []openAIMessage `json:"messages"` +} + +type openAIResponse struct { + Choices []struct { + Message openAIMessage `json:"message"` + } `json:"choices"` +} + +func (c *openAICompat) Generate(ctx context.Context, req IdeationRequest) (IdeationResult, error) { + system, user := buildPrompt(req) + body := openAIRequest{ + Model: c.model, + Messages: []openAIMessage{ + {Role: roleSystem, Content: system}, + {Role: roleUser, Content: user}, + }, + } + header := http.Header{} + if c.apiKey != "" { + header.Set(headerAuth, bearerPrefix+c.apiKey) + } + var resp openAIResponse + if err := postJSON(ctx, c.http, c.baseURL+openAIChatPath, header, body, &resp); err != nil { + return IdeationResult{}, err + } + if len(resp.Choices) == 0 { + return IdeationResult{}, fmt.Errorf("ai: %s returned no choices", c.name) + } + return parseResult(resp.Choices[0].Message.Content) +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ai/prompt.go b/PROJECTS/intermediate/security-news-scraper/internal/ai/prompt.go new file mode 100644 index 00000000..768d9f0c --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/ai/prompt.go @@ -0,0 +1,128 @@ +// ©AngelaMos | 2026 +// prompt.go + +package ai + +import ( + "encoding/json" + "fmt" + "strings" +) + +const ( + systemPrompt = `You are a senior security-content strategist helping a creator turn cybersecurity news into content. You are given a cluster of related news articles (one story covered by one or more outlets), optionally with referenced CVEs and their exploit signals. Produce content ideation for the story. + +Respond with a SINGLE JSON object and NOTHING else. No prose, no markdown, no code fences. The object has exactly these keys: + "summary": a 2-3 sentence plain-language summary of the story. + "why": one paragraph on why it matters to a security audience (impact, who is affected, exploitation). + "angles": an array of 3 to 5 distinct content angles or hooks a creator could lead with. + "format": the single best-fit format, one of: "blog", "newsletter", "video". + +Ground every claim in the provided material. Do not invent CVEs, vendors, or facts not present in the input.` + + jsonObjectOpen byte = '{' + jsonObjectClose byte = '}' +) + +func buildPrompt(req IdeationRequest) (string, string) { + var b strings.Builder + fmt.Fprintf(&b, "Story cluster: %d article(s) across %d outlet(s), spanning ~%dh.\n\n", req.ClusterSize, len(req.Sources), req.SpanHours) + if len(req.Sources) > 0 { + fmt.Fprintf(&b, "Outlets: %s\n\n", strings.Join(req.Sources, ", ")) + } + b.WriteString("Headlines:\n") + for _, t := range req.Titles { + fmt.Fprintf(&b, "- %s\n", t) + } + if len(req.CVEs) > 0 { + b.WriteString("\nReferenced CVEs:\n") + for _, c := range req.CVEs { + b.WriteString("- " + c.ID) + if c.CVSS != nil { + fmt.Fprintf(&b, " CVSS %.1f", *c.CVSS) + } + if c.KEV { + b.WriteString(" [KEV: known exploited]") + } + if c.EPSS != nil { + fmt.Fprintf(&b, " EPSS %.2f", *c.EPSS) + } + b.WriteString("\n") + } + } + return systemPrompt, b.String() +} + +func parseResult(text string) (IdeationResult, error) { + for _, obj := range jsonObjectCandidates(text) { + var res IdeationResult + if err := json.Unmarshal([]byte(obj), &res); err != nil { + continue + } + res.Format = normalizeFormat(res.Format) + if strings.TrimSpace(res.Summary) != "" && len(res.Angles) > 0 { + return res, nil + } + } + return IdeationResult{}, fmt.Errorf("ai: no usable JSON ideation object in model output") +} + +func jsonObjectCandidates(text string) []string { + var out []string + for i := 0; i < len(text); { + if text[i] != jsonObjectOpen { + i++ + continue + } + end := balancedObjectEnd(text, i) + if end < 0 { + break + } + out = append(out, text[i:end+1]) + i = end + 1 + } + return out +} + +func balancedObjectEnd(text string, start int) int { + depth := 0 + inStr := false + esc := false + for i := start; i < len(text); i++ { + ch := text[i] + if inStr { + switch { + case esc: + esc = false + case ch == '\\': + esc = true + case ch == '"': + inStr = false + } + continue + } + switch ch { + case '"': + inStr = true + case jsonObjectOpen: + depth++ + case jsonObjectClose: + depth-- + if depth == 0 { + return i + } + } + } + return -1 +} + +func normalizeFormat(f string) string { + switch strings.ToLower(strings.TrimSpace(f)) { + case FormatNewsletter: + return FormatNewsletter + case FormatVideo: + return FormatVideo + default: + return FormatBlog + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ai/prompt_test.go b/PROJECTS/intermediate/security-news-scraper/internal/ai/prompt_test.go new file mode 100644 index 00000000..2468a542 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/ai/prompt_test.go @@ -0,0 +1,87 @@ +// ©AngelaMos | 2026 +// prompt_test.go + +package ai + +import ( + "strings" + "testing" +) + +func TestBuildPromptIncludesContext(t *testing.T) { + cvss := 9.8 + epss := 0.97 + _, user := buildPrompt(IdeationRequest{ + Titles: []string{"Massive breach at ACME"}, + Sources: []string{"Krebs", "The Register"}, + CVEs: []CVEContext{{ID: "CVE-2025-5777", CVSS: &cvss, KEV: true, EPSS: &epss}}, + ClusterSize: 2, + SpanHours: 6, + }) + for _, want := range []string{"Massive breach at ACME", "Krebs", "The Register", "CVE-2025-5777", "CVSS 9.8", "KEV", "EPSS 0.97"} { + if !strings.Contains(user, want) { + t.Errorf("user prompt missing %q\n---\n%s", want, user) + } + } +} + +func TestParseResultValid(t *testing.T) { + res, err := parseResult(`{"summary":"s","why":"w","angles":["a","b","c"],"format":"video"}`) + if err != nil { + t.Fatalf("parseResult: %v", err) + } + if res.Summary != "s" || res.Why != "w" || len(res.Angles) != 3 || res.Format != FormatVideo { + t.Errorf("got %+v", res) + } +} + +func TestParseResultTolerantWrapper(t *testing.T) { + raw := "Here is the JSON:\n```json\n{\"summary\":\"s\",\"why\":\"w\",\"angles\":[\"a\"],\"format\":\"blog\"}\n```\nHope that helps." + res, err := parseResult(raw) + if err != nil { + t.Fatalf("parseResult: %v", err) + } + if res.Summary != "s" || len(res.Angles) != 1 { + t.Errorf("got %+v", res) + } +} + +func TestParseResultIgnoresPreJSONBraces(t *testing.T) { + raw := `Let me plan {step: outline} then answer: {"summary":"real","why":"w","angles":["a","b"],"format":"blog"} done.` + res, err := parseResult(raw) + if err != nil { + t.Fatalf("parseResult: %v", err) + } + if res.Summary != "real" || len(res.Angles) != 2 { + t.Errorf("got %+v", res) + } +} + +func TestParseResultErrors(t *testing.T) { + cases := []string{ + "no json here", + `{"why":"w","angles":["a"],"format":"blog"}`, + `{"summary":"s","why":"w","angles":[],"format":"blog"}`, + `{"summary":"s",`, + } + for _, c := range cases { + if _, err := parseResult(c); err == nil { + t.Errorf("parseResult(%q) = nil error, want error", c) + } + } +} + +func TestNormalizeFormat(t *testing.T) { + cases := map[string]string{ + "newsletter": FormatNewsletter, + "VIDEO": FormatVideo, + "blog": FormatBlog, + "nonsense": FormatBlog, + "": FormatBlog, + } + for in, want := range cases { + if got := normalizeFormat(in); got != want { + t.Errorf("normalizeFormat(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ai/provider.go b/PROJECTS/intermediate/security-news-scraper/internal/ai/provider.go new file mode 100644 index 00000000..5b82484c --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/ai/provider.go @@ -0,0 +1,126 @@ +// ©AngelaMos | 2026 +// provider.go + +package ai + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const ( + ProviderQwen = "qwen" + ProviderOpenAI = "openai" + ProviderGemini = "gemini" + ProviderAnthropic = "anthropic" + + FormatBlog = "blog" + FormatNewsletter = "newsletter" + FormatVideo = "video" + + envOpenAIKey = "OPENAI_API_KEY" + envGeminiKey = "GEMINI_API_KEY" + envAnthropicKey = "ANTHROPIC_API_KEY" + + requestTimeout = 120 * time.Second + defaultMaxTokens = 2048 + maxJSONBytes = 4 << 20 + secondsPerHour = 3600 +) + +var ErrRefused = errors.New("ai: provider declined to generate for this item") + +type CVEContext struct { + ID string + CVSS *float64 + KEV bool + EPSS *float64 +} + +type IdeationRequest struct { + Titles []string + Sources []string + CVEs []CVEContext + ClusterSize int + SpanHours int +} + +type IdeationResult struct { + Summary string `json:"summary"` + Why string `json:"why"` + Angles []string `json:"angles"` + Format string `json:"format"` +} + +type Provider interface { + Name() string + Generate(ctx context.Context, req IdeationRequest) (IdeationResult, error) +} + +func noRedirect(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse +} + +func Factory(cfg config.AI) (Provider, error) { + client := &http.Client{Timeout: requestTimeout, CheckRedirect: noRedirect} + switch cfg.Provider { + case ProviderQwen: + return newOpenAICompat(ProviderQwen, client, cfg.Qwen.BaseURL, cfg.Qwen.Model, ""), nil + case ProviderOpenAI: + key := os.Getenv(envOpenAIKey) + if key == "" { + return nil, fmt.Errorf("ai: provider %q requires %s in the environment", ProviderOpenAI, envOpenAIKey) + } + return newOpenAICompat(ProviderOpenAI, client, cfg.OpenAI.BaseURL, cfg.OpenAI.Model, key), nil + case ProviderGemini: + key := os.Getenv(envGeminiKey) + if key == "" { + return nil, fmt.Errorf("ai: provider %q requires %s in the environment", ProviderGemini, envGeminiKey) + } + return newOpenAICompat(ProviderGemini, client, cfg.Gemini.BaseURL, cfg.Gemini.Model, key), nil + case ProviderAnthropic: + key := os.Getenv(envAnthropicKey) + if key == "" { + return nil, fmt.Errorf("ai: provider %q requires %s in the environment", ProviderAnthropic, envAnthropicKey) + } + return newAnthropic(client, cfg.Anthropic.BaseURL, cfg.Anthropic.Model, key), nil + default: + return nil, fmt.Errorf("ai: unknown provider %q", cfg.Provider) + } +} + +func RequestFromCluster(c store.DigestCluster) IdeationRequest { + req := IdeationRequest{ClusterSize: c.Size} + if c.LastSeen > c.FirstSeen { + req.SpanHours = int((c.LastSeen - c.FirstSeen) / secondsPerHour) + } + seenTitle := make(map[string]bool) + seenSource := make(map[string]bool) + for _, a := range c.Articles { + if a.Title != "" && !seenTitle[a.Title] { + seenTitle[a.Title] = true + req.Titles = append(req.Titles, a.Title) + } + if a.SourceName != "" && !seenSource[a.SourceName] { + seenSource[a.SourceName] = true + req.Sources = append(req.Sources, a.SourceName) + } + } + for _, v := range c.CVEs { + req.CVEs = append(req.CVEs, CVEContext{ID: v.ID, CVSS: v.CVSSScore, KEV: v.IsKEV, EPSS: v.EPSS}) + } + return req +} + +var ( + _ Provider = (*openAICompat)(nil) + _ Provider = (*anthropicClient)(nil) + _ Provider = (*MockProvider)(nil) +) diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/ai_notes.go b/PROJECTS/intermediate/security-news-scraper/internal/store/ai_notes.go new file mode 100644 index 00000000..229eedcd --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/ai_notes.go @@ -0,0 +1,92 @@ +// ©AngelaMos | 2026 +// ai_notes.go + +package store + +import ( + "database/sql" + "errors" + "fmt" +) + +type AINote struct { + ID int64 + ClusterID int64 + Provider string + Summary string + Why string + AnglesJSON string + Format string + CreatedAt int64 +} + +func (s *Store) InsertAINote(n AINote) error { + _, err := s.db.Exec(` + INSERT INTO ai_notes (cluster_id, provider, summary, why, angles_json, format, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(cluster_id, provider) DO UPDATE SET + summary = excluded.summary, + why = excluded.why, + angles_json = excluded.angles_json, + format = excluded.format, + created_at = excluded.created_at`, + n.ClusterID, n.Provider, n.Summary, n.Why, n.AnglesJSON, n.Format, n.CreatedAt, + ) + if err != nil { + return fmt.Errorf("insert ai_note cluster=%d provider=%q: %w", n.ClusterID, n.Provider, err) + } + return nil +} + +func (s *Store) AINoteExists(clusterID int64, provider string) (bool, error) { + var one int + err := s.db.QueryRow( + `SELECT 1 FROM ai_notes WHERE cluster_id = ? AND provider = ? LIMIT 1`, + clusterID, provider, + ).Scan(&one) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("ai_note exists cluster=%d provider=%q: %w", clusterID, provider, err) + } + return true, nil +} + +func (s *Store) AINotesForCluster(clusterID int64) ([]AINote, error) { + rows, err := s.db.Query(` + SELECT id, cluster_id, provider, summary, why, angles_json, format, created_at + FROM ai_notes WHERE cluster_id = ? ORDER BY provider`, clusterID) + if err != nil { + return nil, fmt.Errorf("ai_notes for cluster %d: %w", clusterID, err) + } + defer rows.Close() + var out []AINote + for rows.Next() { + var n AINote + if err := rows.Scan(&n.ID, &n.ClusterID, &n.Provider, &n.Summary, &n.Why, &n.AnglesJSON, &n.Format, &n.CreatedAt); err != nil { + return nil, fmt.Errorf("ai_notes for cluster %d: scan: %w", clusterID, err) + } + out = append(out, n) + } + return out, rows.Err() +} + +func (s *Store) LatestAINotes() (map[int64]AINote, error) { + rows, err := s.db.Query(` + SELECT id, cluster_id, provider, summary, why, angles_json, format, created_at + FROM ai_notes ORDER BY cluster_id, created_at`) + if err != nil { + return nil, fmt.Errorf("latest ai_notes: %w", err) + } + defer rows.Close() + out := make(map[int64]AINote) + for rows.Next() { + var n AINote + if err := rows.Scan(&n.ID, &n.ClusterID, &n.Provider, &n.Summary, &n.Why, &n.AnglesJSON, &n.Format, &n.CreatedAt); err != nil { + return nil, fmt.Errorf("latest ai_notes: scan: %w", err) + } + out[n.ClusterID] = n + } + return out, rows.Err() +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/ai_notes_test.go b/PROJECTS/intermediate/security-news-scraper/internal/store/ai_notes_test.go new file mode 100644 index 00000000..50a213d0 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/ai_notes_test.go @@ -0,0 +1,127 @@ +// ©AngelaMos | 2026 +// ai_notes_test.go + +package store + +import "testing" + +func insertTestCluster(t *testing.T, s *Store, id int64, key string) { + t.Helper() + _, err := s.DB().Exec( + `INSERT INTO clusters (id, cluster_key, first_seen, last_seen, size) VALUES (?, ?, 0, 0, 1)`, + id, key, + ) + if err != nil { + t.Fatalf("insert cluster: %v", err) + } +} + +func TestAINoteRoundTrip(t *testing.T) { + s := openTemp(t) + insertTestCluster(t, s, 1, "k1") + + note := AINote{ + ClusterID: 1, Provider: "qwen", + Summary: "s", Why: "w", AnglesJSON: `["a","b"]`, Format: "blog", CreatedAt: 100, + } + if err := s.InsertAINote(note); err != nil { + t.Fatalf("InsertAINote: %v", err) + } + + ok, err := s.AINoteExists(1, "qwen") + if err != nil || !ok { + t.Fatalf("AINoteExists(1,qwen) = %v, %v; want true", ok, err) + } + if ok, _ := s.AINoteExists(1, "openai"); ok { + t.Error("AINoteExists(1,openai) = true, want false") + } + if ok, _ := s.AINoteExists(2, "qwen"); ok { + t.Error("AINoteExists(2,qwen) = true, want false") + } + + notes, err := s.AINotesForCluster(1) + if err != nil || len(notes) != 1 { + t.Fatalf("AINotesForCluster = %v, %v; want 1 note", notes, err) + } + if notes[0].Summary != "s" || notes[0].Format != "blog" || notes[0].AnglesJSON != `["a","b"]` { + t.Errorf("note = %+v", notes[0]) + } +} + +func TestAINoteUpsertOverwrites(t *testing.T) { + s := openTemp(t) + insertTestCluster(t, s, 1, "k1") + + if err := s.InsertAINote(AINote{ClusterID: 1, Provider: "qwen", Summary: "first", AnglesJSON: "[]", Format: "blog", CreatedAt: 1}); err != nil { + t.Fatal(err) + } + if err := s.InsertAINote(AINote{ClusterID: 1, Provider: "qwen", Summary: "second", AnglesJSON: "[]", Format: "video", CreatedAt: 2}); err != nil { + t.Fatal(err) + } + notes, err := s.AINotesForCluster(1) + if err != nil { + t.Fatal(err) + } + if len(notes) != 1 { + t.Fatalf("got %d notes, want 1 (upsert should overwrite)", len(notes)) + } + if notes[0].Summary != "second" || notes[0].Format != "video" || notes[0].CreatedAt != 2 { + t.Errorf("upsert did not overwrite: %+v", notes[0]) + } +} + +func TestLatestAINotesNewestPerCluster(t *testing.T) { + s := openTemp(t) + insertTestCluster(t, s, 1, "k1") + insertTestCluster(t, s, 2, "k2") + + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(s.InsertAINote(AINote{ClusterID: 1, Provider: "qwen", Summary: "old", AnglesJSON: "[]", Format: "blog", CreatedAt: 10})) + must(s.InsertAINote(AINote{ClusterID: 1, Provider: "anthropic", Summary: "new", AnglesJSON: "[]", Format: "video", CreatedAt: 20})) + must(s.InsertAINote(AINote{ClusterID: 2, Provider: "qwen", Summary: "two", AnglesJSON: "[]", Format: "blog", CreatedAt: 5})) + + notes, err := s.LatestAINotes() + if err != nil { + t.Fatal(err) + } + if len(notes) != 2 { + t.Fatalf("got %d clusters, want 2", len(notes)) + } + if notes[1].Summary != "new" { + t.Errorf("cluster 1 latest = %q, want new (highest created_at)", notes[1].Summary) + } + if notes[2].Summary != "two" { + t.Errorf("cluster 2 = %q, want two", notes[2].Summary) + } +} + +func TestAINoteForeignKeyRejectsOrphan(t *testing.T) { + s := openTemp(t) + err := s.InsertAINote(AINote{ClusterID: 99, Provider: "qwen", AnglesJSON: "[]", Format: "blog", CreatedAt: 1}) + if err == nil { + t.Error("insert for a nonexistent cluster should be rejected by the foreign key") + } +} + +func TestAINoteCascadesOnClusterDelete(t *testing.T) { + s := openTemp(t) + insertTestCluster(t, s, 1, "k1") + if err := s.InsertAINote(AINote{ClusterID: 1, Provider: "qwen", AnglesJSON: "[]", Format: "blog", CreatedAt: 1}); err != nil { + t.Fatal(err) + } + if _, err := s.DB().Exec(`DELETE FROM clusters WHERE id = 1`); err != nil { + t.Fatal(err) + } + notes, err := s.AINotesForCluster(1) + if err != nil { + t.Fatal(err) + } + if len(notes) != 0 { + t.Errorf("notes should cascade on cluster delete, %d remain", len(notes)) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0003_ai_notes_provider_unique.sql b/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0003_ai_notes_provider_unique.sql new file mode 100644 index 00000000..900803ba --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0003_ai_notes_provider_unique.sql @@ -0,0 +1,4 @@ +-- ©AngelaMos | 2026 +-- 0003_ai_notes_provider_unique.sql + +CREATE UNIQUE INDEX idx_ai_notes_cluster_provider ON ai_notes(cluster_id, provider); diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/browser.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/browser.go index 5de543cf..b54ef90f 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/tui/browser.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/browser.go @@ -8,6 +8,8 @@ import ( "net/url" "os/exec" "runtime" + + "github.com/CarterPerez-dev/nadezhda/internal/setup" ) func openURL(target string) error { @@ -16,7 +18,9 @@ func openURL(target string) error { return fmt.Errorf("refusing to open non-http url: %q", target) } name, args := openerCommand(target) - return exec.Command(name, args...).Start() + cmd := exec.Command(name, args...) + cmd.Env = setup.NonSecretEnviron() + return cmd.Start() } func openerCommand(target string) (string, []string) { diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/detail.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/detail.go index 854bd490..cb659fcd 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/tui/detail.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/detail.go @@ -48,6 +48,17 @@ func (m Model) renderDetailBody() string { b.WriteString(m.renderCVE(v, w, indent)) } } + + if note, ok := m.notes[c.ClusterID]; ok { + b.WriteString(m.sectionHeader("AI IDEAS", w)) + b.WriteString("\n") + b.WriteString(t.fg(colorMagenta).Bold(true).Render(strings.ToUpper(note.Format)) + "\n\n") + b.WriteString(m.wrapIndent(note.Summary, w, t.Text) + "\n\n") + b.WriteString(m.wrapIndent(note.Why, w, t.Muted) + "\n\n") + for i, a := range note.Angles { + b.WriteString(m.wrapIndent(fmt.Sprintf("%d. %s", i+1, a), w, t.Text) + "\n") + } + } return strings.TrimRight(b.String(), "\n") } diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/keys.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/keys.go index 11995599..a6f55fc1 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/tui/keys.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/keys.go @@ -12,6 +12,7 @@ type keyMap struct { Bottom key.Binding Open key.Binding Browser key.Binding + Ideate key.Binding Back key.Binding Quit key.Binding } @@ -42,6 +43,10 @@ func defaultKeyMap() keyMap { key.WithKeys("o"), key.WithHelp("o", "open"), ), + Ideate: key.NewBinding( + key.WithKeys("i"), + key.WithHelp("i", "ideate"), + ), Back: key.NewBinding( key.WithKeys("esc", "backspace"), key.WithHelp("esc", "back"), diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/list.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/list.go index 56474138..ba9dc160 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/tui/list.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/list.go @@ -91,9 +91,7 @@ func (m Model) listBody(capacity int) string { func (m Model) emptyBody(capacity int) string { t := m.theme msg := t.Muted.Render("no stories in the store yet — run ") + - t.KeyGlyph.Render("nadezhda scrape") + - t.Muted.Render(" then ") + - t.KeyGlyph.Render("nadezhda enrich") + t.KeyGlyph.Render("nadezhda scrape") return lipgloss.Place(m.width, capacity, lipgloss.Center, lipgloss.Center, msg) } diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/model.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/model.go index 0f1884ec..e74cdac6 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/tui/model.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/model.go @@ -11,6 +11,7 @@ import ( "github.com/charmbracelet/bubbles/viewport" tea "github.com/charmbracelet/bubbletea" + "github.com/CarterPerez-dev/nadezhda/internal/ai" "github.com/CarterPerez-dev/nadezhda/internal/rank" "github.com/CarterPerez-dev/nadezhda/internal/store" ) @@ -39,6 +40,7 @@ var raveSpinner = spinner.Spinner{ type Data struct { Scored []rank.Scored CVEDetail map[string]store.CVE + Notes map[int64]ai.IdeationResult } type Loader func() (Data, error) @@ -52,6 +54,15 @@ type openedMsg struct { err error } +type Ideator func(store.DigestCluster) (ai.IdeationResult, error) + +type ideatedMsg struct { + clusterID int64 + result ai.IdeationResult +} + +type ideateErrMsg struct{ err error } + type Model struct { state viewState loader Loader @@ -73,9 +84,13 @@ type Model struct { opener func(string) error status string statusErr bool + + ideator Ideator + generating bool + notes map[int64]ai.IdeationResult } -func New(loader Loader, now time.Time) Model { +func New(loader Loader, ideator Ideator, now time.Time) Model { th := NewTheme() sp := spinner.New(spinner.WithSpinner(raveSpinner), spinner.WithStyle(th.Spinner)) m := Model{ @@ -90,6 +105,8 @@ func New(loader Loader, now time.Time) Model { height: defaultHeight, cveDetail: map[string]store.CVE{}, opener: openURL, + ideator: ideator, + notes: map[int64]ai.IdeationResult{}, } return m } @@ -117,6 +134,9 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case dataMsg: m.scored = msg.data.Scored m.cveDetail = msg.data.CVEDetail + if msg.data.Notes != nil { + m.notes = msg.data.Notes + } m.state = stateList return m, nil case errMsg: @@ -130,8 +150,21 @@ func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.status, m.statusErr = "opened in browser", false } return m, nil + case ideatedMsg: + m.generating = false + m.notes[msg.clusterID] = msg.result + m.status, m.statusErr = "ideas ready", false + if m.state == stateDetail { + m.viewport.SetContent(m.renderDetailBody()) + m.viewport.GotoBottom() + } + return m, nil + case ideateErrMsg: + m.generating = false + m.status, m.statusErr = "ideate failed: "+msg.err.Error(), true + return m, nil case spinner.TickMsg: - if m.state != stateLoading { + if m.state != stateLoading && !m.generating { return m, nil } var cmd tea.Cmd @@ -156,6 +189,9 @@ func (m Model) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { if key.Matches(msg, m.keys.Browser) && (m.state == stateList || m.state == stateDetail) { return m, m.openSelected() } + if key.Matches(msg, m.keys.Ideate) && m.state == stateDetail { + return m.startIdeate() + } switch m.state { case stateList: return m.handleListKey(msg) @@ -177,6 +213,31 @@ func (m Model) openSelected() tea.Cmd { } } +func (m Model) startIdeate() (tea.Model, tea.Cmd) { + if m.ideator == nil { + m.status, m.statusErr = "AI not set up — run: nadezhda ai", true + return m, nil + } + if m.generating || len(m.scored) == 0 { + return m, nil + } + m.generating = true + m.status, m.statusErr = "ideating "+headlineOf(m.selected().Cluster), false + return m, tea.Batch(m.ideateSelected(), m.spinner.Tick) +} + +func (m Model) ideateSelected() tea.Cmd { + ideator := m.ideator + cluster := m.selected().Cluster + return func() tea.Msg { + res, err := ideator(cluster) + if err != nil { + return ideateErrMsg{err} + } + return ideatedMsg{clusterID: cluster.ClusterID, result: res} + } +} + func (m Model) handleListKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) { switch { case key.Matches(msg, m.keys.Up): @@ -244,8 +305,8 @@ func (m Model) selected() rank.Scored { return m.scored[m.cursor] } -func Run(loader Loader) error { - m := New(loader, time.Now()) +func Run(loader Loader, ideator Ideator) error { + m := New(loader, ideator, time.Now()) _, err := tea.NewProgram(m, tea.WithAltScreen()).Run() return err } diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/model_test.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/model_test.go index e8c8fd27..204e7a51 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/tui/model_test.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/model_test.go @@ -13,6 +13,7 @@ import ( "github.com/charmbracelet/bubbles/spinner" tea "github.com/charmbracelet/bubbletea" + "github.com/CarterPerez-dev/nadezhda/internal/ai" "github.com/CarterPerez-dev/nadezhda/internal/rank" "github.com/CarterPerez-dev/nadezhda/internal/store" ) @@ -81,11 +82,11 @@ func step(t *testing.T, m Model, msg tea.Msg) Model { func loadedModel(t *testing.T) Model { t.Helper() - return step(t, New(nil, testNow()), dataMsg{sampleData()}) + return step(t, New(nil, nil, testNow()), dataMsg{sampleData()}) } func TestInitialStateIsLoading(t *testing.T) { - if m := New(nil, testNow()); m.state != stateLoading { + if m := New(nil, nil, testNow()); m.state != stateLoading { t.Fatalf("initial state = %v, want stateLoading", m.state) } } @@ -101,7 +102,7 @@ func TestDataMsgTransitionsToList(t *testing.T) { } func TestErrMsgTransitionsToError(t *testing.T) { - m := step(t, New(nil, testNow()), errMsg{errors.New("wire down")}) + m := step(t, New(nil, nil, testNow()), errMsg{errors.New("wire down")}) if m.state != stateError { t.Fatalf("state = %v, want stateError", m.state) } @@ -188,7 +189,7 @@ func TestSpinnerTickIgnoredOutsideLoading(t *testing.T) { } func TestViewsRenderNonEmpty(t *testing.T) { - loading := New(nil, testNow()) + loading := New(nil, nil, testNow()) if strings.TrimSpace(loading.View()) == "" { t.Error("loading view is empty") } @@ -211,14 +212,14 @@ func TestViewsRenderNonEmpty(t *testing.T) { t.Error("detail view missing severity label") } - errv := step(t, New(nil, testNow()), errMsg{errors.New("boom")}) + errv := step(t, New(nil, nil, testNow()), errMsg{errors.New("boom")}) if strings.TrimSpace(errv.View()) == "" { t.Error("error view is empty") } } func TestEmptyStoreRendersHint(t *testing.T) { - m := step(t, New(nil, testNow()), dataMsg{Data{}}) + m := step(t, New(nil, nil, testNow()), dataMsg{Data{}}) if m.state != stateList { t.Fatalf("state = %v, want stateList", m.state) } @@ -320,3 +321,85 @@ func TestOpenURLRejectsNonHTTP(t *testing.T) { } } } + +func TestIdeateDisabledShowsHint(t *testing.T) { + m := loadedModel(t) + m = step(t, m, tea.KeyMsg{Type: tea.KeyEnter}) + m = step(t, m, runeKey('i')) + if !m.statusErr || !strings.Contains(m.status, "nadezhda ai") { + t.Errorf("i with nil ideator: status=%q err=%v, want a setup hint", m.status, m.statusErr) + } + if m.generating { + t.Error("generating should stay false when ideator is nil") + } +} + +func TestIdeateIgnoredInListView(t *testing.T) { + m := loadedModel(t) + m.ideator = func(c store.DigestCluster) (ai.IdeationResult, error) { + return ai.IdeationResult{Summary: "s", Angles: []string{"a"}, Format: "blog"}, nil + } + tm, cmd := m.Update(runeKey('i')) + m = toModel(t, tm) + if m.generating || cmd != nil { + t.Errorf("i in the list must be a no-op: generating=%v cmd=%v", m.generating, cmd) + } +} + +func TestIdeateFlowStoresAndRenders(t *testing.T) { + m := loadedModel(t) + m.ideator = func(c store.DigestCluster) (ai.IdeationResult, error) { + return ai.IdeationResult{Summary: "s", Why: "w", Angles: []string{"angle-one", "angle-two"}, Format: "video"}, nil + } + m = step(t, m, tea.KeyMsg{Type: tea.KeyEnter}) + + tm, cmd := m.Update(runeKey('i')) + m = toModel(t, tm) + if !m.generating || cmd == nil { + t.Fatalf("after i: generating=%v cmd=%v", m.generating, cmd) + } + + msg := m.ideateSelected()() + im, ok := msg.(ideatedMsg) + if !ok { + t.Fatalf("ideateSelected produced %T, want ideatedMsg", msg) + } + if im.clusterID != 1 || im.result.Summary != "s" { + t.Fatalf("ideatedMsg = %+v", im) + } + + m = step(t, m, im) + if m.generating { + t.Error("generating still true after ideatedMsg") + } + if m.notes[1].Summary != "s" || len(m.notes[1].Angles) != 2 { + t.Errorf("note not stored: %+v", m.notes[1]) + } + + m = step(t, m, tea.KeyMsg{Type: tea.KeyEnter}) + dv := m.View() + if !strings.Contains(dv, "AI IDEAS") || !strings.Contains(dv, "angle-one") { + t.Error("detail view missing the ideation section") + } +} + +func TestIdeateErrorSetsStatus(t *testing.T) { + m := loadedModel(t) + m.ideator = func(c store.DigestCluster) (ai.IdeationResult, error) { + return ai.IdeationResult{}, errors.New("boom") + } + m = step(t, m, tea.KeyMsg{Type: tea.KeyEnter}) + m = step(t, m, runeKey('i')) + msg := m.ideateSelected()() + em, ok := msg.(ideateErrMsg) + if !ok { + t.Fatalf("ideateSelected produced %T, want ideateErrMsg", msg) + } + m = step(t, m, em) + if m.generating { + t.Error("generating still true after ideateErrMsg") + } + if !m.statusErr { + t.Error("statusErr not set after ideate failure") + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/tui/view.go b/PROJECTS/intermediate/security-news-scraper/internal/tui/view.go index 15a3278f..c418faf5 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/tui/view.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/tui/view.go @@ -135,7 +135,7 @@ func (m Model) detailView() string { t.Muted.Render(fmt.Sprintf("story %d of %d", m.cursor+1, len(m.scored))) head := m.spread(left, t.Meta.Render(scroll)) foot := m.spread( - m.keyHints(m.keys.Back, m.keys.Browser, m.keys.Down, m.keys.Up, m.keys.Quit), + m.keyHints(m.keys.Back, m.keys.Browser, m.keys.Ideate, m.keys.Down, m.keys.Up, m.keys.Quit), m.statusText(), ) return lipgloss.JoinVertical(lipgloss.Left, @@ -148,6 +148,9 @@ func (m Model) detailView() string { } func (m Model) statusText() string { + if m.generating { + return m.spinner.View() + " " + m.theme.fg(colorCyan).Render(m.status) + } if m.status == "" { return "" } From f65d197c96decb68c7b68305579c7dd651e16753 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Mon, 6 Jul 2026 22:06:44 -0400 Subject: [PATCH 10/15] feat(nadezhda): M7 watch daemon with optional webhook notify Add a watch command (--interval, --once, --no-enrich): a long-lived daemon that re-ingests every enabled source on a ticker and, when a webhook is configured, POSTs genuinely new high-signal stories to it (Slack, Discord, or any JSON endpoint via a text/content/items payload). internal/watch is a pure, stdlib-only scheduler: the pipeline is injected as a Cycle closure, so the daemon is fully unit-testable with a fake ticker and notifier. Graceful shutdown on SIGINT/SIGTERM returns cleanly; a cycle error is logged and the loop continues (fail-soft). Notify uses a dedicated fetch-time watermark (store.NewlyFetchedClusters) so a freshly ingested but older-dated advisory is surfaced, which a publish-time filter would drop. Notable stories are capped by watch.notify_max_items and filtered by score threshold or KEV status. Extract the shared ingest and cluster sequence (pipeline.go) so scrape and watch cannot drift, and route scrape and enrich through cmd.Context() so they honor SIGTERM too. --- .../security-news-scraper/.gitignore | 3 + .../cmd/nadezhda/enrich.go | 7 +- .../cmd/nadezhda/main.go | 3 +- .../cmd/nadezhda/pipeline.go | 30 ++ .../cmd/nadezhda/scrape.go | 18 +- .../cmd/nadezhda/stubs.go | 33 --- .../cmd/nadezhda/watch.go | 258 ++++++++++++++++++ .../cmd/nadezhda/watch_test.go | 149 ++++++++++ .../internal/config/config.go | 35 +++ .../internal/store/store.go | 6 +- .../internal/store/watch.go | 76 ++++++ .../internal/store/watch_test.go | 99 +++++++ .../internal/watch/notify.go | 114 ++++++++ .../internal/watch/notify_test.go | 89 ++++++ .../internal/watch/report.go | 28 ++ .../internal/watch/watch.go | 127 +++++++++ .../internal/watch/watch_test.go | 198 ++++++++++++++ 17 files changed, 1217 insertions(+), 56 deletions(-) create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/pipeline.go delete mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/watch.go create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/watch_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/watch.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/watch_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/watch/notify.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/watch/notify_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/watch/report.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/watch/watch.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/watch/watch_test.go diff --git a/PROJECTS/intermediate/security-news-scraper/.gitignore b/PROJECTS/intermediate/security-news-scraper/.gitignore index 38335067..d938e12c 100644 --- a/PROJECTS/intermediate/security-news-scraper/.gitignore +++ b/PROJECTS/intermediate/security-news-scraper/.gitignore @@ -18,6 +18,9 @@ docs/ *.sqlite /data/ +# large local KEV catalog snapshot (manual download; tests use kev-sample.json) +/testdata/kev/kev-full.json + # env / secrets / local overrides .env *.local.yaml diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/enrich.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/enrich.go index 7b041561..27568097 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/enrich.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/enrich.go @@ -4,11 +4,9 @@ package main import ( - "context" "fmt" "net/http" "os" - "os/signal" "time" "github.com/spf13/cobra" @@ -43,10 +41,7 @@ func runEnrich(cmd *cobra.Command, args []string) error { } defer st.Close() - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) - defer stop() - - stats, err := enrich.Run(ctx, st, buildEnrichClients(cfg), time.Now(), cfg.Enrich.CacheTTLHours, cfg.Enrich.NegativeTTLHours) + stats, err := enrich.Run(cmd.Context(), st, buildEnrichClients(cfg), time.Now(), cfg.Enrich.CacheTTLHours, cfg.Enrich.NegativeTTLHours) if err != nil { return err } diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go index 3c1d5bcc..63e5bdcc 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go @@ -7,10 +7,11 @@ import ( "context" "os" "os/signal" + "syscall" ) func main() { - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() if err := rootCmd.ExecuteContext(ctx); err != nil { os.Exit(1) diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/pipeline.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/pipeline.go new file mode 100644 index 00000000..bd8bdea3 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/pipeline.go @@ -0,0 +1,30 @@ +// ©AngelaMos | 2026 +// pipeline.go + +package main + +import ( + "context" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/cluster" + "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/fetch" + "github.com/CarterPerez-dev/nadezhda/internal/ingest" + "github.com/CarterPerez-dev/nadezhda/internal/source" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +func ingestAndCluster(ctx context.Context, fc *fetch.Client, st *store.Store, cfg config.Config, targets []source.Source, start time.Time) (ingest.Summary, cluster.Stats, error) { + summary, err := ingest.Run(ctx, fc, st, cfg, targets, start) + if err != nil { + return ingest.Summary{}, cluster.Stats{}, err + } + sinceUnix := start.Unix() - int64(cfg.Cluster.LookbackHours)*secondsPerHour + windowSeconds := int64(cfg.Cluster.WindowHours) * secondsPerHour + stats, err := cluster.Rebuild(st, cfg.Cluster.TitleJaccard, windowSeconds, sinceUnix) + if err != nil { + return summary, cluster.Stats{}, err + } + return summary, stats, nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go index fbb1c0f8..e89da4e0 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go @@ -6,13 +6,10 @@ package main import ( "context" "fmt" - "os" - "os/signal" "time" "github.com/spf13/cobra" - "github.com/CarterPerez-dev/nadezhda/internal/cluster" "github.com/CarterPerez-dev/nadezhda/internal/config" "github.com/CarterPerez-dev/nadezhda/internal/enrich" "github.com/CarterPerez-dev/nadezhda/internal/fetch" @@ -78,24 +75,15 @@ func runScrape(cmd *cobra.Command, args []string) error { MaxRetries: cfg.Fetch.MaxRetries, }) - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) - defer stop() - + ctx := cmd.Context() now := time.Now() - summary, err := ingest.Run(ctx, fc, st, cfg, targets, now) + summary, cstats, err := ingestAndCluster(ctx, fc, st, cfg, targets, now) if err != nil { return err } printSummary(cmd, summary) - - sinceUnix := now.Unix() - int64(cfg.Cluster.LookbackHours)*secondsPerHour - windowSeconds := int64(cfg.Cluster.WindowHours) * secondsPerHour - stats, err := cluster.Rebuild(st, cfg.Cluster.TitleJaccard, windowSeconds, sinceUnix) - if err != nil { - return err - } fmt.Fprintf(cmd.OutOrStdout(), "%d clusters (%d multi-source, largest %d)\n", - stats.Total, stats.MultiSource, stats.LargestSize) + cstats.Total, cstats.MultiSource, cstats.LargestSize) if !scrapeNoEnrich { enrichAfterScrape(ctx, cmd, cfg, st) diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go deleted file mode 100644 index 9a0cee9e..00000000 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go +++ /dev/null @@ -1,33 +0,0 @@ -// ©AngelaMos | 2026 -// stubs.go - -package main - -import ( - "fmt" - - "github.com/spf13/cobra" -) - -func notImplemented(milestone string) func(*cobra.Command, []string) error { - return func(cmd *cobra.Command, args []string) error { - return fmt.Errorf("%s is not implemented yet (%s)", cmd.Name(), milestone) - } -} - -func init() { - stubs := []struct { - use string - short string - milestone string - }{ - {"watch", "Run as a daemon, re-ingesting on an interval", "milestone M7"}, - } - for _, s := range stubs { - rootCmd.AddCommand(&cobra.Command{ - Use: s.use, - Short: s.short, - RunE: notImplemented(s.milestone), - }) - } -} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/watch.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/watch.go new file mode 100644 index 00000000..52f63d60 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/watch.go @@ -0,0 +1,258 @@ +// ©AngelaMos | 2026 +// watch.go + +package main + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/enrich" + "github.com/CarterPerez-dev/nadezhda/internal/fetch" + "github.com/CarterPerez-dev/nadezhda/internal/rank" + "github.com/CarterPerez-dev/nadezhda/internal/source" + "github.com/CarterPerez-dev/nadezhda/internal/store" + "github.com/CarterPerez-dev/nadezhda/internal/watch" +) + +const untitledCluster = "(untitled cluster)" + +var ( + watchInterval string + watchOnce bool + watchNoEnrich bool +) + +var watchCmd = &cobra.Command{ + Use: "watch", + Short: "Run as a daemon, re-ingesting on an interval with an optional webhook notify", + Long: "Run nadezhda as a long-lived daemon that re-ingests every enabled source on an interval " + + "(default from config, override with --interval). Each cycle scrapes, clusters, and enriches " + + "exactly like the scrape command. When watch.webhook_url is set, genuinely new high-signal " + + "stories are POSTed to that webhook (Slack, Discord, or any JSON endpoint).", + RunE: runWatch, +} + +func init() { + watchCmd.Flags().StringVar(&watchInterval, "interval", "", "re-ingest interval, e.g. 30m or 1h (overrides watch.interval in config)") + watchCmd.Flags().BoolVar(&watchOnce, "once", false, "run a single cycle and exit instead of looping") + watchCmd.Flags().BoolVar(&watchNoEnrich, "no-enrich", false, "skip the keyless CVE enrichment pass each cycle") + rootCmd.AddCommand(watchCmd) +} + +func runWatch(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + interval, err := resolveWatchInterval(cfg) + if err != nil { + return err + } + if cfg.Watch.WebhookURL != "" { + if err := validateWebhookURL(cfg.Watch.WebhookURL); err != nil { + return err + } + } + + srcs, err := source.Load(cfg.SourcesPath) + if err != nil { + return err + } + targets := source.Enabled(srcs) + if len(targets) == 0 { + return fmt.Errorf("watch: no enabled sources to poll") + } + + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + + fc := fetch.New(fetch.Options{ + UserAgent: cfg.Fetch.UserAgent, + PerHostRate: cfg.Fetch.PerHostRate, + PerHostBurst: cfg.Fetch.PerHostBurst, + Timeout: time.Duration(cfg.Fetch.TimeoutSeconds) * time.Second, + MaxRetries: cfg.Fetch.MaxRetries, + }) + + out := cmd.ErrOrStderr() + if watchNoEnrich && cfg.Watch.NotifyOnKEV { + fmt.Fprintln(out, "watch: --no-enrich with notify_on_kev set: KEV status is never computed, so KEV alerts will not fire") + } + + cycle := func(ctx context.Context) (watch.Report, error) { + start := time.Now() + summary, cstats, err := ingestAndCluster(ctx, fc, st, cfg, targets, start) + if err != nil { + return watch.Report{}, err + } + newArticles, duplicates, failed := summary.Totals() + + var enriched, kevHits int + if !watchNoEnrich { + enriched, kevHits = watchEnrich(ctx, out, st, cfg) + } + + notable, err := buildNotable(st, cfg, start) + if err != nil { + return watch.Report{}, err + } + return watch.Report{ + Start: start, + Duration: time.Since(start), + NewArticles: newArticles, + Duplicates: duplicates, + Clusters: cstats.Total, + Enriched: enriched, + KEVHits: kevHits, + Failed: failed, + Notable: notable, + }, nil + } + + opts := watch.Options{ + Interval: interval, + RunAtStart: true, + Cycle: cycle, + Out: out, + } + if cfg.Watch.WebhookURL != "" { + opts.Notifier = watch.WebhookNotifier{ + URL: cfg.Watch.WebhookURL, + Client: &http.Client{Timeout: time.Duration(cfg.Fetch.TimeoutSeconds) * time.Second}, + } + } + + if watchOnce { + return watch.Once(cmd.Context(), opts) + } + return watch.Run(cmd.Context(), opts) +} + +func resolveWatchInterval(cfg config.Config) (time.Duration, error) { + raw := cfg.Watch.Interval + if watchInterval != "" { + raw = watchInterval + } + d, err := time.ParseDuration(raw) + if err != nil { + return 0, fmt.Errorf("watch: invalid interval %q: %w", raw, err) + } + if d < config.MinWatchInterval { + return 0, fmt.Errorf("watch: interval %s is below the minimum %s", d, config.MinWatchInterval) + } + return d, nil +} + +func validateWebhookURL(raw string) error { + u, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("watch: invalid webhook_url %q: %w", raw, err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("watch: webhook_url must be http or https, got %q", raw) + } + if u.Host == "" { + return fmt.Errorf("watch: webhook_url must include a host, got %q", raw) + } + return nil +} + +func watchEnrich(ctx context.Context, out io.Writer, st *store.Store, cfg config.Config) (enriched, kevHits int) { + ectx, cancel := context.WithTimeout(ctx, enrichBudget) + defer cancel() + stats, err := enrich.Run(ectx, st, buildEnrichClients(cfg), time.Now(), cfg.Enrich.CacheTTLHours, cfg.Enrich.NegativeTTLHours) + if err != nil && ctx.Err() == nil { + fmt.Fprintf(out, "watch: enrich degraded, KEV/CVE data may be stale: %v\n", err) + } + return stats.Enriched, stats.KEVHits +} + +func buildNotable(st *store.Store, cfg config.Config, cycleStart time.Time) ([]watch.NotableItem, error) { + fresh, err := st.NewlyFetchedClusters(cycleStart.Unix()) + if err != nil { + return nil, err + } + scored := rank.Rank(fresh, cfg.Rank, cfg.Watchlist, time.Now()) + items := make([]watch.NotableItem, 0, cfg.Watch.NotifyMaxItems) + for _, sc := range scored { + if len(items) >= cfg.Watch.NotifyMaxItems { + break + } + if isNotable(sc, cfg.Watch) { + items = append(items, toNotable(sc)) + } + } + return items, nil +} + +func isNotable(sc rank.Scored, w config.Watch) bool { + if sc.Score >= w.NotifyMinScore { + return true + } + if w.NotifyOnKEV { + for _, v := range sc.Cluster.CVEs { + if v.IsKEV { + return true + } + } + } + return false +} + +func toNotable(sc rank.Scored) watch.NotableItem { + c := sc.Cluster + title, link := representativeArticle(c.Articles) + var maxCVSS float64 + var isKEV bool + cves := make([]string, 0, len(c.CVEs)) + for _, v := range c.CVEs { + cves = append(cves, v.ID) + if v.CVSSScore != nil && *v.CVSSScore > maxCVSS { + maxCVSS = *v.CVSSScore + } + if v.IsKEV { + isKEV = true + } + } + return watch.NotableItem{ + Title: title, + URL: link, + Score: sc.Score, + MaxCVSS: maxCVSS, + IsKEV: isKEV, + CVEs: cves, + Sources: distinctSources(c.Articles), + } +} + +func representativeArticle(articles []store.DigestArticle) (title, link string) { + if len(articles) == 0 { + return untitledCluster, "" + } + best := articles[0] + for _, a := range articles[1:] { + if a.SourceWeight > best.SourceWeight { + best = a + } + } + return best.Title, best.CanonicalURL +} + +func distinctSources(articles []store.DigestArticle) int { + set := make(map[string]struct{}, len(articles)) + for _, a := range articles { + set[a.SourceName] = struct{}{} + } + return len(set) +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/watch_test.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/watch_test.go new file mode 100644 index 00000000..cfd88fa5 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/watch_test.go @@ -0,0 +1,149 @@ +// ©AngelaMos | 2026 +// watch_test.go + +package main + +import ( + "fmt" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/rank" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +func TestIsNotable(t *testing.T) { + kevCluster := store.DigestCluster{CVEs: []store.DigestCVE{{ID: "CVE-2026-1", IsKEV: true}}} + plainCluster := store.DigestCluster{} + tests := []struct { + name string + scored rank.Scored + watch config.Watch + want bool + }{ + {"score above threshold", rank.Scored{Cluster: plainCluster, Score: 0.9}, config.Watch{NotifyMinScore: 0.5}, true}, + {"score below threshold, not kev", rank.Scored{Cluster: plainCluster, Score: 0.2}, config.Watch{NotifyMinScore: 0.5}, false}, + {"below threshold but kev with notify_on_kev", rank.Scored{Cluster: kevCluster, Score: 0.1}, config.Watch{NotifyMinScore: 0.5, NotifyOnKEV: true}, true}, + {"kev but notify_on_kev off", rank.Scored{Cluster: kevCluster, Score: 0.1}, config.Watch{NotifyMinScore: 0.5, NotifyOnKEV: false}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isNotable(tt.scored, tt.watch); got != tt.want { + t.Errorf("isNotable = %v, want %v", got, tt.want) + } + }) + } +} + +func TestToNotablePicksHighestAuthoritySource(t *testing.T) { + cvss := 9.8 + sc := rank.Scored{ + Score: 0.77, + Cluster: store.DigestCluster{ + Articles: []store.DigestArticle{ + {Title: "low authority", CanonicalURL: "https://low", SourceName: "low", SourceWeight: 0.5}, + {Title: "high authority", CanonicalURL: "https://high", SourceName: "high", SourceWeight: 1.0}, + }, + CVEs: []store.DigestCVE{{ID: "CVE-2026-1", CVSSScore: &cvss, IsKEV: true}}, + }, + } + n := toNotable(sc) + if n.Title != "high authority" || n.URL != "https://high" { + t.Errorf("representative should be the highest-source-weight article, got %q / %q", n.Title, n.URL) + } + if !n.IsKEV || n.MaxCVSS != 9.8 { + t.Errorf("cve signals wrong: kev=%v cvss=%v", n.IsKEV, n.MaxCVSS) + } + if n.Sources != 2 { + t.Errorf("distinct sources = %d, want 2", n.Sources) + } + if len(n.CVEs) != 1 || n.CVEs[0] != "CVE-2026-1" { + t.Errorf("cves wrong: %v", n.CVEs) + } + if n.Score != 0.77 { + t.Errorf("score = %v, want 0.77", n.Score) + } +} + +func TestResolveWatchInterval(t *testing.T) { + cfg := config.Default() + watchInterval = "" + defer func() { watchInterval = "" }() + + if d, err := resolveWatchInterval(cfg); err != nil || d != time.Hour { + t.Errorf("default interval = %v, %v; want 1h, nil", d, err) + } + + watchInterval = "30m" + if d, err := resolveWatchInterval(cfg); err != nil || d != 30*time.Minute { + t.Errorf("flag override = %v, %v; want 30m, nil", d, err) + } + + watchInterval = "5s" + if _, err := resolveWatchInterval(cfg); err == nil { + t.Error("expected an error for a sub-minimum interval") + } + + watchInterval = "nonsense" + if _, err := resolveWatchInterval(cfg); err == nil { + t.Error("expected an error for a garbage interval") + } +} + +func TestValidateWebhookURL(t *testing.T) { + good := []string{"https://hooks.slack.com/services/x", "http://127.0.0.1:39871/hook"} + for _, u := range good { + if err := validateWebhookURL(u); err != nil { + t.Errorf("validateWebhookURL(%q) = %v, want nil", u, err) + } + } + bad := []string{"example.com/hook", "ftp://x/y", "not a url", ""} + for _, u := range bad { + if err := validateWebhookURL(u); err == nil { + t.Errorf("validateWebhookURL(%q) = nil, want error", u) + } + } +} + +func TestBuildNotableCapsToNotifyMax(t *testing.T) { + st, err := store.Open(filepath.Join(t.TempDir(), "watch.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + + src, err := st.UpsertSource(store.SourceInput{Name: "s", URL: "https://s/f", Type: "rss", Weight: 1, Enabled: true}) + if err != nil { + t.Fatal(err) + } + var rows []store.ClusterRow + for i := 0; i < 5; i++ { + id, err := st.InsertArticle(store.Article{ + SourceID: src, CanonicalURL: fmt.Sprintf("https://s/%d", i), + ContentHash: "c" + strconv.Itoa(i), TitleHash: "t" + strconv.Itoa(i), + Title: "story " + strconv.Itoa(i), PublishedAt: 10000, FetchedAt: 10000, + }) + if err != nil { + t.Fatal(err) + } + rows = append(rows, store.ClusterRow{Key: strconv.Itoa(i), Members: []int64{id}, FirstSeen: 10000, LastSeen: 10000}) + } + if err := st.ReplaceClusters(rows); err != nil { + t.Fatal(err) + } + + cfg := config.Default() + cfg.Watch.NotifyMinScore = 0 + cfg.Watch.NotifyMaxItems = 2 + + items, err := buildNotable(st, cfg, time.Unix(9999, 0)) + if err != nil { + t.Fatal(err) + } + if len(items) != 2 { + t.Fatalf("buildNotable returned %d items, want the notify_max_items cap of 2", len(items)) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go index b26b0f96..b773d46a 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go @@ -6,6 +6,7 @@ package config import ( "fmt" "os" + "time" "gopkg.in/yaml.v3" ) @@ -50,6 +51,12 @@ const ( defaultGeminiModel = "gemini-2.5-flash" defaultAnthropicURL = "https://api.anthropic.com/v1" defaultClaudeModel = "claude-sonnet-4-6" + + defaultWatchInterval = "1h" + defaultWatchNotifyMinScore = 0.5 + defaultWatchNotifyMax = 10 + + MinWatchInterval = time.Minute ) var defaultTrackingParams = []string{ @@ -109,6 +116,14 @@ type AI struct { Anthropic Provider `yaml:"anthropic"` } +type Watch struct { + Interval string `yaml:"interval"` + WebhookURL string `yaml:"webhook_url"` + NotifyMinScore float64 `yaml:"notify_min_score"` + NotifyMaxItems int `yaml:"notify_max_items"` + NotifyOnKEV bool `yaml:"notify_on_kev"` +} + type Config struct { DBPath string `yaml:"db_path"` SourcesPath string `yaml:"sources_path"` @@ -118,6 +133,7 @@ type Config struct { Cluster Cluster `yaml:"cluster"` Rank Rank `yaml:"rank"` AI AI `yaml:"ai"` + Watch Watch `yaml:"watch"` } func Default() Config { @@ -163,6 +179,12 @@ func Default() Config { Gemini: Provider{BaseURL: defaultGeminiURL, Model: defaultGeminiModel}, Anthropic: Provider{BaseURL: defaultAnthropicURL, Model: defaultClaudeModel}, }, + Watch: Watch{ + Interval: defaultWatchInterval, + NotifyMinScore: defaultWatchNotifyMinScore, + NotifyMaxItems: defaultWatchNotifyMax, + NotifyOnKEV: true, + }, } } @@ -232,5 +254,18 @@ func (c Config) validate() error { default: return fmt.Errorf("config: ai.provider must be one of qwen|openai|gemini|anthropic, got %q", c.AI.Provider) } + d, err := time.ParseDuration(c.Watch.Interval) + if err != nil { + return fmt.Errorf("config: watch.interval %q is not a valid duration: %w", c.Watch.Interval, err) + } + if d < MinWatchInterval { + return fmt.Errorf("config: watch.interval must be >= %s, got %s", MinWatchInterval, c.Watch.Interval) + } + if c.Watch.NotifyMinScore < 0 || c.Watch.NotifyMinScore > 1 { + return fmt.Errorf("config: watch.notify_min_score must be in [0,1], got %v", c.Watch.NotifyMinScore) + } + if c.Watch.NotifyMaxItems < 1 { + return fmt.Errorf("config: watch.notify_max_items must be >= 1, got %d", c.Watch.NotifyMaxItems) + } return nil } diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go index e28bb698..f689ceff 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go @@ -347,12 +347,16 @@ func (s *Store) digestClusterRows(since int64) (map[int64]*DigestCluster, []int6 return nil, nil, fmt.Errorf("digest clusters: %w", err) } defer rows.Close() + return scanClusterRows(rows) +} + +func scanClusterRows(rows *sql.Rows) (map[int64]*DigestCluster, []int64, error) { byID := make(map[int64]*DigestCluster) var order []int64 for rows.Next() { var dc DigestCluster if err := rows.Scan(&dc.ClusterID, &dc.Key, &dc.Size, &dc.FirstSeen, &dc.LastSeen); err != nil { - return nil, nil, fmt.Errorf("digest clusters: scan: %w", err) + return nil, nil, fmt.Errorf("scan cluster rows: %w", err) } clone := dc byID[dc.ClusterID] = &clone diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/watch.go b/PROJECTS/intermediate/security-news-scraper/internal/store/watch.go new file mode 100644 index 00000000..d5b89796 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/watch.go @@ -0,0 +1,76 @@ +// ©AngelaMos | 2026 +// watch.go + +package store + +import ( + "fmt" + "strings" +) + +func (s *Store) NewlyFetchedClusters(sinceFetched int64) ([]DigestCluster, error) { + ids, err := s.clusterIDsFetchedSince(sinceFetched) + if err != nil { + return nil, err + } + if len(ids) == 0 { + return nil, nil + } + byID, order, err := s.clusterRowsByID(ids) + if err != nil { + return nil, err + } + if err := s.digestAttachArticles(byID); err != nil { + return nil, err + } + if err := s.digestAttachCVEs(byID); err != nil { + return nil, err + } + out := make([]DigestCluster, 0, len(order)) + for _, id := range order { + out = append(out, *byID[id]) + } + return out, nil +} + +func (s *Store) clusterIDsFetchedSince(sinceFetched int64) ([]int64, error) { + rows, err := s.db.Query(` + SELECT DISTINCT cm.cluster_id + FROM cluster_members cm + JOIN articles a ON a.id = cm.article_id + WHERE a.fetched_at >= ? + ORDER BY cm.cluster_id`, sinceFetched) + if err != nil { + return nil, fmt.Errorf("newly fetched clusters: %w", err) + } + defer rows.Close() + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("newly fetched clusters: scan: %w", err) + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +func (s *Store) clusterRowsByID(ids []int64) (map[int64]*DigestCluster, []int64, error) { + if len(ids) == 0 { + return map[int64]*DigestCluster{}, nil, nil + } + placeholders := make([]string, len(ids)) + args := make([]any, len(ids)) + for i, id := range ids { + placeholders[i] = "?" + args[i] = id + } + query := `SELECT id, cluster_key, size, first_seen, last_seen FROM clusters WHERE id IN (` + + strings.Join(placeholders, ",") + `) ORDER BY id` + rows, err := s.db.Query(query, args...) + if err != nil { + return nil, nil, fmt.Errorf("cluster rows by id: %w", err) + } + defer rows.Close() + return scanClusterRows(rows) +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/watch_test.go b/PROJECTS/intermediate/security-news-scraper/internal/store/watch_test.go new file mode 100644 index 00000000..a6199e7f --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/watch_test.go @@ -0,0 +1,99 @@ +// ©AngelaMos | 2026 +// watch_test.go + +package store + +import "testing" + +func TestNewlyFetchedClustersFiltersByFetchTimeNotPublishTime(t *testing.T) { + s := openTemp(t) + src, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1, Enabled: true}) + + aOld, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/old", ContentHash: "cold", TitleHash: "told", Title: "old", PublishedAt: 100, FetchedAt: 100}) + aRecent, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/recent", ContentHash: "crecent", TitleHash: "trecent", Title: "recently published, freshly fetched", PublishedAt: 940, FetchedAt: 1000}) + aFresh, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/fresh", ContentHash: "cfresh", TitleHash: "tfresh", Title: "fresh", PublishedAt: 1000, FetchedAt: 1000}) + + if err := s.ReplaceClusters([]ClusterRow{ + {Key: "old", Members: []int64{aOld}, FirstSeen: 100, LastSeen: 100}, + {Key: "recent", Members: []int64{aRecent}, FirstSeen: 940, LastSeen: 940}, + {Key: "fresh", Members: []int64{aFresh}, FirstSeen: 1000, LastSeen: 1000}, + }); err != nil { + t.Fatal(err) + } + + got, err := s.NewlyFetchedClusters(1000) + if err != nil { + t.Fatal(err) + } + seen := map[string]bool{} + for _, c := range got { + for _, a := range c.Articles { + seen[a.CanonicalURL] = true + } + } + if len(got) != 2 { + t.Fatalf("newly fetched clusters = %d, want 2 (recent + fresh)", len(got)) + } + if !seen["https://a/recent"] { + t.Error("a story fetched this cycle whose publish time is just before the watermark must be surfaced by the fetch-time query") + } + if !seen["https://a/fresh"] { + t.Error("the fresh story must be surfaced") + } + if seen["https://a/old"] { + t.Error("a story fetched before the watermark must be excluded") + } + + dig, err := s.DigestClusters(1000) + if err != nil { + t.Fatal(err) + } + for _, c := range dig { + for _, a := range c.Articles { + if a.CanonicalURL == "https://a/recent" { + t.Error("this is the whole point of the separate query: the publish-time DigestClusters filter drops the recent-but-just-fetched story, so watch must use the fetch-time query instead") + } + } + } +} + +func TestNewlyFetchedClustersEmpty(t *testing.T) { + s := openTemp(t) + got, err := s.NewlyFetchedClusters(0) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("expected no clusters on an empty store, got %d", len(got)) + } +} + +func TestNewlyFetchedClustersAttachesCVEs(t *testing.T) { + s := openTemp(t) + src, _ := s.UpsertSource(SourceInput{Name: "a", URL: "https://a/f", Type: "rss", Weight: 1, Enabled: true}) + a1, _ := s.InsertArticle(Article{SourceID: src, CanonicalURL: "https://a/1", ContentHash: "c1", TitleHash: "t1", Title: "kev story", PublishedAt: 5000, FetchedAt: 5000}) + if err := s.UpsertCVEStub("CVE-2026-9"); err != nil { + t.Fatal(err) + } + score := 9.8 + if err := s.UpdateCVEEnrichment(CVE{ID: "CVE-2026-9", CVSSScore: &score, IsKEV: true, EnrichedAt: 1, EnrichStatus: EnrichStatusOK}); err != nil { + t.Fatal(err) + } + if err := s.LinkArticleCVE(a1, "CVE-2026-9"); err != nil { + t.Fatal(err) + } + if err := s.ReplaceClusters([]ClusterRow{{Key: "1", Members: []int64{a1}, FirstSeen: 5000, LastSeen: 5000}}); err != nil { + t.Fatal(err) + } + + got, err := s.NewlyFetchedClusters(1000) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || len(got[0].CVEs) != 1 { + t.Fatalf("want 1 cluster with 1 CVE, got %d clusters", len(got)) + } + if !got[0].CVEs[0].IsKEV { + t.Error("KEV flag should be attached to the newly fetched cluster") + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/watch/notify.go b/PROJECTS/intermediate/security-news-scraper/internal/watch/notify.go new file mode 100644 index 00000000..134c1138 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/watch/notify.go @@ -0,0 +1,114 @@ +// ©AngelaMos | 2026 +// notify.go + +package watch + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" +) + +const ( + notifyMaxStatus = 300 + defaultNotifyTimeout = 15 * time.Second + headerContentType = "Content-Type" + mimeJSON = "application/json" +) + +type Notifier interface { + Notify(ctx context.Context, r Report) error +} + +type WebhookNotifier struct { + URL string + Client *http.Client +} + +type webhookPayload struct { + Text string `json:"text"` + Content string `json:"content"` + Items []webhookItem `json:"items"` +} + +type webhookItem struct { + Title string `json:"title"` + URL string `json:"url"` + Score float64 `json:"score"` + MaxCVSS float64 `json:"max_cvss"` + IsKEV bool `json:"is_kev"` + CVEs []string `json:"cves,omitempty"` + Sources int `json:"sources"` +} + +func (w WebhookNotifier) Notify(ctx context.Context, r Report) error { + if len(r.Notable) == 0 { + return nil + } + body, err := json.Marshal(buildPayload(r)) + if err != nil { + return fmt.Errorf("notify: marshal: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, w.URL, bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("notify: new request: %w", err) + } + req.Header.Set(headerContentType, mimeJSON) + resp, err := w.client().Do(req) + if err != nil { + return fmt.Errorf("notify: post: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode >= notifyMaxStatus { + return fmt.Errorf("notify: webhook returned %s", resp.Status) + } + return nil +} + +func (w WebhookNotifier) client() *http.Client { + if w.Client != nil { + return w.Client + } + return &http.Client{Timeout: defaultNotifyTimeout} +} + +func buildPayload(r Report) webhookPayload { + summary := summarize(r) + items := make([]webhookItem, len(r.Notable)) + for i, n := range r.Notable { + items[i] = webhookItem{ + Title: n.Title, + URL: n.URL, + Score: n.Score, + MaxCVSS: n.MaxCVSS, + IsKEV: n.IsKEV, + CVEs: n.CVEs, + Sources: n.Sources, + } + } + return webhookPayload{Text: summary, Content: summary, Items: items} +} + +func summarize(r Report) string { + var sb strings.Builder + fmt.Fprintf(&sb, "nadezhda: %d notable %s\n", len(r.Notable), storyWord(len(r.Notable))) + for _, n := range r.Notable { + tag := "" + if n.IsKEV { + tag = " [KEV]" + } + fmt.Fprintf(&sb, "- %s%s (%s)\n", n.Title, tag, n.URL) + } + return strings.TrimRight(sb.String(), "\n") +} + +func storyWord(n int) string { + if n == 1 { + return "story" + } + return "stories" +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/watch/notify_test.go b/PROJECTS/intermediate/security-news-scraper/internal/watch/notify_test.go new file mode 100644 index 00000000..619225c3 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/watch/notify_test.go @@ -0,0 +1,89 @@ +// ©AngelaMos | 2026 +// notify_test.go + +package watch + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func sampleReport() Report { + return Report{ + Notable: []NotableItem{ + {Title: "Critical bug in Foo", URL: "https://x/1", Score: 0.91, MaxCVSS: 9.8, IsKEV: true, CVEs: []string{"CVE-2026-1"}, Sources: 3}, + {Title: "Breach at Bar", URL: "https://x/2", Score: 0.62, Sources: 2}, + }, + } +} + +func TestWebhookNotifierPostsPayload(t *testing.T) { + got := make(chan webhookPayload, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Errorf("method = %s, want POST", r.Method) + } + if ct := r.Header.Get("Content-Type"); ct != "application/json" { + t.Errorf("content-type = %q, want application/json", ct) + } + var p webhookPayload + if err := json.NewDecoder(r.Body).Decode(&p); err != nil { + t.Errorf("decode: %v", err) + } + got <- p + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + n := WebhookNotifier{URL: srv.URL, Client: srv.Client()} + if err := n.Notify(context.Background(), sampleReport()); err != nil { + t.Fatal(err) + } + + p := <-got + if len(p.Items) != 2 { + t.Fatalf("items = %d, want 2", len(p.Items)) + } + if p.Items[0].Title != "Critical bug in Foo" || !p.Items[0].IsKEV { + t.Errorf("first item wrong: %+v", p.Items[0]) + } + if p.Text == "" || p.Content == "" { + t.Error("text and content must both be set for Slack/Discord compatibility") + } + if !strings.Contains(p.Text, "[KEV]") { + t.Errorf("summary should flag KEV items, got %q", p.Text) + } +} + +func TestWebhookNotifierNoopWhenNothingNotable(t *testing.T) { + hits := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + n := WebhookNotifier{URL: srv.URL, Client: srv.Client()} + if err := n.Notify(context.Background(), Report{NewArticles: 5}); err != nil { + t.Fatal(err) + } + if hits != 0 { + t.Errorf("webhook was called %d times, want 0 when nothing is notable", hits) + } +} + +func TestWebhookNotifierErrorsOnBadStatus(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + n := WebhookNotifier{URL: srv.URL, Client: srv.Client()} + if err := n.Notify(context.Background(), sampleReport()); err == nil { + t.Error("expected an error when the webhook returns 500") + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/watch/report.go b/PROJECTS/intermediate/security-news-scraper/internal/watch/report.go new file mode 100644 index 00000000..f293c25f --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/watch/report.go @@ -0,0 +1,28 @@ +// ©AngelaMos | 2026 +// report.go + +package watch + +import "time" + +type NotableItem struct { + Title string + URL string + Score float64 + MaxCVSS float64 + IsKEV bool + CVEs []string + Sources int +} + +type Report struct { + Start time.Time + Duration time.Duration + NewArticles int + Duplicates int + Clusters int + Enriched int + KEVHits int + Failed int + Notable []NotableItem +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/watch/watch.go b/PROJECTS/intermediate/security-news-scraper/internal/watch/watch.go new file mode 100644 index 00000000..d35c9a7c --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/watch/watch.go @@ -0,0 +1,127 @@ +// ©AngelaMos | 2026 +// watch.go + +package watch + +import ( + "context" + "fmt" + "io" + "time" +) + +type Ticker interface { + C() <-chan time.Time + Stop() +} + +type realTicker struct{ t *time.Ticker } + +func (r realTicker) C() <-chan time.Time { return r.t.C } +func (r realTicker) Stop() { r.t.Stop() } + +func NewRealTicker(d time.Duration) Ticker { return realTicker{t: time.NewTicker(d)} } + +type Options struct { + Interval time.Duration + RunAtStart bool + Cycle func(context.Context) (Report, error) + Notifier Notifier + NewTicker func(time.Duration) Ticker + Out io.Writer +} + +func (o Options) validate() error { + if o.Cycle == nil { + return fmt.Errorf("watch: Cycle is required") + } + if o.Interval <= 0 { + return fmt.Errorf("watch: Interval must be > 0, got %v", o.Interval) + } + return nil +} + +func Run(ctx context.Context, opts Options) error { + if err := opts.validate(); err != nil { + return err + } + out := resolveOut(opts.Out) + newTicker := opts.NewTicker + if newTicker == nil { + newTicker = NewRealTicker + } + + fmt.Fprintf(out, "watch: starting, interval %s\n", opts.Interval) + + if opts.RunAtStart { + if stop, err := cycleAndNotify(ctx, opts, out); stop { + return shutdown(out) + } else if err != nil { + fmt.Fprintf(out, "watch: cycle error: %v\n", err) + } + } + + ticker := newTicker(opts.Interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return shutdown(out) + case <-ticker.C(): + if stop, err := cycleAndNotify(ctx, opts, out); stop { + return shutdown(out) + } else if err != nil { + fmt.Fprintf(out, "watch: cycle error: %v\n", err) + } + } + } +} + +func Once(ctx context.Context, opts Options) error { + if err := opts.validate(); err != nil { + return err + } + out := resolveOut(opts.Out) + _, err := cycleAndNotify(ctx, opts, out) + return err +} + +func cycleAndNotify(ctx context.Context, opts Options, out io.Writer) (stop bool, err error) { + if ctx.Err() != nil { + return true, nil + } + report, err := opts.Cycle(ctx) + if err != nil { + if ctx.Err() != nil { + return true, nil + } + return false, err + } + logReport(out, report) + if opts.Notifier != nil && len(report.Notable) > 0 { + if nerr := opts.Notifier.Notify(ctx, report); nerr != nil { + fmt.Fprintf(out, "watch: notify error: %v\n", nerr) + } else { + fmt.Fprintf(out, "watch: notified %d notable\n", len(report.Notable)) + } + } + return false, nil +} + +func shutdown(out io.Writer) error { + fmt.Fprintln(out, "watch: shutdown") + return nil +} + +func resolveOut(out io.Writer) io.Writer { + if out != nil { + return out + } + return io.Discard +} + +func logReport(out io.Writer, r Report) { + fmt.Fprintf(out, "watch: cycle done in %s: %d new, %d dup, %d clusters, %d enriched (%d KEV), %d failed, %d notable\n", + r.Duration.Round(time.Millisecond), r.NewArticles, r.Duplicates, r.Clusters, r.Enriched, r.KEVHits, r.Failed, len(r.Notable)) +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/watch/watch_test.go b/PROJECTS/intermediate/security-news-scraper/internal/watch/watch_test.go new file mode 100644 index 00000000..cd9732a3 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/watch/watch_test.go @@ -0,0 +1,198 @@ +// ©AngelaMos | 2026 +// watch_test.go + +package watch + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +type fakeTicker struct{ ch chan time.Time } + +func (f *fakeTicker) C() <-chan time.Time { return f.ch } +func (f *fakeTicker) Stop() {} + +type fakeNotifier struct { + mu sync.Mutex + calls int + last Report +} + +func (f *fakeNotifier) Notify(_ context.Context, r Report) error { + f.mu.Lock() + defer f.mu.Unlock() + f.calls++ + f.last = r + return nil +} + +func (f *fakeNotifier) count() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls +} + +func recvWithin(t *testing.T, ch <-chan int, d time.Duration) int { + t.Helper() + select { + case v := <-ch: + return v + case <-time.After(d): + t.Fatal("timed out waiting for a watch cycle") + return 0 + } +} + +func TestRunAtStartThenTicksThenGracefulShutdown(t *testing.T) { + ticker := &fakeTicker{ch: make(chan time.Time)} + ran := make(chan int, 4) + count := 0 + ctx, cancel := context.WithCancel(context.Background()) + + opts := Options{ + Interval: time.Hour, + RunAtStart: true, + NewTicker: func(time.Duration) Ticker { return ticker }, + Cycle: func(context.Context) (Report, error) { + count++ + ran <- count + return Report{}, nil + }, + } + + done := make(chan error, 1) + go func() { done <- Run(ctx, opts) }() + + if n := recvWithin(t, ran, 2*time.Second); n != 1 { + t.Fatalf("RunAtStart cycle = %d, want 1", n) + } + ticker.ch <- time.Unix(0, 0) + if n := recvWithin(t, ran, 2*time.Second); n != 2 { + t.Fatalf("post-tick cycle = %d, want 2", n) + } + + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("Run returned %v, want nil on graceful shutdown", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Run did not return after ctx cancel") + } +} + +func TestRunReturnsNilAndSkipsCycleOnImmediateCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + called := false + err := Run(ctx, Options{ + Interval: time.Hour, + RunAtStart: true, + NewTicker: func(time.Duration) Ticker { return &fakeTicker{ch: make(chan time.Time)} }, + Cycle: func(context.Context) (Report, error) { + called = true + return Report{}, nil + }, + }) + if err != nil { + t.Fatalf("Run = %v, want nil", err) + } + if called { + t.Error("cycle should not run when the context is already cancelled") + } +} + +func TestRunContinuesAfterCycleError(t *testing.T) { + ticker := &fakeTicker{ch: make(chan time.Time)} + ran := make(chan int, 4) + count := 0 + ctx, cancel := context.WithCancel(context.Background()) + + err := make(chan error, 1) + go func() { + err <- Run(ctx, Options{ + Interval: time.Hour, + RunAtStart: true, + NewTicker: func(time.Duration) Ticker { return ticker }, + Cycle: func(context.Context) (Report, error) { + count++ + if count == 1 { + return Report{}, errors.New("boom") + } + ran <- count + return Report{}, nil + }, + }) + }() + + ticker.ch <- time.Unix(0, 0) + if n := recvWithin(t, ran, 2*time.Second); n != 2 { + t.Fatalf("second cycle = %d, want 2 (daemon survived the first cycle error)", n) + } + cancel() + if e := <-err; e != nil { + t.Fatalf("Run = %v, want nil", e) + } +} + +func TestOnceNotifiesWhenNotable(t *testing.T) { + fn := &fakeNotifier{} + err := Once(context.Background(), Options{ + Interval: time.Hour, + Cycle: func(context.Context) (Report, error) { + return Report{Notable: []NotableItem{{Title: "x"}}}, nil + }, + Notifier: fn, + }) + if err != nil { + t.Fatal(err) + } + if fn.count() != 1 { + t.Fatalf("notify calls = %d, want 1", fn.count()) + } +} + +func TestOnceSkipsNotifyWhenNoNotable(t *testing.T) { + fn := &fakeNotifier{} + err := Once(context.Background(), Options{ + Interval: time.Hour, + Cycle: func(context.Context) (Report, error) { + return Report{NewArticles: 3}, nil + }, + Notifier: fn, + }) + if err != nil { + t.Fatal(err) + } + if fn.count() != 0 { + t.Fatalf("notify calls = %d, want 0 (nothing notable)", fn.count()) + } +} + +func TestOnceReturnsCycleError(t *testing.T) { + want := errors.New("cycle failed") + err := Once(context.Background(), Options{ + Interval: time.Hour, + Cycle: func(context.Context) (Report, error) { + return Report{}, want + }, + }) + if !errors.Is(err, want) { + t.Fatalf("Once error = %v, want %v", err, want) + } +} + +func TestValidateRejectsBadOptions(t *testing.T) { + if err := (Options{Interval: time.Hour}).validate(); err == nil { + t.Error("validate should reject a nil Cycle") + } + if err := (Options{Cycle: func(context.Context) (Report, error) { return Report{}, nil }}).validate(); err == nil { + t.Error("validate should reject a non-positive Interval") + } +} From a27cae6ab740aa6efce22797679a179857b1802e Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Mon, 6 Jul 2026 22:56:00 -0400 Subject: [PATCH 11/15] build(nadezhda): M8 release packaging - install.sh, goreleaser, CI, LICENSE One-shot install.sh (curl|bash -> prebuilt release binary, or go install fallback with Go-toolchain bootstrap; installs to ~/.local/bin and wires PATH). .goreleaser.yaml + .github/workflows/release.yml build cross-platform binaries on a v* tag (inert in the monorepo, active once the project dir is the standalone repo root). Add the AGPL-3.0 LICENSE and rewrite README.md with install + quickstart. Make internal/version.Version a var so goreleaser injects the real version via ldflags. justfile gains watch (run daemon from source) and publish (drift-free rsync mirror to the standalone github.com/CarterPerez-dev/nadezhda checkout). --- .../.github/workflows/release.yml | 32 + .../security-news-scraper/.goreleaser.yaml | 45 ++ .../security-news-scraper/LICENSE | 661 ++++++++++++++++++ .../security-news-scraper/README.md | 117 +++- .../security-news-scraper/install.sh | 259 +++++++ .../internal/version/version.go | 7 +- .../security-news-scraper/justfile | 13 + 7 files changed, 1110 insertions(+), 24 deletions(-) create mode 100644 PROJECTS/intermediate/security-news-scraper/.github/workflows/release.yml create mode 100644 PROJECTS/intermediate/security-news-scraper/.goreleaser.yaml create mode 100644 PROJECTS/intermediate/security-news-scraper/LICENSE create mode 100755 PROJECTS/intermediate/security-news-scraper/install.sh diff --git a/PROJECTS/intermediate/security-news-scraper/.github/workflows/release.yml b/PROJECTS/intermediate/security-news-scraper/.github/workflows/release.yml new file mode 100644 index 00000000..f77de352 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/.github/workflows/release.yml @@ -0,0 +1,32 @@ +# ©AngelaMos | 2026 +# release.yml + +name: release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - uses: goreleaser/goreleaser-action@v6 + with: + distribution: goreleaser + version: "~> v2" + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/PROJECTS/intermediate/security-news-scraper/.goreleaser.yaml b/PROJECTS/intermediate/security-news-scraper/.goreleaser.yaml new file mode 100644 index 00000000..af2df796 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/.goreleaser.yaml @@ -0,0 +1,45 @@ +# ©AngelaMos | 2026 +# .goreleaser.yaml + +version: 2 + +project_name: nadezhda + +before: + hooks: + - go mod tidy + +builds: + - id: nadezhda + main: ./cmd/nadezhda + binary: nadezhda + env: + - CGO_ENABLED=0 + goos: [linux, darwin] + goarch: [amd64, arm64] + ldflags: + - -s -w -X github.com/CarterPerez-dev/nadezhda/internal/version.Version={{ .Version }} + +archives: + - id: nadezhda + name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + formats: [tar.gz] + files: + - README.md + - LICENSE + +checksum: + name_template: "checksums.txt" + +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" + - "^chore:" + +release: + github: + owner: CarterPerez-dev + name: nadezhda diff --git a/PROJECTS/intermediate/security-news-scraper/LICENSE b/PROJECTS/intermediate/security-news-scraper/LICENSE new file mode 100644 index 00000000..0ad25db4 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/PROJECTS/intermediate/security-news-scraper/README.md b/PROJECTS/intermediate/security-news-scraper/README.md index f7e62dff..715d196f 100644 --- a/PROJECTS/intermediate/security-news-scraper/README.md +++ b/PROJECTS/intermediate/security-news-scraper/README.md @@ -1,35 +1,112 @@ -# Nadezhda +# nadezhda -A concurrent security-news and CVE aggregation engine, written in Go. +Security news and CVE intelligence, aggregated in your terminal. Keyless by default. -Nadezhda ingests cybersecurity news from reliable RSS feeds, enriches every -referenced CVE with authoritative exploit intelligence (NVD, CISA KEV, FIRST -EPSS), clusters the same story across outlets, ranks items by real-world -significance, and surfaces content angles. It ships as a single static binary -with a local SQLite store, a colorful terminal UI, and Markdown/JSON export. +Nadezhda pulls cybersecurity news from reliable RSS/Atom feeds, clusters the same +story across outlets, enriches every referenced CVE with authoritative exploit +intelligence, ranks what actually matters, and hands you a browsable dossier. It +ships as a single static binary with a local SQLite store, a colorful terminal UI, +Markdown/JSON export, an optional AI ideation layer, and an optional watch daemon. -## Status +No API key is required to run it. CVE enrichment comes from keyless authoritative +sources (the CVE Program list, CISA KEV, and FIRST EPSS); an NVD key is an optional +booster, never a requirement. -Early development. The scaffold is in place: configuration, source registry, -SQLite store with forward-only migrations, and the command skeleton. +## Install ``` -nadezhda version # print version -nadezhda sources # list configured feeds and persist them to the store +curl -fsSL https://angelamos.com/nadezhda/install.sh | bash ``` -Ingestion, CVE enrichment, ranking, the TUI, and the AI ideation layer land in -subsequent milestones. +One command, zero further steps: it grabs a prebuilt binary for your platform (no Go +toolchain needed), drops it on your `PATH`, and leaves `nadezhda` runnable by name. -## Build +Prefer the Go toolchain? A real `go install` works too: ``` -just build # -> ./nadezhda -just test +go install github.com/CarterPerez-dev/nadezhda/cmd/nadezhda@latest ``` -Requires Go 1.25+. +Or build from a clone: ---- +``` +git clone https://github.com/CarterPerez-dev/nadezhda +cd nadezhda +just build # -> ./nadezhda (or: go build -o nadezhda ./cmd/nadezhda) +``` -Full documentation lands in `learn/` as the project matures. +Requires Go 1.25+ only when building from source; the toolchain is fetched +automatically if you are on an older Go. + +## Quick start + +``` +nadezhda scrape # pull every enabled feed, cluster, enrich CVEs +nadezhda tui # browse the ranked dossier (press ? for keys, o to open a story) +nadezhda digest --top 20 # render a ranked digest to Markdown (or --format json) +nadezhda list --kev # list stored articles, filtered +nadezhda cve CVE-2021-44228 # show one enriched CVE and the stories that mention it +``` + +The default flow is `scrape` then `tui`. Scrape already runs enrichment (best effort, +never blocking the news), so there is nothing else to wire up. + +## Content ideation (optional AI) + +Turn ranked clusters into summaries and content angles. Run the built-in setup wizard +once and paste a single key, or point it at a local Ollama: + +``` +nadezhda ai # interactive, re-runnable: Claude / OpenAI / Gemini / Ollama +nadezhda ideate --top 5 # generate angles for the top clusters +``` + +The default provider is a local Qwen model via Ollama (keyless). Keys, when used, are +read from your environment and a `0600` credentials file; they are never logged. With +AI disabled, nadezhda is a complete aggregator. + +## Watch daemon (optional) + +Run nadezhda as a long-lived daemon that re-ingests on an interval and, when a webhook +is configured, posts genuinely new high-signal stories to Slack, Discord, or any JSON +endpoint: + +``` +nadezhda watch --interval 1h +nadezhda watch --once # a single cycle, for cron +``` + +Configure `watch.webhook_url` (and thresholds) in `config.yaml`. The daemon shuts down +cleanly on Ctrl-C / SIGTERM and never crashes on a transient feed or network error. + +## How it works + +``` +feeds -> fetch (rate-limited, conditional GET) -> parse -> normalize -> dedup + -> cluster (cross-outlet + shared-CVE) -> enrich (CVE list / KEV / EPSS) + -> rank (news-first weighted score) -> tui | digest | ideate | watch +``` + +Ranking is pure and deterministic: recency, cross-outlet velocity, source trust, and a +keyword watchlist dominate, with CVSS / KEV / EPSS as supporting signals. Everything is +configurable in `config.yaml`; nothing is a magic constant in the code. + +## Configuration + +Nadezhda runs with sensible defaults and an embedded source list. To customize, pass +`--config config.yaml`. Notable keys: `watchlist`, `fetch.*`, `cluster.*`, `rank.*`, +`ai.*`, and `watch.*`. The Ollama host port defaults to a non-standard `39847` +(override with `OLLAMA_HOST_PORT`) so it never collides with an existing Ollama. + +## Development + +``` +just build # build the binary +just test # go test ./... +just watch # run the daemon from source +just ollama-up # stand up the local Qwen runtime (Docker) +``` + +## License + +AGPL-3.0. See [LICENSE](LICENSE). diff --git a/PROJECTS/intermediate/security-news-scraper/install.sh b/PROJECTS/intermediate/security-news-scraper/install.sh new file mode 100755 index 00000000..115d1e6a --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/install.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash +# ©AngelaMos | 2026 +# install.sh +# +# One-shot installer for nadezhda. Takes a fresh machine to `nadezhda` runnable +# by its bare name, with zero further steps, whether run from a clone or piped +# from a domain via curl. Prefers a prebuilt release binary (no Go needed); +# falls back to building from source (auto-installs the Go toolchain if absent). + +set -euo pipefail + +# ============================================================================ +# CONFIG +# ============================================================================ +REPO_OWNER="CarterPerez-dev" +REPO_NAME="nadezhda" +BINARY="nadezhda" +TAGLINE="Security news + CVE intelligence in your terminal. Keyless by default." +REPO_URL="https://github.com/${REPO_OWNER}/${REPO_NAME}.git" +INSTALL_DIR="${NADEZHDA_INSTALL_DIR:-$HOME/.local/bin}" +DEFAULT_BRANCH="main" +GO_MIN="1.21" +PREBUILT=1 + +# ============================================================================ +# Colors +# ============================================================================ +if [ -t 2 ] && [ -z "${NO_COLOR:-}" ]; then + BOLD=$'\033[1m'; DIM=$'\033[2m'; RED=$'\033[31m'; GREEN=$'\033[32m' + YELLOW=$'\033[33m'; CYAN=$'\033[36m'; RESET=$'\033[0m' +else + BOLD=""; DIM=""; RED=""; GREEN=""; YELLOW=""; CYAN=""; RESET="" +fi + +info() { printf '%s\n' " ${CYAN}+${RESET} $*" >&2; } +ok() { printf '%s\n' " ${GREEN}+${RESET} $*" >&2; } +warn() { printf '%s\n' " ${YELLOW}!${RESET} $*" >&2; } +die() { printf '%s\n' " ${RED}x $*${RESET}" >&2; exit 1; } +header(){ printf '\n%s\n\n' "${BOLD}${CYAN}--- $* ---${RESET}" >&2; } +have() { command -v "$1" >/dev/null 2>&1; } + +trap 'printf "%s\n" "${RED}x install failed${RESET}" >&2' ERR +TMP_DIR="" +cleanup() { [ -n "$TMP_DIR" ] && rm -rf "$TMP_DIR"; return 0; } +trap cleanup EXIT + +banner() { + printf '%s' "${CYAN}${BOLD}" >&2 + cat >&2 <<'ART' + ╭───────────────────────╮ + │ n a d e z h d a │ + ╰───────────────────────╯ +ART + printf '%s\n' "${RESET}" >&2 + printf '%s\n' " ${DIM}${TAGLINE}${RESET}" >&2 +} + +# ============================================================================ +# Privilege + package-manager fan +# ============================================================================ +SUDO="" +if [ "$(id -u)" -ne 0 ]; then + if have sudo; then SUDO="sudo"; fi +fi + +pkg_install() { + if have apt-get; then $SUDO apt-get update -y || warn "apt update had errors; continuing" + $SUDO apt-get install -y --no-install-recommends "$@" + elif have dnf; then $SUDO dnf install -y "$@" + elif have pacman; then $SUDO pacman -S --needed --noconfirm "$@" + elif have zypper; then $SUDO zypper install -y "$@" + elif have apk; then $SUDO apk add "$@" + elif have brew; then brew install "$@" + else die "no known package manager. Install manually: $*"; fi +} + +download() { + if have curl; then curl -fsSL "$1" -o "$2" || return 1 + elif have wget; then wget -qO "$2" "$1" || return 1 + else die "need curl or wget"; fi +} + +# ============================================================================ +# Args +# ============================================================================ +usage() { + cat >&2 </dev/null || warn "pull failed; using existing clone" + else + info "cloning ${REPO_URL}" + git clone --depth 1 --branch "$DEFAULT_BRANCH" --quiet "$REPO_URL" "$cache" \ + || die "clone failed from ${REPO_URL}" + fi + printf '%s\n' "$cache" +} + +# ============================================================================ +# Toolchain (Go) + build from source +# ============================================================================ +install_go() { + info "installing a current Go toolchain" + local latest tgz + latest="$(download "https://go.dev/VERSION?m=text" /dev/stdout 2>/dev/null | head -n1)" || latest="" + case "$latest" in go*) ;; *) latest="go1.25.5" ;; esac + tgz="${latest}.${OS}-${ARCH}.tar.gz" + TMP_DIR="${TMP_DIR:-$(mktemp -d)}" + download "https://go.dev/dl/${tgz}" "$TMP_DIR/go.tgz" || die "failed to download ${tgz} from go.dev/dl" + rm -rf "$HOME/.local/go" + mkdir -p "$HOME/.local" + tar -C "$HOME/.local" -xzf "$TMP_DIR/go.tgz" || die "failed to extract Go" + export PATH="$HOME/.local/go/bin:$PATH" + export GOTOOLCHAIN=auto + have go || die "Go toolchain install failed" + ok "go $(go env GOVERSION 2>/dev/null | sed 's/^go//') at ~/.local/go" +} + +need_toolchain() { + local cur + if have go; then + cur="$(go env GOVERSION 2>/dev/null | sed 's/^go//')" + if [ -n "$cur" ] && [ "$(printf '%s\n%s\n' "$GO_MIN" "$cur" | sort -V | head -n1)" = "$GO_MIN" ]; then + export GOTOOLCHAIN=auto + ok "go $cur (auto-toolchain fetches the go.mod-pinned version if newer)" + return + fi + warn "go ${cur:-unknown} predates toolchain auto-download; installing a current Go" + fi + install_go +} + +build_from_source() { + info "building ${BINARY} (compiles the pure-Go SQLite driver; give it a minute)" + mkdir -p "$INSTALL_DIR" + GOBIN="$INSTALL_DIR" go install ./cmd/nadezhda || die "go install failed" + ok "installed ${BINARY} -> ${INSTALL_DIR}/${BINARY}" +} + +try_prebuilt() { + [ "$PREBUILT" = "1" ] || return 1 + local ver archive url + ver="$(download "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/releases/latest" /dev/stdout 2>/dev/null \ + | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')" || true + [ -n "$ver" ] || return 1 + archive="${BINARY}_${ver#v}_${OS}_${ARCH}.tar.gz" + url="https://github.com/${REPO_OWNER}/${REPO_NAME}/releases/download/${ver}/${archive}" + TMP_DIR="$(mktemp -d)" + download "$url" "$TMP_DIR/a.tgz" || { warn "no prebuilt for ${OS}/${ARCH}; will build from source"; return 1; } + tar -xzf "$TMP_DIR/a.tgz" -C "$TMP_DIR" || return 1 + mkdir -p "$INSTALL_DIR"; install -m 0755 "$TMP_DIR/$BINARY" "$INSTALL_DIR/$BINARY" + ok "installed prebuilt ${ver} -> ${INSTALL_DIR}/${BINARY}" +} + +# ============================================================================ +# PATH wiring +# ============================================================================ +wire_path() { + case ":$PATH:" in *":$INSTALL_DIR:"*) ok "$INSTALL_DIR already on PATH"; return ;; esac + local shell rc="" + shell="$(basename "${SHELL:-bash}")" + case "$shell" in + zsh) rc="$HOME/.zshrc" ;; + fish) mkdir -p "$HOME/.config/fish/conf.d" + echo "fish_add_path $INSTALL_DIR" > "$HOME/.config/fish/conf.d/${BINARY}.fish" + ok "added to fish conf.d" ;; + bash) rc="$HOME/.bashrc"; [ -f "$rc" ] || rc="$HOME/.bash_profile" ;; + *) rc="$HOME/.profile" ;; + esac + if [ -n "$rc" ] && ! grep -q "$INSTALL_DIR" "$rc" 2>/dev/null; then + printf '\nexport PATH="%s:$PATH"\n' "$INSTALL_DIR" >> "$rc" + ok "added $INSTALL_DIR to PATH in $rc" + fi + export PATH="$INSTALL_DIR:$PATH" +} + +# ============================================================================ +# Main +# ============================================================================ +main() { + banner + have "$BINARY" && info "existing install at $(command -v "$BINARY") — updating" + + REPO="" + if ! try_prebuilt; then + header "Building from source" + REPO="$(resolve_repo)"; cd "$REPO" + need_toolchain + build_from_source + fi + + wire_path + + header "Verify" + if have "$BINARY"; then + ok "$BINARY -> $(command -v "$BINARY")" + "$BINARY" version 2>/dev/null || true + else + warn "installed to $INSTALL_DIR but not yet on PATH — open a new shell" + fi + + printf '\n%s\n\n' " ${GREEN}${BOLD}${BINARY} is ready.${RESET}" >&2 + if have just && [ -n "$REPO" ] && [ -f "${REPO}/justfile" ]; then + printf '%s\n' " ${DIM}dev commands:${RESET} just" >&2 + fi + cat >&2 <