feat(nadezhda): M3 CVE extraction + NVD/KEV/EPSS enrichment
The intelligence layer. scrape now extracts CVE IDs (cheap, no network) into
article_cves, which activates M2's shared-CVE clustering. A separate 'enrich'
command does the slow, cached API work (Carter's split: fast scrape, deliberate
enrich).
- internal/cve: regex CVE-ID extraction + three clients (stdlib net/http):
- nvd: API 2.0, CVSS precedence v4.0>v3.1>v3.0>v2 with nullable metrics and v2
metric-level severity fallback, totalResults==0 => not-found, apiKey header
(exact case), token-bucket rate limit (5/30s anon, 50/30s keyed) + retry on
429/5xx, timeouts non-retryable.
- kev: one catalog download -> membership map; knownRansomwareCampaignUse is the
STRING 'Known'/'Unknown' mapped explicitly to bool.
- epss: batched; epss/percentile are QUOTED STRINGS parsed with ParseFloat (a
parse failure is skipped, a legitimate 0.0 is kept); partial batches survive
a transient error.
- internal/enrich: enriches only unenriched/stale CVEs (positive + negative TTL),
KEV once per run, EPSS batched, NVD per-CVE. NVD/EPSS soft per-item (resumable),
KEV fatal by design (enriching without it would persist wrong is_kev for a TTL).
- store: UpsertCVEStub/LinkArticleCVE, CVEsNeedingEnrichment (TTL), UpdateCVEEnrichment
(nullable *float64 -> SQL NULL), GetCVE, ArticlesForCVE, ListArticles (parameterized
--source/--since/--min-cvss/--kev/--keyword via EXISTS subqueries).
- commands: enrich, cve, list (cve/list off the stub list). Trimmed kev-sample.json
(7KB) fixture for KAT tests; nvd/epss fixtures committed.
- KAT proven offline AND live: CVE-2021-44228 -> CVSS 10.0 CRITICAL (3.1), CWE-20,
KEV added 2021-12-10 ransomware yes, EPSS 0.99999. 215 articles -> 82 CVEs, and
shared-CVE clustering lit up (multi-source clusters 2 -> 5). Suite offline + -race.
One read-only audit agent run; 0 Crit/High/Med, Low/Nit fixes applied in-phase.
NVD apiKey header set with exact case to bypass Go header canonicalization.
This commit is contained in:
parent
312b13b348
commit
5865bb6149
|
|
@ -0,0 +1,105 @@
|
||||||
|
// ©AngelaMos | 2026
|
||||||
|
// cve.go
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"github.com/CarterPerez-dev/nadezhda/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
const naValue = "n/a"
|
||||||
|
|
||||||
|
var cveCmd = &cobra.Command{
|
||||||
|
Use: "cve CVE-YYYY-NNNN",
|
||||||
|
Short: "Show an enriched CVE and the articles mentioning it",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
RunE: runCVE,
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
rootCmd.AddCommand(cveCmd)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runCVE(cmd *cobra.Command, args []string) error {
|
||||||
|
cfg, err := loadConfig()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
st, err := store.Open(cfg.DBPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer st.Close()
|
||||||
|
|
||||||
|
id := strings.ToUpper(args[0])
|
||||||
|
c, err := st.GetCVE(id)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
return fmt.Errorf("%s not found; run 'nadezhda scrape' to discover it", id)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
out := cmd.OutOrStdout()
|
||||||
|
fmt.Fprintf(out, "%s\n", c.ID)
|
||||||
|
if c.EnrichedAt == 0 {
|
||||||
|
fmt.Fprintln(out, " not enriched yet; run 'nadezhda enrich'")
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(out, " CVSS: %s\n", cvssLine(c))
|
||||||
|
fmt.Fprintf(out, " CWE: %s\n", orNA(c.CWE))
|
||||||
|
fmt.Fprintf(out, " KEV: %s\n", kevLine(c))
|
||||||
|
fmt.Fprintf(out, " EPSS: %s\n", epssLine(c))
|
||||||
|
if c.Description != "" {
|
||||||
|
fmt.Fprintf(out, "\n %s\n", c.Description)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
articles, err := st.ArticlesForCVE(id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "\nMentioned in %d article(s):\n", len(articles))
|
||||||
|
for _, a := range articles {
|
||||||
|
fmt.Fprintf(out, " [%s] %s\n %s\n", a.SourceName, a.Title, a.CanonicalURL)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cvssLine(c store.CVE) string {
|
||||||
|
if c.CVSSScore == nil {
|
||||||
|
return naValue
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%.1f %s (%s) %s", *c.CVSSScore, orNA(c.CVSSSeverity), orNA(c.CVSSVersion), c.CVSSVector)
|
||||||
|
}
|
||||||
|
|
||||||
|
func kevLine(c store.CVE) string {
|
||||||
|
if !c.IsKEV {
|
||||||
|
return "no"
|
||||||
|
}
|
||||||
|
ransom := "no"
|
||||||
|
if c.KEVRansomware {
|
||||||
|
ransom = "yes"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("yes (added %s, ransomware: %s)", orNA(c.KEVDateAdded), ransom)
|
||||||
|
}
|
||||||
|
|
||||||
|
func epssLine(c store.CVE) string {
|
||||||
|
if c.EPSS == nil || c.EPSSPercentile == nil {
|
||||||
|
return naValue
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%.5f (percentile %.5f)", *c.EPSS, *c.EPSSPercentile)
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNA(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return naValue
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,67 @@
|
||||||
|
// ©AngelaMos | 2026
|
||||||
|
// enrich.go
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"github.com/CarterPerez-dev/nadezhda/internal/cve"
|
||||||
|
"github.com/CarterPerez-dev/nadezhda/internal/enrich"
|
||||||
|
"github.com/CarterPerez-dev/nadezhda/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
const nvdAPIKeyEnv = "NVD_API_KEY"
|
||||||
|
|
||||||
|
var enrichCmd = &cobra.Command{
|
||||||
|
Use: "enrich",
|
||||||
|
Short: "Enrich extracted CVEs with NVD, CISA KEV, and EPSS intelligence",
|
||||||
|
RunE: runEnrich,
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
rootCmd.AddCommand(enrichCmd)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runEnrich(cmd *cobra.Command, args []string) error {
|
||||||
|
cfg, err := loadConfig()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
st, err := store.Open(cfg.DBPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer st.Close()
|
||||||
|
|
||||||
|
httpClient := &http.Client{Timeout: time.Duration(cfg.Fetch.TimeoutSeconds) * time.Second}
|
||||||
|
apiKey := os.Getenv(nvdAPIKeyEnv)
|
||||||
|
if apiKey == "" {
|
||||||
|
apiKey = cfg.Enrich.NVDAPIKey
|
||||||
|
}
|
||||||
|
clients := enrich.Clients{
|
||||||
|
NVD: cve.NewNVDClient(httpClient, cve.NVDEndpoint, apiKey),
|
||||||
|
KEV: cve.NewKEVClient(httpClient, cve.KEVEndpoint),
|
||||||
|
EPSS: cve.NewEPSSClient(httpClient, cve.EPSSEndpoint),
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
stats, err := enrich.Run(ctx, st, clients, time.Now(), cfg.Enrich.CacheTTLHours, cfg.Enrich.NegativeTTLHours)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintf(cmd.OutOrStdout(),
|
||||||
|
"enriched %d/%d CVEs (%d not in NVD, %d KEV, %d errors)\n",
|
||||||
|
stats.Enriched, stats.Total, stats.NotFound, stats.KEVHits, stats.Errors)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,92 @@
|
||||||
|
// ©AngelaMos | 2026
|
||||||
|
// list.go
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
|
"github.com/CarterPerez-dev/nadezhda/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultListLimit = 50
|
||||||
|
dateLayout = "2006-01-02"
|
||||||
|
noDate = "----------"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
listSource string
|
||||||
|
listSince string
|
||||||
|
listMinCVSS float64
|
||||||
|
listKEV bool
|
||||||
|
listKeyword string
|
||||||
|
listLimit int
|
||||||
|
)
|
||||||
|
|
||||||
|
var listCmd = &cobra.Command{
|
||||||
|
Use: "list",
|
||||||
|
Short: "List stored articles with filters",
|
||||||
|
RunE: runList,
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
listCmd.Flags().StringVar(&listSource, "source", "", "filter by source name")
|
||||||
|
listCmd.Flags().StringVar(&listSince, "since", "", "only articles published within this window (e.g. 24h, 168h)")
|
||||||
|
listCmd.Flags().Float64Var(&listMinCVSS, "min-cvss", 0, "only articles referencing a CVE with CVSS >= this")
|
||||||
|
listCmd.Flags().BoolVar(&listKEV, "kev", false, "only articles referencing a KEV-listed CVE")
|
||||||
|
listCmd.Flags().StringVar(&listKeyword, "keyword", "", "filter by keyword in title or summary")
|
||||||
|
listCmd.Flags().IntVar(&listLimit, "limit", defaultListLimit, "maximum rows to show")
|
||||||
|
rootCmd.AddCommand(listCmd)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runList(cmd *cobra.Command, args []string) error {
|
||||||
|
cfg, err := loadConfig()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
filter := store.ListFilter{
|
||||||
|
Source: listSource,
|
||||||
|
MinCVSS: listMinCVSS,
|
||||||
|
KEV: listKEV,
|
||||||
|
Keyword: listKeyword,
|
||||||
|
Limit: listLimit,
|
||||||
|
}
|
||||||
|
if listSince != "" {
|
||||||
|
d, err := time.ParseDuration(listSince)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid --since %q: %w", listSince, err)
|
||||||
|
}
|
||||||
|
filter.Since = time.Now().Add(-d).Unix()
|
||||||
|
}
|
||||||
|
|
||||||
|
st, err := store.Open(cfg.DBPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer st.Close()
|
||||||
|
|
||||||
|
articles, err := st.ListArticles(filter)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
out := cmd.OutOrStdout()
|
||||||
|
fmt.Fprintf(out, "%-10s %-16s %s\n", "DATE", "SOURCE", "TITLE")
|
||||||
|
for _, a := range articles {
|
||||||
|
fmt.Fprintf(out, "%-10s %-16s %s\n", formatDate(a.PublishedAt), a.SourceName, a.Title)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "\n%d article(s)\n", len(articles))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func formatDate(unix int64) string {
|
||||||
|
if unix == 0 {
|
||||||
|
return noDate
|
||||||
|
}
|
||||||
|
return time.Unix(unix, 0).UTC().Format(dateLayout)
|
||||||
|
}
|
||||||
|
|
@ -105,14 +105,16 @@ func selectTargets(srcs []source.Source, only string) ([]source.Source, error) {
|
||||||
|
|
||||||
func printSummary(cmd *cobra.Command, summary ingest.Summary) {
|
func printSummary(cmd *cobra.Command, summary ingest.Summary) {
|
||||||
out := cmd.OutOrStdout()
|
out := cmd.OutOrStdout()
|
||||||
fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s\n", "SOURCE", "STATUS", "PARSED", "NEW", "DUP", "ERR")
|
fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s %-5s\n", "SOURCE", "STATUS", "PARSED", "NEW", "DUP", "CVE", "ERR")
|
||||||
|
totalCVEs := 0
|
||||||
for _, r := range summary.Results {
|
for _, r := range summary.Results {
|
||||||
fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s\n",
|
totalCVEs += r.CVEs
|
||||||
r.Name, status(r), count(r, r.Parsed), count(r, r.New), count(r, r.Duplicates), count(r, r.ItemErrors))
|
fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s %-5s\n",
|
||||||
|
r.Name, status(r), count(r, r.Parsed), count(r, r.New), count(r, r.Duplicates), count(r, r.CVEs), count(r, r.ItemErrors))
|
||||||
}
|
}
|
||||||
newArticles, duplicates, failed := summary.Totals()
|
newArticles, duplicates, failed := summary.Totals()
|
||||||
fmt.Fprintf(out, "\n%d new, %d duplicate across %d sources (%d failed)\n",
|
fmt.Fprintf(out, "\n%d new, %d duplicate, %d CVE refs across %d sources (%d failed)\n",
|
||||||
newArticles, duplicates, len(summary.Results), failed)
|
newArticles, duplicates, totalCVEs, len(summary.Results), failed)
|
||||||
for _, r := range summary.Results {
|
for _, r := range summary.Results {
|
||||||
if r.Err != nil {
|
if r.Err != nil {
|
||||||
fmt.Fprintf(out, " %s: %v\n", r.Name, r.Err)
|
fmt.Fprintf(out, " %s: %v\n", r.Name, r.Err)
|
||||||
|
|
|
||||||
|
|
@ -21,8 +21,6 @@ func init() {
|
||||||
short string
|
short string
|
||||||
milestone string
|
milestone string
|
||||||
}{
|
}{
|
||||||
{"list", "List stored articles with filters", "milestone M3"},
|
|
||||||
{"cve", "Show an enriched CVE and articles mentioning it", "milestone M3"},
|
|
||||||
{"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"},
|
{"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"},
|
||||||
{"tui", "Browse aggregated news in an interactive terminal UI", "milestone M5"},
|
{"tui", "Browse aggregated news in an interactive terminal UI", "milestone M5"},
|
||||||
{"ideate", "Generate content angles from ranked clusters via an AI provider", "milestone M6"},
|
{"ideate", "Generate content angles from ranked clusters via an AI provider", "milestone M6"},
|
||||||
|
|
|
||||||
|
|
@ -67,8 +67,9 @@ type Fetch struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
type Enrich struct {
|
type Enrich struct {
|
||||||
CacheTTLHours int `yaml:"cache_ttl_hours"`
|
CacheTTLHours int `yaml:"cache_ttl_hours"`
|
||||||
NegativeTTLHours int `yaml:"negative_ttl_hours"`
|
NegativeTTLHours int `yaml:"negative_ttl_hours"`
|
||||||
|
NVDAPIKey string `yaml:"nvd_api_key"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Cluster struct {
|
type Cluster struct {
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,207 @@
|
||||||
|
// ©AngelaMos | 2026
|
||||||
|
// cve_test.go
|
||||||
|
|
||||||
|
package cve
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"math"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/time/rate"
|
||||||
|
)
|
||||||
|
|
||||||
|
func fileServer(t *testing.T, path string) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
body, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read fixture %s: %v", path, err)
|
||||||
|
}
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func jsonServer(t *testing.T, body string) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write([]byte(body))
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtract(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
in []string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{[]string{"See CVE-2021-44228 for details"}, []string{"CVE-2021-44228"}},
|
||||||
|
{[]string{"cve-2021-44228 lowercase"}, []string{"CVE-2021-44228"}},
|
||||||
|
{[]string{"CVE-2021-44228 and CVE-2021-44228 again"}, []string{"CVE-2021-44228"}},
|
||||||
|
{[]string{"CVE-2026-1 too short", "CVE-2026-12345 ok"}, []string{"CVE-2026-12345"}},
|
||||||
|
{[]string{"CVE-2021-45046, CVE-2021-44228"}, []string{"CVE-2021-44228", "CVE-2021-45046"}},
|
||||||
|
{[]string{"no identifiers here"}, nil},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
if got := Extract(tc.in...); !reflect.DeepEqual(got, tc.want) {
|
||||||
|
t.Errorf("Extract(%v) = %v, want %v", tc.in, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNVDParsesLog4Shell(t *testing.T) {
|
||||||
|
srv := fileServer(t, "../../testdata/nvd/CVE-2021-44228.json")
|
||||||
|
c := NewNVDClient(srv.Client(), srv.URL, "")
|
||||||
|
|
||||||
|
res, err := c.Fetch(context.Background(), "CVE-2021-44228")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Fetch: %v", err)
|
||||||
|
}
|
||||||
|
if !res.Found {
|
||||||
|
t.Fatal("expected Found")
|
||||||
|
}
|
||||||
|
if res.CVSSScore == nil || *res.CVSSScore != 10.0 {
|
||||||
|
t.Errorf("cvss score = %v, want 10.0", res.CVSSScore)
|
||||||
|
}
|
||||||
|
if res.CVSSSeverity != "CRITICAL" {
|
||||||
|
t.Errorf("severity = %q, want CRITICAL", res.CVSSSeverity)
|
||||||
|
}
|
||||||
|
if res.CVSSVersion != "3.1" {
|
||||||
|
t.Errorf("version = %q, want 3.1", res.CVSSVersion)
|
||||||
|
}
|
||||||
|
if res.CWE != "CWE-20" {
|
||||||
|
t.Errorf("cwe = %q, want CWE-20", res.CWE)
|
||||||
|
}
|
||||||
|
if res.Description == "" {
|
||||||
|
t.Error("description empty")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNVDNotFound(t *testing.T) {
|
||||||
|
srv := jsonServer(t, `{"totalResults":0,"vulnerabilities":[]}`)
|
||||||
|
c := NewNVDClient(srv.Client(), srv.URL, "")
|
||||||
|
res, err := c.Fetch(context.Background(), "CVE-2099-99999")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Fetch: %v", err)
|
||||||
|
}
|
||||||
|
if res.Found {
|
||||||
|
t.Error("expected not found for totalResults 0")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNVDPrecedencePrefersV40(t *testing.T) {
|
||||||
|
body := `{"totalResults":1,"vulnerabilities":[{"cve":{
|
||||||
|
"id":"CVE-2026-1","descriptions":[{"lang":"en","value":"d"}],
|
||||||
|
"metrics":{
|
||||||
|
"cvssMetricV40":[{"cvssData":{"version":"4.0","baseScore":9.3,"baseSeverity":"CRITICAL","vectorString":"CVSS:4.0/x"}}],
|
||||||
|
"cvssMetricV31":[{"cvssData":{"version":"3.1","baseScore":7.5,"baseSeverity":"HIGH","vectorString":"CVSS:3.1/y"}}]
|
||||||
|
}}}]}`
|
||||||
|
srv := jsonServer(t, body)
|
||||||
|
c := NewNVDClient(srv.Client(), srv.URL, "")
|
||||||
|
res, err := c.Fetch(context.Background(), "CVE-2026-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.CVSSVersion != "4.0" || res.CVSSScore == nil || *res.CVSSScore != 9.3 {
|
||||||
|
t.Errorf("expected v4.0/9.3, got %s/%v", res.CVSSVersion, res.CVSSScore)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNVDV2SeverityFromMetricLevel(t *testing.T) {
|
||||||
|
body := `{"totalResults":1,"vulnerabilities":[{"cve":{
|
||||||
|
"id":"CVE-2005-1","descriptions":[{"lang":"en","value":"d"}],
|
||||||
|
"metrics":{"cvssMetricV2":[{"baseSeverity":"HIGH","cvssData":{"version":"2.0","baseScore":7.5,"vectorString":"AV:N"}}]}
|
||||||
|
}}]}`
|
||||||
|
srv := jsonServer(t, body)
|
||||||
|
c := NewNVDClient(srv.Client(), srv.URL, "")
|
||||||
|
res, err := c.Fetch(context.Background(), "CVE-2005-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if res.CVSSVersion != "2.0" || res.CVSSSeverity != "HIGH" {
|
||||||
|
t.Errorf("v2 severity should fall back to metric level, got %s/%q", res.CVSSVersion, res.CVSSSeverity)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestKEVParsesSample(t *testing.T) {
|
||||||
|
srv := fileServer(t, "../../testdata/kev/kev-sample.json")
|
||||||
|
c := NewKEVClient(srv.Client(), srv.URL)
|
||||||
|
catalog, err := c.Fetch(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Fetch: %v", err)
|
||||||
|
}
|
||||||
|
log4, ok := catalog.Lookup("CVE-2021-44228")
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("Log4Shell should be in KEV")
|
||||||
|
}
|
||||||
|
if !log4.Ransomware {
|
||||||
|
t.Error("Log4Shell knownRansomwareCampaignUse=Known should map to true")
|
||||||
|
}
|
||||||
|
unknown, ok := catalog.Lookup("CVE-2026-45659")
|
||||||
|
if !ok || unknown.Ransomware {
|
||||||
|
t.Error("Unknown ransomware entry should map to false")
|
||||||
|
}
|
||||||
|
if _, ok := catalog.Lookup("CVE-1999-0001"); ok {
|
||||||
|
t.Error("absent CVE should not be a KEV member")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEPSSParsesQuotedStrings(t *testing.T) {
|
||||||
|
srv := fileServer(t, "../../testdata/epss/CVE-2021-44228.json")
|
||||||
|
c := NewEPSSClient(srv.Client(), srv.URL)
|
||||||
|
scores, err := c.Fetch(context.Background(), []string{"CVE-2021-44228"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Fetch: %v", err)
|
||||||
|
}
|
||||||
|
s, ok := scores["CVE-2021-44228"]
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("expected a score for Log4Shell")
|
||||||
|
}
|
||||||
|
if math.Abs(s.EPSS-0.99999) > 1e-5 {
|
||||||
|
t.Errorf("epss = %v, want ~0.99999 (string must be ParseFloat'd, not zeroed)", s.EPSS)
|
||||||
|
}
|
||||||
|
if math.Abs(s.Percentile-1.0) > 1e-9 {
|
||||||
|
t.Errorf("percentile = %v, want 1.0", s.Percentile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestChunk(t *testing.T) {
|
||||||
|
got := chunk([]string{"a", "b", "c", "d", "e"}, 2)
|
||||||
|
want := [][]string{{"a", "b"}, {"c", "d"}, {"e"}}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("chunk = %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
if chunk(nil, 2) != nil {
|
||||||
|
t.Error("chunk(nil) should be nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNVDRetriesOn503(t *testing.T) {
|
||||||
|
var calls int
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
calls++
|
||||||
|
if calls == 1 {
|
||||||
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(`{"totalResults":0,"vulnerabilities":[]}`))
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
c := NewNVDClient(srv.Client(), srv.URL, "")
|
||||||
|
c.limiter = rate.NewLimiter(rate.Inf, 1)
|
||||||
|
c.backoffBase = time.Millisecond
|
||||||
|
if _, err := c.Fetch(context.Background(), "CVE-2026-1"); err != nil {
|
||||||
|
t.Fatalf("Fetch after retry: %v", err)
|
||||||
|
}
|
||||||
|
if calls != 2 {
|
||||||
|
t.Errorf("calls = %d, want 2 (one 503, one 200)", calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,83 @@
|
||||||
|
// ©AngelaMos | 2026
|
||||||
|
// epss.go
|
||||||
|
|
||||||
|
package cve
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"golang.org/x/time/rate"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
EPSSEndpoint = "https://api.first.org/data/v1/epss"
|
||||||
|
|
||||||
|
epssBatchSize = 100
|
||||||
|
epssRate = 5
|
||||||
|
)
|
||||||
|
|
||||||
|
type EPSSScore struct {
|
||||||
|
EPSS float64
|
||||||
|
Percentile float64
|
||||||
|
}
|
||||||
|
|
||||||
|
type EPSSClient struct {
|
||||||
|
http *http.Client
|
||||||
|
baseURL string
|
||||||
|
limiter *rate.Limiter
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewEPSSClient(client *http.Client, baseURL string) *EPSSClient {
|
||||||
|
return &EPSSClient{
|
||||||
|
http: client,
|
||||||
|
baseURL: baseURL,
|
||||||
|
limiter: rate.NewLimiter(rate.Limit(epssRate), 1),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *EPSSClient) Fetch(ctx context.Context, cveIDs []string) (map[string]EPSSScore, error) {
|
||||||
|
out := make(map[string]EPSSScore, len(cveIDs))
|
||||||
|
for _, batch := range chunk(cveIDs, epssBatchSize) {
|
||||||
|
if err := c.limiter.Wait(ctx); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
endpoint := c.baseURL + "?" + url.Values{"cve": {strings.Join(batch, ",")}}.Encode()
|
||||||
|
var raw epssRaw
|
||||||
|
if err := getJSON(ctx, c.http, endpoint, nil, &raw); err != nil {
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
for _, d := range raw.Data {
|
||||||
|
epss, err1 := strconv.ParseFloat(d.EPSS, 64)
|
||||||
|
percentile, err2 := strconv.ParseFloat(d.Percentile, 64)
|
||||||
|
if err1 != nil || err2 != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[d.CVE] = EPSSScore{EPSS: epss, Percentile: percentile}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func chunk(ids []string, size int) [][]string {
|
||||||
|
var out [][]string
|
||||||
|
for i := 0; i < len(ids); i += size {
|
||||||
|
end := i + size
|
||||||
|
if end > len(ids) {
|
||||||
|
end = len(ids)
|
||||||
|
}
|
||||||
|
out = append(out, ids[i:end])
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
type epssRaw struct {
|
||||||
|
Data []struct {
|
||||||
|
CVE string `json:"cve"`
|
||||||
|
EPSS string `json:"epss"`
|
||||||
|
Percentile string `json:"percentile"`
|
||||||
|
} `json:"data"`
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,29 @@
|
||||||
|
// ©AngelaMos | 2026
|
||||||
|
// extract.go
|
||||||
|
|
||||||
|
package cve
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
var pattern = regexp.MustCompile(`(?i)CVE-\d{4}-\d{4,7}`)
|
||||||
|
|
||||||
|
func Extract(texts ...string) []string {
|
||||||
|
seen := make(map[string]struct{})
|
||||||
|
var out []string
|
||||||
|
for _, text := range texts {
|
||||||
|
for _, match := range pattern.FindAllString(text, -1) {
|
||||||
|
id := strings.ToUpper(match)
|
||||||
|
if _, ok := seen[id]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[id] = struct{}{}
|
||||||
|
out = append(out, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,53 @@
|
||||||
|
// ©AngelaMos | 2026
|
||||||
|
// http.go
|
||||||
|
|
||||||
|
package cve
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
headerAccept = "Accept"
|
||||||
|
acceptJSON = "application/json"
|
||||||
|
maxJSONBytes = 32 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
type statusError struct {
|
||||||
|
code int
|
||||||
|
url string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *statusError) Error() string {
|
||||||
|
return fmt.Sprintf("cve: GET %s: status %d", e.url, e.code)
|
||||||
|
}
|
||||||
|
|
||||||
|
func getJSON(ctx context.Context, client *http.Client, url string, header http.Header, out any) error {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cve: build request %s: %w", url, err)
|
||||||
|
}
|
||||||
|
req.Header.Set(headerAccept, acceptJSON)
|
||||||
|
for key, values := range header {
|
||||||
|
req.Header[key] = values
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cve: GET %s: %w", url, err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxJSONBytes))
|
||||||
|
return &statusError{code: resp.StatusCode, url: url}
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(io.LimitReader(resp.Body, maxJSONBytes)).Decode(out); err != nil {
|
||||||
|
return fmt.Errorf("cve: decode %s: %w", url, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,65 @@
|
||||||
|
// ©AngelaMos | 2026
|
||||||
|
// kev.go
|
||||||
|
|
||||||
|
package cve
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
KEVEndpoint = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
|
||||||
|
|
||||||
|
kevRansomwareKnown = "Known"
|
||||||
|
)
|
||||||
|
|
||||||
|
type KEVEntry struct {
|
||||||
|
DateAdded string
|
||||||
|
Ransomware bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type KEVCatalog struct {
|
||||||
|
Version string
|
||||||
|
Released string
|
||||||
|
Entries map[string]KEVEntry
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c KEVCatalog) Lookup(cveID string) (KEVEntry, bool) {
|
||||||
|
e, ok := c.Entries[cveID]
|
||||||
|
return e, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
type KEVClient struct {
|
||||||
|
http *http.Client
|
||||||
|
baseURL string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewKEVClient(client *http.Client, baseURL string) *KEVClient {
|
||||||
|
return &KEVClient{http: client, baseURL: baseURL}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *KEVClient) Fetch(ctx context.Context) (KEVCatalog, error) {
|
||||||
|
var raw kevRaw
|
||||||
|
if err := getJSON(ctx, c.http, c.baseURL, nil, &raw); err != nil {
|
||||||
|
return KEVCatalog{}, err
|
||||||
|
}
|
||||||
|
entries := make(map[string]KEVEntry, len(raw.Vulnerabilities))
|
||||||
|
for _, v := range raw.Vulnerabilities {
|
||||||
|
entries[v.CveID] = KEVEntry{
|
||||||
|
DateAdded: v.DateAdded,
|
||||||
|
Ransomware: v.KnownRansomwareCampaignUse == kevRansomwareKnown,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return KEVCatalog{Version: raw.CatalogVersion, Released: raw.DateReleased, Entries: entries}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type kevRaw struct {
|
||||||
|
CatalogVersion string `json:"catalogVersion"`
|
||||||
|
DateReleased string `json:"dateReleased"`
|
||||||
|
Vulnerabilities []struct {
|
||||||
|
CveID string `json:"cveID"`
|
||||||
|
DateAdded string `json:"dateAdded"`
|
||||||
|
KnownRansomwareCampaignUse string `json:"knownRansomwareCampaignUse"`
|
||||||
|
} `json:"vulnerabilities"`
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,196 @@
|
||||||
|
// ©AngelaMos | 2026
|
||||||
|
// nvd.go
|
||||||
|
|
||||||
|
package cve
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/time/rate"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
NVDEndpoint = "https://services.nvd.nist.gov/rest/json/cves/2.0"
|
||||||
|
|
||||||
|
nvdAPIKeyHeader = "apiKey"
|
||||||
|
nvdRateNoKey = 5.0 / 30.0
|
||||||
|
nvdRateWithKey = 50.0 / 30.0
|
||||||
|
nvdMaxRetries = 3
|
||||||
|
nvdBackoffBase = 2 * time.Second
|
||||||
|
langEnglish = "en"
|
||||||
|
)
|
||||||
|
|
||||||
|
type NVDResult struct {
|
||||||
|
Found bool
|
||||||
|
Description string
|
||||||
|
CVSSScore *float64
|
||||||
|
CVSSVersion string
|
||||||
|
CVSSSeverity string
|
||||||
|
CVSSVector string
|
||||||
|
CWE string
|
||||||
|
Published string
|
||||||
|
Modified string
|
||||||
|
}
|
||||||
|
|
||||||
|
type NVDClient struct {
|
||||||
|
http *http.Client
|
||||||
|
baseURL string
|
||||||
|
apiKey string
|
||||||
|
limiter *rate.Limiter
|
||||||
|
backoffBase time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewNVDClient(client *http.Client, baseURL, apiKey string) *NVDClient {
|
||||||
|
limit := nvdRateNoKey
|
||||||
|
if apiKey != "" {
|
||||||
|
limit = nvdRateWithKey
|
||||||
|
}
|
||||||
|
return &NVDClient{
|
||||||
|
http: client,
|
||||||
|
baseURL: baseURL,
|
||||||
|
apiKey: apiKey,
|
||||||
|
limiter: rate.NewLimiter(rate.Limit(limit), 1),
|
||||||
|
backoffBase: nvdBackoffBase,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *NVDClient) Fetch(ctx context.Context, cveID string) (NVDResult, error) {
|
||||||
|
endpoint := c.baseURL + "?" + url.Values{"cveId": {cveID}}.Encode()
|
||||||
|
header := http.Header{}
|
||||||
|
if c.apiKey != "" {
|
||||||
|
header[nvdAPIKeyHeader] = []string{c.apiKey}
|
||||||
|
}
|
||||||
|
|
||||||
|
var env nvdEnvelope
|
||||||
|
for attempt := 0; attempt <= nvdMaxRetries; attempt++ {
|
||||||
|
if attempt > 0 {
|
||||||
|
if err := sleep(ctx, c.backoffBase*time.Duration(1<<(attempt-1))); err != nil {
|
||||||
|
return NVDResult{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := c.limiter.Wait(ctx); err != nil {
|
||||||
|
return NVDResult{}, err
|
||||||
|
}
|
||||||
|
err := getJSON(ctx, c.http, endpoint, header, &env)
|
||||||
|
if err == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if !retriable(err) || attempt == nvdMaxRetries {
|
||||||
|
return NVDResult{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if env.TotalResults == 0 || len(env.Vulnerabilities) == 0 {
|
||||||
|
return NVDResult{Found: false}, nil
|
||||||
|
}
|
||||||
|
return env.Vulnerabilities[0].CVE.toResult(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func retriable(err error) bool {
|
||||||
|
var se *statusError
|
||||||
|
if errors.As(err, &se) {
|
||||||
|
return se.code == http.StatusTooManyRequests || se.code >= http.StatusInternalServerError
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
type nvdEnvelope struct {
|
||||||
|
TotalResults int `json:"totalResults"`
|
||||||
|
Vulnerabilities []struct {
|
||||||
|
CVE nvdCVE `json:"cve"`
|
||||||
|
} `json:"vulnerabilities"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type nvdCVE struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Published string `json:"published"`
|
||||||
|
LastModified string `json:"lastModified"`
|
||||||
|
Descriptions []nvdLangValue `json:"descriptions"`
|
||||||
|
Weaknesses []struct {
|
||||||
|
Description []nvdLangValue `json:"description"`
|
||||||
|
} `json:"weaknesses"`
|
||||||
|
Metrics nvdMetrics `json:"metrics"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type nvdLangValue struct {
|
||||||
|
Lang string `json:"lang"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type nvdMetrics struct {
|
||||||
|
V40 []nvdMetric `json:"cvssMetricV40"`
|
||||||
|
V31 []nvdMetric `json:"cvssMetricV31"`
|
||||||
|
V30 []nvdMetric `json:"cvssMetricV30"`
|
||||||
|
V2 []nvdMetric `json:"cvssMetricV2"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type nvdMetric struct {
|
||||||
|
BaseSeverity string `json:"baseSeverity"`
|
||||||
|
CVSSData nvdCVSSData `json:"cvssData"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type nvdCVSSData struct {
|
||||||
|
Version string `json:"version"`
|
||||||
|
VectorString string `json:"vectorString"`
|
||||||
|
BaseScore float64 `json:"baseScore"`
|
||||||
|
BaseSeverity string `json:"baseSeverity"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v nvdCVE) toResult() NVDResult {
|
||||||
|
res := NVDResult{
|
||||||
|
Found: true,
|
||||||
|
Description: english(v.Descriptions),
|
||||||
|
Published: v.Published,
|
||||||
|
Modified: v.LastModified,
|
||||||
|
}
|
||||||
|
for _, w := range v.Weaknesses {
|
||||||
|
if cwe := english(w.Description); cwe != "" {
|
||||||
|
res.CWE = cwe
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m, ok := v.Metrics.selected(); ok {
|
||||||
|
score := m.CVSSData.BaseScore
|
||||||
|
res.CVSSScore = &score
|
||||||
|
res.CVSSVersion = m.CVSSData.Version
|
||||||
|
res.CVSSVector = m.CVSSData.VectorString
|
||||||
|
res.CVSSSeverity = m.CVSSData.BaseSeverity
|
||||||
|
if res.CVSSSeverity == "" {
|
||||||
|
res.CVSSSeverity = m.BaseSeverity
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m nvdMetrics) selected() (nvdMetric, bool) {
|
||||||
|
for _, tier := range [][]nvdMetric{m.V40, m.V31, m.V30, m.V2} {
|
||||||
|
if len(tier) > 0 {
|
||||||
|
return tier[0], true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nvdMetric{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func english(values []nvdLangValue) string {
|
||||||
|
for _, lv := range values {
|
||||||
|
if lv.Lang == langEnglish {
|
||||||
|
return lv.Value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func sleep(ctx context.Context, d time.Duration) error {
|
||||||
|
timer := time.NewTimer(d)
|
||||||
|
defer timer.Stop()
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-timer.C:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,105 @@
|
||||||
|
// ©AngelaMos | 2026
|
||||||
|
// enrich.go
|
||||||
|
|
||||||
|
package enrich
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/CarterPerez-dev/nadezhda/internal/cve"
|
||||||
|
"github.com/CarterPerez-dev/nadezhda/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
const secondsPerHour = 3600
|
||||||
|
|
||||||
|
type Clients struct {
|
||||||
|
NVD *cve.NVDClient
|
||||||
|
KEV *cve.KEVClient
|
||||||
|
EPSS *cve.EPSSClient
|
||||||
|
}
|
||||||
|
|
||||||
|
type Stats struct {
|
||||||
|
Total int
|
||||||
|
Enriched int
|
||||||
|
NotFound int
|
||||||
|
KEVHits int
|
||||||
|
Errors int
|
||||||
|
}
|
||||||
|
|
||||||
|
func Run(ctx context.Context, st *store.Store, clients Clients, now time.Time, positiveTTLHours, negativeTTLHours int) (Stats, error) {
|
||||||
|
positiveTTL := int64(positiveTTLHours) * secondsPerHour
|
||||||
|
negativeTTL := int64(negativeTTLHours) * secondsPerHour
|
||||||
|
|
||||||
|
ids, err := st.CVEsNeedingEnrichment(now.Unix(), positiveTTL, negativeTTL)
|
||||||
|
if err != nil {
|
||||||
|
return Stats{}, err
|
||||||
|
}
|
||||||
|
if len(ids) == 0 {
|
||||||
|
return Stats{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
catalog, err := clients.KEV.Fetch(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return Stats{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
epssScores, _ := clients.EPSS.Fetch(ctx, ids)
|
||||||
|
if epssScores == nil {
|
||||||
|
epssScores = map[string]cve.EPSSScore{}
|
||||||
|
}
|
||||||
|
|
||||||
|
stats := Stats{Total: len(ids)}
|
||||||
|
for _, id := range ids {
|
||||||
|
nvdRes, err := clients.NVD.Fetch(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return stats, ctx.Err()
|
||||||
|
}
|
||||||
|
stats.Errors++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := store.CVE{ID: id, EnrichedAt: now.Unix()}
|
||||||
|
if nvdRes.Found {
|
||||||
|
rec.EnrichStatus = store.EnrichStatusOK
|
||||||
|
rec.Description = nvdRes.Description
|
||||||
|
rec.CVSSScore = nvdRes.CVSSScore
|
||||||
|
rec.CVSSVersion = nvdRes.CVSSVersion
|
||||||
|
rec.CVSSSeverity = nvdRes.CVSSSeverity
|
||||||
|
rec.CVSSVector = nvdRes.CVSSVector
|
||||||
|
rec.CWE = nvdRes.CWE
|
||||||
|
rec.NVDPublished = nvdRes.Published
|
||||||
|
rec.NVDModified = nvdRes.Modified
|
||||||
|
} else {
|
||||||
|
rec.EnrichStatus = store.EnrichStatusNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
if entry, ok := catalog.Lookup(id); ok {
|
||||||
|
rec.IsKEV = true
|
||||||
|
rec.KEVDateAdded = entry.DateAdded
|
||||||
|
rec.KEVRansomware = entry.Ransomware
|
||||||
|
}
|
||||||
|
|
||||||
|
if score, ok := epssScores[id]; ok {
|
||||||
|
epssVal := score.EPSS
|
||||||
|
percentileVal := score.Percentile
|
||||||
|
rec.EPSS = &epssVal
|
||||||
|
rec.EPSSPercentile = &percentileVal
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := st.UpdateCVEEnrichment(rec); err != nil {
|
||||||
|
stats.Errors++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if nvdRes.Found {
|
||||||
|
stats.Enriched++
|
||||||
|
} else {
|
||||||
|
stats.NotFound++
|
||||||
|
}
|
||||||
|
if rec.IsKEV {
|
||||||
|
stats.KEVHits++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return stats, nil
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,121 @@
|
||||||
|
// ©AngelaMos | 2026
|
||||||
|
// enrich_test.go
|
||||||
|
|
||||||
|
package enrich
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/CarterPerez-dev/nadezhda/internal/cve"
|
||||||
|
"github.com/CarterPerez-dev/nadezhda/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func fileServer(t *testing.T, path string) *httptest.Server {
|
||||||
|
t.Helper()
|
||||||
|
body, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %s: %v", path, err)
|
||||||
|
}
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStore(t *testing.T) *store.Store {
|
||||||
|
t.Helper()
|
||||||
|
st, err := store.Open(filepath.Join(t.TempDir(), "enrich.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("open store: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = st.Close() })
|
||||||
|
return st
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunEnrichesLog4Shell(t *testing.T) {
|
||||||
|
nvd := fileServer(t, "../../testdata/nvd/CVE-2021-44228.json")
|
||||||
|
kev := fileServer(t, "../../testdata/kev/kev-sample.json")
|
||||||
|
epss := fileServer(t, "../../testdata/epss/CVE-2021-44228.json")
|
||||||
|
|
||||||
|
st := newStore(t)
|
||||||
|
if err := st.UpsertCVEStub("CVE-2021-44228"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
clients := Clients{
|
||||||
|
NVD: cve.NewNVDClient(nvd.Client(), nvd.URL, ""),
|
||||||
|
KEV: cve.NewKEVClient(kev.Client(), kev.URL),
|
||||||
|
EPSS: cve.NewEPSSClient(epss.Client(), epss.URL),
|
||||||
|
}
|
||||||
|
|
||||||
|
stats, err := Run(context.Background(), st, clients, time.Unix(1_800_000_000, 0), 24, 3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run: %v", err)
|
||||||
|
}
|
||||||
|
if stats.Enriched != 1 || stats.KEVHits != 1 {
|
||||||
|
t.Fatalf("stats = %+v, want Enriched:1 KEVHits:1", stats)
|
||||||
|
}
|
||||||
|
|
||||||
|
c, err := st.GetCVE("CVE-2021-44228")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if c.CVSSScore == nil || *c.CVSSScore != 10.0 || c.CVSSSeverity != "CRITICAL" {
|
||||||
|
t.Errorf("cvss = %v/%q, want 10.0/CRITICAL", c.CVSSScore, c.CVSSSeverity)
|
||||||
|
}
|
||||||
|
if !c.IsKEV || !c.KEVRansomware {
|
||||||
|
t.Errorf("expected KEV + ransomware, got is_kev=%v ransomware=%v", c.IsKEV, c.KEVRansomware)
|
||||||
|
}
|
||||||
|
if c.EPSS == nil || *c.EPSS < 0.9 {
|
||||||
|
t.Errorf("epss = %v, want ~0.99999", c.EPSS)
|
||||||
|
}
|
||||||
|
if c.EnrichStatus != store.EnrichStatusOK {
|
||||||
|
t.Errorf("status = %q, want ok", c.EnrichStatus)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunSkipsFreshAndMarksNotFound(t *testing.T) {
|
||||||
|
nvd := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write([]byte(`{"totalResults":0,"vulnerabilities":[]}`))
|
||||||
|
}))
|
||||||
|
t.Cleanup(nvd.Close)
|
||||||
|
kev := fileServer(t, "../../testdata/kev/kev-sample.json")
|
||||||
|
epss := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
_, _ = w.Write([]byte(`{"data":[]}`))
|
||||||
|
}))
|
||||||
|
t.Cleanup(epss.Close)
|
||||||
|
|
||||||
|
st := newStore(t)
|
||||||
|
if err := st.UpsertCVEStub("CVE-2099-99999"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
clients := Clients{
|
||||||
|
NVD: cve.NewNVDClient(nvd.Client(), nvd.URL, ""),
|
||||||
|
KEV: cve.NewKEVClient(kev.Client(), kev.URL),
|
||||||
|
EPSS: cve.NewEPSSClient(epss.Client(), epss.URL),
|
||||||
|
}
|
||||||
|
now := time.Unix(1_800_000_000, 0)
|
||||||
|
|
||||||
|
stats, err := Run(context.Background(), st, clients, now, 24, 3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Run: %v", err)
|
||||||
|
}
|
||||||
|
if stats.NotFound != 1 {
|
||||||
|
t.Errorf("NotFound = %d, want 1", stats.NotFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
fresh, err := Run(context.Background(), st, clients, now, 24, 3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second Run: %v", err)
|
||||||
|
}
|
||||||
|
if fresh.Total != 0 {
|
||||||
|
t.Errorf("second run Total = %d, want 0 (not_found within negative TTL is skipped)", fresh.Total)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -12,6 +12,7 @@ import (
|
||||||
"golang.org/x/sync/errgroup"
|
"golang.org/x/sync/errgroup"
|
||||||
|
|
||||||
"github.com/CarterPerez-dev/nadezhda/internal/config"
|
"github.com/CarterPerez-dev/nadezhda/internal/config"
|
||||||
|
"github.com/CarterPerez-dev/nadezhda/internal/cve"
|
||||||
"github.com/CarterPerez-dev/nadezhda/internal/fetch"
|
"github.com/CarterPerez-dev/nadezhda/internal/fetch"
|
||||||
"github.com/CarterPerez-dev/nadezhda/internal/normalize"
|
"github.com/CarterPerez-dev/nadezhda/internal/normalize"
|
||||||
"github.com/CarterPerez-dev/nadezhda/internal/parse"
|
"github.com/CarterPerez-dev/nadezhda/internal/parse"
|
||||||
|
|
@ -24,6 +25,7 @@ type SourceResult struct {
|
||||||
Parsed int
|
Parsed int
|
||||||
New int
|
New int
|
||||||
Duplicates int
|
Duplicates int
|
||||||
|
CVEs int
|
||||||
ItemErrors int
|
ItemErrors int
|
||||||
NotModified bool
|
NotModified bool
|
||||||
Err error
|
Err error
|
||||||
|
|
@ -134,19 +136,22 @@ func storeItem(st *store.Store, cfg config.Config, sourceID int64, it parse.Item
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
summary := normalize.StripHTML(it.Summary)
|
||||||
|
body := normalize.StripHTML(it.Body)
|
||||||
|
|
||||||
var publishedAt int64
|
var publishedAt int64
|
||||||
if !it.Published.IsZero() {
|
if !it.Published.IsZero() {
|
||||||
publishedAt = it.Published.Unix()
|
publishedAt = it.Published.Unix()
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = st.InsertArticle(store.Article{
|
id, err := st.InsertArticle(store.Article{
|
||||||
SourceID: sourceID,
|
SourceID: sourceID,
|
||||||
CanonicalURL: canonical,
|
CanonicalURL: canonical,
|
||||||
ContentHash: normalize.ContentHash(canonical),
|
ContentHash: normalize.ContentHash(canonical),
|
||||||
TitleHash: normalize.TitleHash(normalize.NormalizeTitle(it.Title)),
|
TitleHash: normalize.TitleHash(normalize.NormalizeTitle(it.Title)),
|
||||||
Title: it.Title,
|
Title: it.Title,
|
||||||
Summary: normalize.StripHTML(it.Summary),
|
Summary: summary,
|
||||||
Body: normalize.StripHTML(it.Body),
|
Body: body,
|
||||||
Author: it.Author,
|
Author: it.Author,
|
||||||
PublishedAt: publishedAt,
|
PublishedAt: publishedAt,
|
||||||
FetchedAt: now.Unix(),
|
FetchedAt: now.Unix(),
|
||||||
|
|
@ -154,9 +159,24 @@ func storeItem(st *store.Store, cfg config.Config, sourceID int64, it parse.Item
|
||||||
switch {
|
switch {
|
||||||
case err == nil:
|
case err == nil:
|
||||||
out.New++
|
out.New++
|
||||||
|
linkCVEs(st, id, cve.Extract(it.Title, summary, body), out)
|
||||||
case errors.Is(err, store.ErrDuplicate):
|
case errors.Is(err, store.ErrDuplicate):
|
||||||
out.Duplicates++
|
out.Duplicates++
|
||||||
default:
|
default:
|
||||||
out.ItemErrors++
|
out.ItemErrors++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func linkCVEs(st *store.Store, articleID int64, ids []string, out *SourceResult) {
|
||||||
|
for _, id := range ids {
|
||||||
|
if err := st.UpsertCVEStub(id); err != nil {
|
||||||
|
out.ItemErrors++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := st.LinkArticleCVE(articleID, id); err != nil {
|
||||||
|
out.ItemErrors++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out.CVEs++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,106 @@
|
||||||
|
// ©AngelaMos | 2026
|
||||||
|
// cve_test.go
|
||||||
|
|
||||||
|
package store
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func seedArticleWithCVE(t *testing.T, s *Store, cveID string, cvss float64, kev bool) int64 {
|
||||||
|
t.Helper()
|
||||||
|
sourceID, err := s.UpsertSource(SourceInput{
|
||||||
|
Name: "src", URL: "https://ex.example/feed", Type: "rss", Weight: 1, Enabled: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
articleID, err := s.InsertArticle(Article{
|
||||||
|
SourceID: sourceID, CanonicalURL: "https://ex.example/a", ContentHash: "ch", TitleHash: "th",
|
||||||
|
Title: "Exploit in the wild", Summary: "details", PublishedAt: 1000, FetchedAt: 1000,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := s.UpsertCVEStub(cveID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := s.LinkArticleCVE(articleID, cveID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
score := cvss
|
||||||
|
if err := s.UpdateCVEEnrichment(CVE{
|
||||||
|
ID: cveID, CVSSScore: &score, CVSSSeverity: "CRITICAL", IsKEV: kev,
|
||||||
|
EnrichedAt: 2000, EnrichStatus: EnrichStatusOK,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return articleID
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArticlesForCVE(t *testing.T) {
|
||||||
|
s := openTemp(t)
|
||||||
|
seedArticleWithCVE(t, s, "CVE-2026-1", 9.8, true)
|
||||||
|
got, err := s.ArticlesForCVE("CVE-2026-1")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 1 || got[0].Title != "Exploit in the wild" {
|
||||||
|
t.Errorf("ArticlesForCVE = %+v, want one article", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLinkArticleCVEIsIdempotent(t *testing.T) {
|
||||||
|
s := openTemp(t)
|
||||||
|
id := seedArticleWithCVE(t, s, "CVE-2026-1", 5.0, false)
|
||||||
|
if err := s.LinkArticleCVE(id, "CVE-2026-1"); err != nil {
|
||||||
|
t.Fatalf("re-link should be a no-op: %v", err)
|
||||||
|
}
|
||||||
|
got, _ := s.ArticlesForCVE("CVE-2026-1")
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Errorf("duplicate link produced %d rows, want 1", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListArticlesFilters(t *testing.T) {
|
||||||
|
s := openTemp(t)
|
||||||
|
seedArticleWithCVE(t, s, "CVE-2026-1", 9.8, true)
|
||||||
|
|
||||||
|
if got, _ := s.ListArticles(ListFilter{KEV: true}); len(got) != 1 {
|
||||||
|
t.Errorf("--kev returned %d, want 1", len(got))
|
||||||
|
}
|
||||||
|
if got, _ := s.ListArticles(ListFilter{MinCVSS: 9.0}); len(got) != 1 {
|
||||||
|
t.Errorf("--min-cvss 9 returned %d, want 1", len(got))
|
||||||
|
}
|
||||||
|
if got, _ := s.ListArticles(ListFilter{MinCVSS: 10.0}); len(got) != 0 {
|
||||||
|
t.Errorf("--min-cvss 10 returned %d, want 0", len(got))
|
||||||
|
}
|
||||||
|
if got, _ := s.ListArticles(ListFilter{Keyword: "exploit"}); len(got) != 1 {
|
||||||
|
t.Errorf("--keyword returned %d, want 1", len(got))
|
||||||
|
}
|
||||||
|
if got, _ := s.ListArticles(ListFilter{Source: "nope"}); len(got) != 0 {
|
||||||
|
t.Errorf("unknown source returned %d, want 0", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCVEsNeedingEnrichment(t *testing.T) {
|
||||||
|
s := openTemp(t)
|
||||||
|
if err := s.UpsertCVEStub("CVE-2026-1"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
need, err := s.CVEsNeedingEnrichment(10000, 3600, 1800)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(need) != 1 {
|
||||||
|
t.Fatalf("fresh stub should need enrichment, got %d", len(need))
|
||||||
|
}
|
||||||
|
score := 5.0
|
||||||
|
if err := s.UpdateCVEEnrichment(CVE{ID: "CVE-2026-1", CVSSScore: &score, EnrichedAt: 9000, EnrichStatus: EnrichStatusOK}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if need, _ := s.CVEsNeedingEnrichment(10000, 3600, 1800); len(need) != 0 {
|
||||||
|
t.Errorf("recently enriched cve should be skipped, got %d", len(need))
|
||||||
|
}
|
||||||
|
if need, _ := s.CVEsNeedingEnrichment(99999, 3600, 1800); len(need) != 1 {
|
||||||
|
t.Errorf("stale cve (past TTL) should need re-enrichment, got %d", len(need))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -15,6 +15,11 @@ import (
|
||||||
|
|
||||||
var ErrDuplicate = errors.New("store: article already exists")
|
var ErrDuplicate = errors.New("store: article already exists")
|
||||||
|
|
||||||
|
const (
|
||||||
|
EnrichStatusOK = "ok"
|
||||||
|
EnrichStatusNotFound = "not_found"
|
||||||
|
)
|
||||||
|
|
||||||
type Store struct {
|
type Store struct {
|
||||||
db *sql.DB
|
db *sql.DB
|
||||||
version int
|
version int
|
||||||
|
|
@ -179,6 +184,135 @@ func (s *Store) UpsertFetchState(sourceID int64, fs FetchState) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type CVE struct {
|
||||||
|
ID string
|
||||||
|
Description string
|
||||||
|
CVSSScore *float64
|
||||||
|
CVSSVersion string
|
||||||
|
CVSSSeverity string
|
||||||
|
CVSSVector string
|
||||||
|
CWE string
|
||||||
|
IsKEV bool
|
||||||
|
KEVDateAdded string
|
||||||
|
KEVRansomware bool
|
||||||
|
EPSS *float64
|
||||||
|
EPSSPercentile *float64
|
||||||
|
NVDPublished string
|
||||||
|
NVDModified string
|
||||||
|
EnrichedAt int64
|
||||||
|
EnrichStatus string
|
||||||
|
}
|
||||||
|
|
||||||
|
type ArticleSummary struct {
|
||||||
|
ID int64
|
||||||
|
SourceName string
|
||||||
|
Title string
|
||||||
|
CanonicalURL string
|
||||||
|
PublishedAt int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type ListFilter struct {
|
||||||
|
Source string
|
||||||
|
Since int64
|
||||||
|
MinCVSS float64
|
||||||
|
KEV bool
|
||||||
|
Keyword string
|
||||||
|
Limit int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) CVEsNeedingEnrichment(now, positiveTTL, negativeTTL int64) ([]string, error) {
|
||||||
|
rows, err := s.db.Query(`
|
||||||
|
SELECT id FROM cves
|
||||||
|
WHERE enriched_at = 0
|
||||||
|
OR (enrich_status = ? AND enriched_at < ?)
|
||||||
|
OR (enrich_status = ? AND enriched_at < ?)
|
||||||
|
ORDER BY id`, EnrichStatusOK, now-positiveTTL, EnrichStatusNotFound, now-negativeTTL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cves needing enrichment: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var out []string
|
||||||
|
for rows.Next() {
|
||||||
|
var id string
|
||||||
|
if err := rows.Scan(&id); err != nil {
|
||||||
|
return nil, fmt.Errorf("cves needing enrichment: scan: %w", err)
|
||||||
|
}
|
||||||
|
out = append(out, id)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) UpdateCVEEnrichment(c CVE) error {
|
||||||
|
_, err := s.db.Exec(`
|
||||||
|
UPDATE cves SET
|
||||||
|
description = ?, cvss_score = ?, cvss_version = ?, cvss_severity = ?, cvss_vector = ?,
|
||||||
|
cwe = ?, is_kev = ?, kev_date_added = ?, kev_ransomware = ?,
|
||||||
|
epss = ?, epss_percentile = ?, nvd_published = ?, nvd_modified = ?,
|
||||||
|
enriched_at = ?, enrich_status = ?
|
||||||
|
WHERE id = ?`,
|
||||||
|
c.Description, c.CVSSScore, c.CVSSVersion, c.CVSSSeverity, c.CVSSVector,
|
||||||
|
c.CWE, boolToInt(c.IsKEV), c.KEVDateAdded, boolToInt(c.KEVRansomware),
|
||||||
|
c.EPSS, c.EPSSPercentile, c.NVDPublished, c.NVDModified,
|
||||||
|
c.EnrichedAt, c.EnrichStatus, c.ID,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("update cve enrichment %q: %w", c.ID, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) GetCVE(id string) (CVE, error) {
|
||||||
|
var c CVE
|
||||||
|
var isKEV, ransomware int
|
||||||
|
err := s.db.QueryRow(`
|
||||||
|
SELECT id, description, cvss_score, cvss_version, cvss_severity, cvss_vector,
|
||||||
|
cwe, is_kev, kev_date_added, kev_ransomware, epss, epss_percentile,
|
||||||
|
nvd_published, nvd_modified, enriched_at, enrich_status
|
||||||
|
FROM cves WHERE id = ?`, id,
|
||||||
|
).Scan(&c.ID, &c.Description, &c.CVSSScore, &c.CVSSVersion, &c.CVSSSeverity, &c.CVSSVector,
|
||||||
|
&c.CWE, &isKEV, &c.KEVDateAdded, &ransomware, &c.EPSS, &c.EPSSPercentile,
|
||||||
|
&c.NVDPublished, &c.NVDModified, &c.EnrichedAt, &c.EnrichStatus)
|
||||||
|
if err != nil {
|
||||||
|
return CVE{}, fmt.Errorf("get cve %q: %w", id, err)
|
||||||
|
}
|
||||||
|
c.IsKEV = isKEV != 0
|
||||||
|
c.KEVRansomware = ransomware != 0
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) ArticlesForCVE(id string) ([]ArticleSummary, error) {
|
||||||
|
rows, err := s.db.Query(`
|
||||||
|
SELECT a.id, s.name, a.title, a.canonical_url, a.published_at
|
||||||
|
FROM article_cves ac
|
||||||
|
JOIN articles a ON a.id = ac.article_id
|
||||||
|
JOIN sources s ON s.id = a.source_id
|
||||||
|
WHERE ac.cve_id = ?
|
||||||
|
ORDER BY a.published_at DESC`, id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("articles for cve %q: %w", id, err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
return scanArticleSummaries(rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) UpsertCVEStub(id string) error {
|
||||||
|
_, err := s.db.Exec(`INSERT INTO cves (id) VALUES (?) ON CONFLICT(id) DO NOTHING`, id)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("upsert cve %q: %w", id, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) LinkArticleCVE(articleID int64, cveID string) error {
|
||||||
|
_, err := s.db.Exec(`
|
||||||
|
INSERT INTO article_cves (article_id, cve_id) VALUES (?, ?)
|
||||||
|
ON CONFLICT(article_id, cve_id) DO NOTHING`, articleID, cveID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("link article %d cve %q: %w", articleID, cveID, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
type CandidateArticle struct {
|
type CandidateArticle struct {
|
||||||
ID int64
|
ID int64
|
||||||
SourceID int64
|
SourceID int64
|
||||||
|
|
@ -273,6 +407,64 @@ func (s *Store) ReplaceClusters(rows []ClusterRow) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Store) ListArticles(f ListFilter) ([]ArticleSummary, error) {
|
||||||
|
query := `
|
||||||
|
SELECT a.id, s.name, a.title, a.canonical_url, a.published_at
|
||||||
|
FROM articles a
|
||||||
|
JOIN sources s ON s.id = a.source_id
|
||||||
|
WHERE 1 = 1`
|
||||||
|
var args []any
|
||||||
|
|
||||||
|
if f.Source != "" {
|
||||||
|
query += ` AND s.name = ?`
|
||||||
|
args = append(args, f.Source)
|
||||||
|
}
|
||||||
|
if f.Since > 0 {
|
||||||
|
query += ` AND a.published_at >= ?`
|
||||||
|
args = append(args, f.Since)
|
||||||
|
}
|
||||||
|
if f.Keyword != "" {
|
||||||
|
query += ` AND (a.title LIKE ? OR a.summary LIKE ?)`
|
||||||
|
like := "%" + f.Keyword + "%"
|
||||||
|
args = append(args, like, like)
|
||||||
|
}
|
||||||
|
if f.MinCVSS > 0 {
|
||||||
|
query += ` AND EXISTS (
|
||||||
|
SELECT 1 FROM article_cves ac JOIN cves c ON c.id = ac.cve_id
|
||||||
|
WHERE ac.article_id = a.id AND c.cvss_score >= ?)`
|
||||||
|
args = append(args, f.MinCVSS)
|
||||||
|
}
|
||||||
|
if f.KEV {
|
||||||
|
query += ` AND EXISTS (
|
||||||
|
SELECT 1 FROM article_cves ac JOIN cves c ON c.id = ac.cve_id
|
||||||
|
WHERE ac.article_id = a.id AND c.is_kev = 1)`
|
||||||
|
}
|
||||||
|
query += ` ORDER BY a.published_at DESC`
|
||||||
|
if f.Limit > 0 {
|
||||||
|
query += ` LIMIT ?`
|
||||||
|
args = append(args, f.Limit)
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := s.db.Query(query, args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("list articles: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
return scanArticleSummaries(rows)
|
||||||
|
}
|
||||||
|
|
||||||
|
func scanArticleSummaries(rows *sql.Rows) ([]ArticleSummary, error) {
|
||||||
|
var out []ArticleSummary
|
||||||
|
for rows.Next() {
|
||||||
|
var a ArticleSummary
|
||||||
|
if err := rows.Scan(&a.ID, &a.SourceName, &a.Title, &a.CanonicalURL, &a.PublishedAt); err != nil {
|
||||||
|
return nil, fmt.Errorf("scan article summary: %w", err)
|
||||||
|
}
|
||||||
|
out = append(out, a)
|
||||||
|
}
|
||||||
|
return out, rows.Err()
|
||||||
|
}
|
||||||
|
|
||||||
func boolToInt(b bool) int {
|
func boolToInt(b bool) int {
|
||||||
if b {
|
if b {
|
||||||
return 1
|
return 1
|
||||||
|
|
|
||||||
1
PROJECTS/intermediate/security-news-scraper/testdata/epss/CVE-2021-44228.json
vendored
Normal file
1
PROJECTS/intermediate/security-news-scraper/testdata/epss/CVE-2021-44228.json
vendored
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
{"status":"OK","status-code":200,"version":"1.0","access":"public","total":1,"offset":0,"limit":100,"data":[{"cve":"CVE-2021-44228","epss":"0.999990000","percentile":"1.000000000","date":"2026-07-05"}]}
|
||||||
|
|
@ -0,0 +1,85 @@
|
||||||
|
{
|
||||||
|
"title": "CISA Catalog of Known Exploited Vulnerabilities",
|
||||||
|
"catalogVersion": "2026.07.01",
|
||||||
|
"dateReleased": "2026-07-01T19:00:06.9016Z",
|
||||||
|
"count": 5,
|
||||||
|
"vulnerabilities": [
|
||||||
|
{
|
||||||
|
"cveID": "CVE-2021-44228",
|
||||||
|
"vendorProject": "Apache",
|
||||||
|
"product": "Log4j2",
|
||||||
|
"vulnerabilityName": "Apache Log4j2 Remote Code Execution Vulnerability",
|
||||||
|
"dateAdded": "2021-12-10",
|
||||||
|
"shortDescription": "Apache Log4j2 contains a vulnerability where JNDI features do not protect against attacker-controlled JNDI-related endpoints, allowing for remote code execution.",
|
||||||
|
"requiredAction": "For all affected software assets for which updates exist, the only acceptable remediation actions are: 1) Apply updates; OR 2) remove affected assets from agency networks. Temporary mitigations using one of the measures provided at https://www.cisa.gov/uscert/ed-22-02-apache-log4j-recommended-mitigation-measures are only acceptable until updates are available.",
|
||||||
|
"dueDate": "2021-12-24",
|
||||||
|
"knownRansomwareCampaignUse": "Known",
|
||||||
|
"notes": "https://nvd.nist.gov/vuln/detail/CVE-2021-44228",
|
||||||
|
"cwes": [
|
||||||
|
"CWE-20",
|
||||||
|
"CWE-400",
|
||||||
|
"CWE-502"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cveID": "CVE-2026-45659",
|
||||||
|
"vendorProject": "Microsoft",
|
||||||
|
"product": "SharePoint Server",
|
||||||
|
"vulnerabilityName": "Microsoft SharePoint Server Deserialization of Untrusted Data Vulnerability",
|
||||||
|
"dateAdded": "2026-07-01",
|
||||||
|
"shortDescription": "Microsoft SharePoint Server contains a deserialization of untrusted data vulnerability which allows an authorized attacker to execute code over a network.",
|
||||||
|
"requiredAction": "Apply mitigations in accordance with vendor instructions, ensuring compliance with CISA\u2019s BOD 26-04 Prioritizing Security Updates Based on Risk (see URL in Notes) guidance and CISA\u2019s \u201cForensics Triage Requirements\u201d (see URL in Notes). Follow applicable BOD 26-04 guidance for cloud services or discontinue use of the product if mitigations are unavailable. Stakeholders are responsible for evaluating each asset's internet exposure and ensuring adherence to BOD 26-04 patching guidelines.",
|
||||||
|
"dueDate": "2026-07-04",
|
||||||
|
"knownRansomwareCampaignUse": "Unknown",
|
||||||
|
"notes": "https://msrc.microsoft.com/update-guide/vulnerability/CVE-2026-45659 ; BOD 26-04: https://www.cisa.gov/news-events/directives/bod-26-04-prioritizing-security-updates-based-risk ; Forensics Triage Requirements: https://www.cisa.gov/news-events/directives/bod-26-04-implementation-guidance-prioritizing-security-updates-based-risk ; https://nvd.nist.gov/vuln/detail/CVE-2026-45659",
|
||||||
|
"cwes": [
|
||||||
|
"CWE-502"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cveID": "CVE-2026-48558",
|
||||||
|
"vendorProject": "SimpleHelp ",
|
||||||
|
"product": "SimpleHelp",
|
||||||
|
"vulnerabilityName": "SimpleHelp Authentication Bypass Vulnerability",
|
||||||
|
"dateAdded": "2026-06-29",
|
||||||
|
"shortDescription": "SimpleHelp contains an authentication bypass vulnerability in the OIDC authentication flow. When OIDC authentication is configured, identity tokens submitted during login are accepted without verifying their cryptographic signature. In a vulnerable configuration, a remote, unauthenticated attacker can submit a forged token containing arbitrary identity claims to obtain a fully authenticated technician session. In some configurations, this may also allow bypass of multi-factor authentication.",
|
||||||
|
"requiredAction": "Apply mitigations in accordance with vendor instructions, ensuring compliance with CISA\u2019s BOD 26-04 Prioritizing Security Updates Based on Risk (see URL in Notes) guidance and CISA\u2019s \u201cForensics Triage Requirements\u201d (see URL in Notes). Follow applicable BOD 26-04 guidance for cloud services or discontinue use of the product if mitigations are unavailable. Stakeholders are responsible for evaluating each asset's internet exposure and ensuring adherence to BOD 26-04 patching guidelines.",
|
||||||
|
"dueDate": "2026-07-02",
|
||||||
|
"knownRansomwareCampaignUse": "Unknown",
|
||||||
|
"notes": "https://simple-help.com/security/simplehelp-security-update-2026-05 ; BOD 26-04: https://www.cisa.gov/news-events/directives/bod-26-04-prioritizing-security-updates-based-risk ; Forensics Triage Requirements: https://www.cisa.gov/news-events/directives/bod-26-04-implementation-guidance-prioritizing-security-updates-based-risk ; https://nvd.nist.gov/vuln/detail/CVE-2026-48558",
|
||||||
|
"cwes": [
|
||||||
|
"CWE-347"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cveID": "CVE-2026-35273",
|
||||||
|
"vendorProject": "Oracle",
|
||||||
|
"product": " PeopleSoft Enterprise PeopleTools",
|
||||||
|
"vulnerabilityName": "Oracle PeopleSoft Enterprise PeopleTools Missing Authentication for Critical Function Vulnerability",
|
||||||
|
"dateAdded": "2026-06-12",
|
||||||
|
"shortDescription": "Oracle PeopleSoft Enterprise PeopleTools contains a missing authentication for critical function vulnerability which could allow an unauthenticated attacker to obtain takeover of PeopleSoft Enterprise PeopleTools.",
|
||||||
|
"requiredAction": "Apply mitigations in accordance with vendor instructions, ensuring compliance with CISA\u2019s BOD 26-04 Prioritizing Security Updates Based on Risk (see URL in Notes) guidance and CISA\u2019s \u201cForensics Triage Requirements\u201d (see URL in Notes). Follow applicable BOD 26-04 guidance for cloud services or discontinue use of the product if mitigations are unavailable. Stakeholders are responsible for evaluating each asset's internet exposure and ensuring adherence to BOD 26-04 patching guidelines.",
|
||||||
|
"dueDate": "2026-06-15",
|
||||||
|
"knownRansomwareCampaignUse": "Known",
|
||||||
|
"notes": "https://www.oracle.com/security-alerts/alert-cve-2026-35273.html ; https://support.oracle.com/signin/ ; BOD 26-04: https://www.cisa.gov/news-events/directives/bod-26-04-prioritizing-security-updates-based-risk ; Forensics Triage Requirements: https://www.cisa.gov/news-events/directives/bod-26-04-implementation-guidance-prioritizing-security-updates-based-risk ; https://nvd.nist.gov/vuln/detail/CVE-2026-35273",
|
||||||
|
"cwes": [
|
||||||
|
"CWE-306"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cveID": "CVE-2026-50751",
|
||||||
|
"vendorProject": "Check Point",
|
||||||
|
"product": "Security Gateway",
|
||||||
|
"vulnerabilityName": "Check Point Security Gateway Improper Authentication Vulnerability",
|
||||||
|
"dateAdded": "2026-06-08",
|
||||||
|
"shortDescription": "Check Point Security Gateway contains an improper authentication vulnerability in IKEv1 key exchange that could allow an unauthenticated remote attacker to bypass user authentication and establish a remote access VPN connection without a valid user password.",
|
||||||
|
"requiredAction": "Apply mitigations per vendor instructions, follow applicable BOD 22-01 guidance for cloud services, or discontinue use of the product if mitigations are unavailable.",
|
||||||
|
"dueDate": "2026-06-11",
|
||||||
|
"knownRansomwareCampaignUse": "Known",
|
||||||
|
"notes": "https://blog.checkpoint.com/security/check-point-releases-important-hotfix-for-vulnerabilities-in-deprecated-ikev1-vpn-protocol/ ; https://support.checkpoint.com/results/sk/sk185033?_gl=1*1wqeqhc*_gcl_au*MTI1MzE5MjI2LjE3ODA5MzQ1NTM. ; https://nvd.nist.gov/vuln/detail/CVE-2026-50751",
|
||||||
|
"cwes": [
|
||||||
|
"CWE-287"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue