feat(nadezhda): M0 scaffold for security news + CVE aggregator
Go engine that aggregates cybersecurity news and enriches CVEs. This first commit lays the M0 foundation: - config: fetch/enrich/cluster/rank/AI settings, YAML + validation, defaults as named constants, AI opt-in and off by default - source: Source registry with embedded default feeds (7 verified RSS sources), external-file override, dedup/URL validation - store: pure-Go SQLite (modernc), WAL + foreign_keys, forward-only embedded migrations with schema_migrations tracking and a loud newer-than-binary guard; ErrDuplicate sentinel for dedup - cmd: cobra skeleton, version + sources wired, remaining commands stubbed to their milestones go vet / gofmt / go test all clean.
This commit is contained in:
CarterPerez-dev
parent
6593aa5689
commit
8b8eaafa1f
|
|
@ -0,0 +1,24 @@
|
|||
# ©AngelaMos | 2026
|
||||
# .gitignore
|
||||
|
||||
# private dev + research material (never public)
|
||||
docs/
|
||||
|
||||
# Go build artifacts
|
||||
/nadezhda
|
||||
/bin/
|
||||
*.test
|
||||
*.out
|
||||
*.prof
|
||||
|
||||
# local runtime data (SQLite store + WAL sidecars)
|
||||
*.db
|
||||
*.db-wal
|
||||
*.db-shm
|
||||
*.sqlite
|
||||
/data/
|
||||
|
||||
# env / secrets / local overrides
|
||||
.env
|
||||
*.local.yaml
|
||||
config.local.yaml
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
# Nadezhda
|
||||
|
||||
A concurrent security-news and CVE aggregation engine, written in Go.
|
||||
|
||||
Nadezhda ingests cybersecurity news from reliable RSS feeds, enriches every
|
||||
referenced CVE with authoritative exploit intelligence (NVD, CISA KEV, FIRST
|
||||
EPSS), clusters the same story across outlets, ranks items by real-world
|
||||
significance, and surfaces content angles. It ships as a single static binary
|
||||
with a local SQLite store, a colorful terminal UI, and Markdown/JSON export.
|
||||
|
||||
## Status
|
||||
|
||||
Early development. The scaffold is in place: configuration, source registry,
|
||||
SQLite store with forward-only migrations, and the command skeleton.
|
||||
|
||||
```
|
||||
nadezhda version # print version
|
||||
nadezhda sources # list configured feeds and persist them to the store
|
||||
```
|
||||
|
||||
Ingestion, CVE enrichment, ranking, the TUI, and the AI ideation layer land in
|
||||
subsequent milestones.
|
||||
|
||||
## Build
|
||||
|
||||
```
|
||||
just build # -> ./nadezhda
|
||||
just test
|
||||
```
|
||||
|
||||
Requires Go 1.25+.
|
||||
|
||||
---
|
||||
|
||||
Full documentation lands in `learn/` as the project matures.
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
// ©AngelaMos | 2026
|
||||
// main.go
|
||||
|
||||
package main
|
||||
|
||||
import "os"
|
||||
|
||||
func main() {
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
// ©AngelaMos | 2026
|
||||
// root.go
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/config"
|
||||
)
|
||||
|
||||
var (
|
||||
flagConfig string
|
||||
flagDB string
|
||||
)
|
||||
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "nadezhda",
|
||||
Short: "Security news and CVE aggregation engine",
|
||||
Long: "Nadezhda aggregates cybersecurity news, enriches CVEs with NVD, CISA KEV, and EPSS intelligence, clusters stories across outlets, and ranks what matters.",
|
||||
SilenceUsage: true,
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.PersistentFlags().StringVar(&flagConfig, "config", "", "path to config.yaml (optional)")
|
||||
rootCmd.PersistentFlags().StringVar(&flagDB, "db", "", "override database path")
|
||||
}
|
||||
|
||||
func loadConfig() (config.Config, error) {
|
||||
cfg, err := config.Load(flagConfig)
|
||||
if err != nil {
|
||||
return config.Config{}, err
|
||||
}
|
||||
if flagDB != "" {
|
||||
cfg.DBPath = flagDB
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
// ©AngelaMos | 2026
|
||||
// sources.go
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/source"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/store"
|
||||
)
|
||||
|
||||
var sourcesCmd = &cobra.Command{
|
||||
Use: "sources",
|
||||
Short: "List configured sources and persist them to the store",
|
||||
RunE: runSources,
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(sourcesCmd)
|
||||
}
|
||||
|
||||
func runSources(cmd *cobra.Command, args []string) error {
|
||||
cfg, err := loadConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
srcs, err := source.Load(cfg.SourcesPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
st, err := store.Open(cfg.DBPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer st.Close()
|
||||
|
||||
for _, s := range srcs {
|
||||
if _, err := st.UpsertSource(store.SourceInput{
|
||||
Name: s.Name, Title: s.Title, URL: s.URL, Type: string(s.Type),
|
||||
Weight: s.Weight, Tags: s.Tags, Enabled: s.Enabled,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("%-18s %-8s %-7s %s\n", "NAME", "ENABLED", "WEIGHT", "URL")
|
||||
for _, s := range srcs {
|
||||
enabled := "no"
|
||||
if s.Enabled {
|
||||
enabled = "yes"
|
||||
}
|
||||
fmt.Printf("%-18s %-8s %-7.2f %s\n", s.Name, enabled, s.Weight, s.URL)
|
||||
}
|
||||
fmt.Printf("\n%d sources (%d enabled) persisted to %s\n",
|
||||
len(srcs), len(source.Enabled(srcs)), cfg.DBPath)
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
// ©AngelaMos | 2026
|
||||
// stubs.go
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func notImplemented(milestone string) func(*cobra.Command, []string) error {
|
||||
return func(cmd *cobra.Command, args []string) error {
|
||||
return fmt.Errorf("%s is not implemented yet (%s)", cmd.Name(), milestone)
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
stubs := []struct {
|
||||
use string
|
||||
short string
|
||||
milestone string
|
||||
}{
|
||||
{"scrape", "Ingest all enabled sources once", "milestone M1"},
|
||||
{"list", "List stored articles with filters", "milestone M3"},
|
||||
{"cve", "Show an enriched CVE and articles mentioning it", "milestone M3"},
|
||||
{"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"},
|
||||
{"tui", "Browse aggregated news in an interactive terminal UI", "milestone M5"},
|
||||
{"ideate", "Generate content angles from ranked clusters via an AI provider", "milestone M6"},
|
||||
{"watch", "Run as a daemon, re-ingesting on an interval", "milestone M7"},
|
||||
}
|
||||
for _, s := range stubs {
|
||||
rootCmd.AddCommand(&cobra.Command{
|
||||
Use: s.use,
|
||||
Short: s.short,
|
||||
RunE: notImplemented(s.milestone),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
// ©AngelaMos | 2026
|
||||
// version.go
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/version"
|
||||
)
|
||||
|
||||
var versionCmd = &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "Print version",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
fmt.Printf("%s %s\n", version.Name, version.Version)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(versionCmd)
|
||||
}
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
module github.com/CarterPerez-dev/nadezhda
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/spf13/cobra v1.10.2
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.53.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/spf13/pflag v1.0.9 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
modernc.org/libc v1.73.4 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
)
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
|
||||
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
|
||||
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c=
|
||||
modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||
modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws=
|
||||
modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE=
|
||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc=
|
||||
modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||
modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA=
|
||||
modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
|
||||
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M=
|
||||
modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s=
|
||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
|
|
@ -0,0 +1,206 @@
|
|||
// ©AngelaMos | 2026
|
||||
// config.go
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultDBPath = "nadezhda.db"
|
||||
|
||||
defaultUserAgent = "nadezhda/0.1 (+https://github.com/CarterPerez-dev/nadezhda)"
|
||||
defaultPerHostRate = 0.5
|
||||
defaultPerHostBurst = 1
|
||||
defaultTimeoutSeconds = 25
|
||||
defaultWorkers = 8
|
||||
defaultMaxRetries = 3
|
||||
|
||||
defaultCacheTTLHours = 24
|
||||
defaultNegativeTTLHours = 3
|
||||
|
||||
defaultTitleJaccard = 0.6
|
||||
defaultWindowHours = 72
|
||||
|
||||
defaultHalfLifeHours = 48
|
||||
defaultVelocityNorm = 0.5
|
||||
|
||||
defaultWeightRecency = 0.20
|
||||
defaultWeightCVSS = 0.15
|
||||
defaultWeightKEV = 0.25
|
||||
defaultWeightEPSS = 0.15
|
||||
defaultWeightVelocity = 0.15
|
||||
defaultWeightSource = 0.05
|
||||
defaultWeightKeyword = 0.05
|
||||
|
||||
defaultAIProvider = "qwen"
|
||||
defaultQwenBaseURL = "http://localhost:11434/v1"
|
||||
defaultQwenModel = "qwen2.5:7b"
|
||||
defaultOpenAIURL = "https://api.openai.com/v1"
|
||||
defaultOpenAIModel = "gpt-4o-mini"
|
||||
defaultGeminiURL = "https://generativelanguage.googleapis.com/v1beta/openai/"
|
||||
defaultGeminiModel = "gemini-2.5-flash"
|
||||
defaultAnthropicURL = "https://api.anthropic.com/v1"
|
||||
defaultClaudeModel = "claude-opus-4-8"
|
||||
)
|
||||
|
||||
type Fetch struct {
|
||||
UserAgent string `yaml:"user_agent"`
|
||||
PerHostRate float64 `yaml:"per_host_rate"`
|
||||
PerHostBurst int `yaml:"per_host_burst"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
||||
Workers int `yaml:"workers"`
|
||||
MaxRetries int `yaml:"max_retries"`
|
||||
}
|
||||
|
||||
type Enrich struct {
|
||||
CacheTTLHours int `yaml:"cache_ttl_hours"`
|
||||
NegativeTTLHours int `yaml:"negative_ttl_hours"`
|
||||
}
|
||||
|
||||
type Cluster struct {
|
||||
TitleJaccard float64 `yaml:"title_jaccard_threshold"`
|
||||
WindowHours int `yaml:"window_hours"`
|
||||
}
|
||||
|
||||
type Weights struct {
|
||||
Recency float64 `yaml:"recency"`
|
||||
CVSS float64 `yaml:"cvss"`
|
||||
KEV float64 `yaml:"kev"`
|
||||
EPSS float64 `yaml:"epss"`
|
||||
Velocity float64 `yaml:"velocity"`
|
||||
Source float64 `yaml:"source"`
|
||||
Keyword float64 `yaml:"keyword"`
|
||||
}
|
||||
|
||||
type Rank struct {
|
||||
HalfLifeHours int `yaml:"half_life_hours"`
|
||||
VelocityNorm float64 `yaml:"velocity_norm"`
|
||||
Weights Weights `yaml:"weights"`
|
||||
}
|
||||
|
||||
type Provider struct {
|
||||
BaseURL string `yaml:"base_url"`
|
||||
Model string `yaml:"model"`
|
||||
}
|
||||
|
||||
type AI struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Provider string `yaml:"provider"`
|
||||
Qwen Provider `yaml:"qwen"`
|
||||
OpenAI Provider `yaml:"openai"`
|
||||
Gemini Provider `yaml:"gemini"`
|
||||
Anthropic Provider `yaml:"anthropic"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
DBPath string `yaml:"db_path"`
|
||||
SourcesPath string `yaml:"sources_path"`
|
||||
Watchlist []string `yaml:"watchlist"`
|
||||
Fetch Fetch `yaml:"fetch"`
|
||||
Enrich Enrich `yaml:"enrich"`
|
||||
Cluster Cluster `yaml:"cluster"`
|
||||
Rank Rank `yaml:"rank"`
|
||||
AI AI `yaml:"ai"`
|
||||
}
|
||||
|
||||
func Default() Config {
|
||||
return Config{
|
||||
DBPath: defaultDBPath,
|
||||
Fetch: Fetch{
|
||||
UserAgent: defaultUserAgent,
|
||||
PerHostRate: defaultPerHostRate,
|
||||
PerHostBurst: defaultPerHostBurst,
|
||||
TimeoutSeconds: defaultTimeoutSeconds,
|
||||
Workers: defaultWorkers,
|
||||
MaxRetries: defaultMaxRetries,
|
||||
},
|
||||
Enrich: Enrich{
|
||||
CacheTTLHours: defaultCacheTTLHours,
|
||||
NegativeTTLHours: defaultNegativeTTLHours,
|
||||
},
|
||||
Cluster: Cluster{
|
||||
TitleJaccard: defaultTitleJaccard,
|
||||
WindowHours: defaultWindowHours,
|
||||
},
|
||||
Rank: Rank{
|
||||
HalfLifeHours: defaultHalfLifeHours,
|
||||
VelocityNorm: defaultVelocityNorm,
|
||||
Weights: Weights{
|
||||
Recency: defaultWeightRecency,
|
||||
CVSS: defaultWeightCVSS,
|
||||
KEV: defaultWeightKEV,
|
||||
EPSS: defaultWeightEPSS,
|
||||
Velocity: defaultWeightVelocity,
|
||||
Source: defaultWeightSource,
|
||||
Keyword: defaultWeightKeyword,
|
||||
},
|
||||
},
|
||||
AI: AI{
|
||||
Enabled: false,
|
||||
Provider: defaultAIProvider,
|
||||
Qwen: Provider{BaseURL: defaultQwenBaseURL, Model: defaultQwenModel},
|
||||
OpenAI: Provider{BaseURL: defaultOpenAIURL, Model: defaultOpenAIModel},
|
||||
Gemini: Provider{BaseURL: defaultGeminiURL, Model: defaultGeminiModel},
|
||||
Anthropic: Provider{BaseURL: defaultAnthropicURL, Model: defaultClaudeModel},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func Load(path string) (Config, error) {
|
||||
cfg := Default()
|
||||
if path == "" {
|
||||
return cfg, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return cfg, nil
|
||||
}
|
||||
return Config{}, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
if err := yaml.Unmarshal(raw, &cfg); err != nil {
|
||||
return Config{}, fmt.Errorf("parse config %s: %w", path, err)
|
||||
}
|
||||
if err := cfg.validate(); err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func (c Config) validate() error {
|
||||
if c.DBPath == "" {
|
||||
return fmt.Errorf("config: db_path must not be empty")
|
||||
}
|
||||
if c.Fetch.Workers < 1 {
|
||||
return fmt.Errorf("config: fetch.workers must be >= 1, got %d", c.Fetch.Workers)
|
||||
}
|
||||
if c.Fetch.PerHostRate <= 0 {
|
||||
return fmt.Errorf("config: fetch.per_host_rate must be > 0, got %v", c.Fetch.PerHostRate)
|
||||
}
|
||||
if c.Fetch.PerHostBurst < 1 {
|
||||
return fmt.Errorf("config: fetch.per_host_burst must be >= 1, got %d", c.Fetch.PerHostBurst)
|
||||
}
|
||||
if c.Enrich.CacheTTLHours < 0 || c.Enrich.NegativeTTLHours < 0 {
|
||||
return fmt.Errorf("config: enrich TTLs must be >= 0, got cache=%d negative=%d", c.Enrich.CacheTTLHours, c.Enrich.NegativeTTLHours)
|
||||
}
|
||||
if c.Cluster.TitleJaccard < 0 || c.Cluster.TitleJaccard > 1 {
|
||||
return fmt.Errorf("config: cluster.title_jaccard_threshold must be in [0,1], got %v", c.Cluster.TitleJaccard)
|
||||
}
|
||||
if c.Rank.HalfLifeHours < 1 {
|
||||
return fmt.Errorf("config: rank.half_life_hours must be >= 1, got %d", c.Rank.HalfLifeHours)
|
||||
}
|
||||
if c.Rank.VelocityNorm <= 0 {
|
||||
return fmt.Errorf("config: rank.velocity_norm must be > 0, got %v", c.Rank.VelocityNorm)
|
||||
}
|
||||
switch c.AI.Provider {
|
||||
case "qwen", "openai", "gemini", "anthropic":
|
||||
default:
|
||||
return fmt.Errorf("config: ai.provider must be one of qwen|openai|gemini|anthropic, got %q", c.AI.Provider)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
// ©AngelaMos | 2026
|
||||
// config_test.go
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDefaultHasSaneValues(t *testing.T) {
|
||||
c := Default()
|
||||
if c.DBPath != defaultDBPath {
|
||||
t.Errorf("db_path = %q, want %q", c.DBPath, defaultDBPath)
|
||||
}
|
||||
if c.AI.Enabled {
|
||||
t.Error("ai.enabled must default to false (opt-in)")
|
||||
}
|
||||
if c.AI.Provider != "qwen" {
|
||||
t.Errorf("ai.provider = %q, want qwen", c.AI.Provider)
|
||||
}
|
||||
if c.AI.Qwen.Model != defaultQwenModel {
|
||||
t.Errorf("qwen model = %q, want %q", c.AI.Qwen.Model, defaultQwenModel)
|
||||
}
|
||||
sum := c.Rank.Weights.Recency + c.Rank.Weights.CVSS + c.Rank.Weights.KEV +
|
||||
c.Rank.Weights.EPSS + c.Rank.Weights.Velocity + c.Rank.Weights.Source +
|
||||
c.Rank.Weights.Keyword
|
||||
if sum < 0.99 || sum > 1.01 {
|
||||
t.Errorf("rank weights sum = %v, want ~1.0", sum)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadMissingFileReturnsDefaults(t *testing.T) {
|
||||
c, err := Load(filepath.Join(t.TempDir(), "nope.yaml"))
|
||||
if err != nil {
|
||||
t.Fatalf("Load missing file: %v", err)
|
||||
}
|
||||
if c.DBPath != defaultDBPath {
|
||||
t.Errorf("missing file should yield defaults, got db_path %q", c.DBPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadOverridesDefaults(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
body := `
|
||||
db_path: /tmp/custom.db
|
||||
watchlist: [fortinet, "cisco ios"]
|
||||
fetch:
|
||||
workers: 4
|
||||
ai:
|
||||
enabled: true
|
||||
provider: anthropic
|
||||
anthropic:
|
||||
model: claude-sonnet-4-6
|
||||
`
|
||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if c.DBPath != "/tmp/custom.db" {
|
||||
t.Errorf("db_path = %q, want /tmp/custom.db", c.DBPath)
|
||||
}
|
||||
if len(c.Watchlist) != 2 || c.Watchlist[0] != "fortinet" {
|
||||
t.Errorf("watchlist = %v", c.Watchlist)
|
||||
}
|
||||
if c.Fetch.Workers != 4 {
|
||||
t.Errorf("workers = %d, want 4", c.Fetch.Workers)
|
||||
}
|
||||
if !c.AI.Enabled || c.AI.Provider != "anthropic" {
|
||||
t.Errorf("ai override failed: %+v", c.AI)
|
||||
}
|
||||
if c.AI.Anthropic.Model != "claude-sonnet-4-6" {
|
||||
t.Errorf("anthropic model = %q, want claude-sonnet-4-6", c.AI.Anthropic.Model)
|
||||
}
|
||||
if c.AI.Qwen.Model != defaultQwenModel {
|
||||
t.Errorf("unset qwen model should keep default, got %q", c.AI.Qwen.Model)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsBadProvider(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "bad.yaml")
|
||||
if err := os.WriteFile(path, []byte("ai:\n provider: llama\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Load(path); err == nil {
|
||||
t.Error("expected error for invalid ai.provider")
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
// ©AngelaMos | 2026
|
||||
// source.go
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
//go:embed sources.yaml
|
||||
var embedded []byte
|
||||
|
||||
type Kind string
|
||||
|
||||
const (
|
||||
KindRSS Kind = "rss"
|
||||
KindAtom Kind = "atom"
|
||||
KindHTML Kind = "html"
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
Name string `yaml:"name"`
|
||||
Title string `yaml:"title"`
|
||||
URL string `yaml:"url"`
|
||||
Type Kind `yaml:"type"`
|
||||
Extractor string `yaml:"extractor"`
|
||||
Weight float64 `yaml:"weight"`
|
||||
Tags []string `yaml:"tags"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
}
|
||||
|
||||
func Defaults() ([]Source, error) {
|
||||
return parse(embedded)
|
||||
}
|
||||
|
||||
func Load(path string) ([]Source, error) {
|
||||
if path == "" {
|
||||
return Defaults()
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return Defaults()
|
||||
}
|
||||
return nil, fmt.Errorf("read sources %s: %w", path, err)
|
||||
}
|
||||
return parse(raw)
|
||||
}
|
||||
|
||||
func parse(raw []byte) ([]Source, error) {
|
||||
var out []Source
|
||||
if err := yaml.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("parse sources: %w", err)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, fmt.Errorf("sources: no entries")
|
||||
}
|
||||
seen := make(map[string]struct{}, len(out))
|
||||
for i, s := range out {
|
||||
if s.Name == "" {
|
||||
return nil, fmt.Errorf("sources[%d]: name is required", i)
|
||||
}
|
||||
if _, dup := seen[s.Name]; dup {
|
||||
return nil, fmt.Errorf("sources: duplicate name %q", s.Name)
|
||||
}
|
||||
seen[s.Name] = struct{}{}
|
||||
if _, err := url.ParseRequestURI(s.URL); err != nil {
|
||||
return nil, fmt.Errorf("sources[%s]: invalid url %q: %w", s.Name, s.URL, err)
|
||||
}
|
||||
switch s.Type {
|
||||
case KindRSS, KindAtom, KindHTML:
|
||||
default:
|
||||
return nil, fmt.Errorf("sources[%s]: type must be rss|atom|html, got %q", s.Name, s.Type)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func Enabled(all []Source) []Source {
|
||||
out := make([]Source, 0, len(all))
|
||||
for _, s := range all {
|
||||
if s.Enabled {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
// ©AngelaMos | 2026
|
||||
// source_test.go
|
||||
|
||||
package source
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEmbeddedDefaultsParse(t *testing.T) {
|
||||
got, err := Defaults()
|
||||
if err != nil {
|
||||
t.Fatalf("Defaults: %v", err)
|
||||
}
|
||||
want := map[string]bool{
|
||||
"krebs": false, "thehackernews": false, "bleepingcomputer": false,
|
||||
"securityweek": false, "darkreading": false, "theregister": false,
|
||||
"cisa": false,
|
||||
}
|
||||
for _, s := range got {
|
||||
if _, ok := want[s.Name]; !ok {
|
||||
continue
|
||||
}
|
||||
want[s.Name] = true
|
||||
if s.Type != KindRSS {
|
||||
t.Errorf("%s: type = %q, want rss", s.Name, s.Type)
|
||||
}
|
||||
if s.Weight <= 0 || s.Weight > 1 {
|
||||
t.Errorf("%s: weight %v out of (0,1]", s.Name, s.Weight)
|
||||
}
|
||||
if !s.Enabled {
|
||||
t.Errorf("%s: expected enabled by default", s.Name)
|
||||
}
|
||||
}
|
||||
for name, seen := range want {
|
||||
if !seen {
|
||||
t.Errorf("seed source %q missing from embedded defaults", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExternalOverride(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "sources.yaml")
|
||||
body := `
|
||||
- name: custom
|
||||
title: Custom Feed
|
||||
url: https://example.com/feed.xml
|
||||
type: rss
|
||||
weight: 0.5
|
||||
enabled: true
|
||||
`
|
||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Load(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Name != "custom" {
|
||||
t.Errorf("external override = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRejectsBadURL(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "bad.yaml")
|
||||
body := "- name: x\n url: not-a-url\n type: rss\n enabled: true\n"
|
||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Load(path); err == nil {
|
||||
t.Error("expected error for invalid url")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRejectsDuplicateName(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "dup.yaml")
|
||||
body := `
|
||||
- {name: a, url: "https://a.com/f", type: rss, enabled: true}
|
||||
- {name: a, url: "https://b.com/f", type: rss, enabled: true}
|
||||
`
|
||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Load(path); err == nil {
|
||||
t.Error("expected error for duplicate source name")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnabledFilter(t *testing.T) {
|
||||
all := []Source{
|
||||
{Name: "on", Enabled: true},
|
||||
{Name: "off", Enabled: false},
|
||||
}
|
||||
got := Enabled(all)
|
||||
if len(got) != 1 || got[0].Name != "on" {
|
||||
t.Errorf("Enabled filter = %+v", got)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
# ©AngelaMos | 2026
|
||||
# sources.yaml
|
||||
|
||||
- name: krebs
|
||||
title: Krebs on Security
|
||||
url: https://krebsonsecurity.com/feed/
|
||||
type: rss
|
||||
weight: 1.0
|
||||
tags: [news]
|
||||
enabled: true
|
||||
|
||||
- name: thehackernews
|
||||
title: The Hacker News
|
||||
url: https://feeds.feedburner.com/TheHackersNews
|
||||
type: rss
|
||||
weight: 0.8
|
||||
tags: [news]
|
||||
enabled: true
|
||||
|
||||
- name: bleepingcomputer
|
||||
title: BleepingComputer
|
||||
url: https://www.bleepingcomputer.com/feed/
|
||||
type: rss
|
||||
weight: 0.9
|
||||
tags: [news]
|
||||
enabled: true
|
||||
|
||||
- name: securityweek
|
||||
title: SecurityWeek
|
||||
url: https://www.securityweek.com/feed/
|
||||
type: rss
|
||||
weight: 0.8
|
||||
tags: [news]
|
||||
enabled: true
|
||||
|
||||
- name: darkreading
|
||||
title: Dark Reading
|
||||
url: https://www.darkreading.com/rss.xml
|
||||
type: rss
|
||||
weight: 0.8
|
||||
tags: [news]
|
||||
enabled: true
|
||||
|
||||
- name: theregister
|
||||
title: The Register (Security)
|
||||
url: https://www.theregister.com/security/headlines.atom
|
||||
type: rss
|
||||
weight: 0.8
|
||||
tags: [news]
|
||||
enabled: true
|
||||
|
||||
- name: cisa
|
||||
title: CISA Advisories
|
||||
url: https://www.cisa.gov/cybersecurity-advisories/all.xml
|
||||
type: rss
|
||||
weight: 1.0
|
||||
tags: [advisory, gov]
|
||||
enabled: true
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
// ©AngelaMos | 2026
|
||||
// migrate.go
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"embed"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
//go:embed migrations/*.sql
|
||||
var migrationsFS embed.FS
|
||||
|
||||
type migration struct {
|
||||
version int
|
||||
name string
|
||||
sql string
|
||||
}
|
||||
|
||||
func loadMigrations() ([]migration, error) {
|
||||
entries, err := fs.ReadDir(migrationsFS, "migrations")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read migrations dir: %w", err)
|
||||
}
|
||||
out := make([]migration, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".sql") {
|
||||
continue
|
||||
}
|
||||
verStr, _, ok := strings.Cut(e.Name(), "_")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("migration %q: expected NNNN_name.sql", e.Name())
|
||||
}
|
||||
ver, err := strconv.Atoi(verStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("migration %q: bad version prefix: %w", e.Name(), err)
|
||||
}
|
||||
body, err := migrationsFS.ReadFile("migrations/" + e.Name())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read migration %q: %w", e.Name(), err)
|
||||
}
|
||||
out = append(out, migration{version: ver, name: e.Name(), sql: string(body)})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].version < out[j].version })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func migrate(db *sql.DB) (int, error) {
|
||||
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version INTEGER PRIMARY KEY,
|
||||
applied_at INTEGER NOT NULL DEFAULT (strftime('%s','now'))
|
||||
)`); err != nil {
|
||||
return 0, fmt.Errorf("create schema_migrations: %w", err)
|
||||
}
|
||||
|
||||
var current int
|
||||
if err := db.QueryRow(`SELECT COALESCE(MAX(version), 0) FROM schema_migrations`).Scan(¤t); err != nil {
|
||||
return 0, fmt.Errorf("read current version: %w", err)
|
||||
}
|
||||
|
||||
migrations, err := loadMigrations()
|
||||
if err != nil {
|
||||
return current, err
|
||||
}
|
||||
|
||||
maxKnown := 0
|
||||
for _, m := range migrations {
|
||||
if m.version > maxKnown {
|
||||
maxKnown = m.version
|
||||
}
|
||||
}
|
||||
if current > maxKnown {
|
||||
return current, fmt.Errorf("store schema version %d is newer than this binary supports (max %d); refusing to run against a store written by a newer build", current, maxKnown)
|
||||
}
|
||||
|
||||
for _, m := range migrations {
|
||||
if m.version <= current {
|
||||
continue
|
||||
}
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return current, fmt.Errorf("begin migration %d: %w", m.version, err)
|
||||
}
|
||||
if _, err := tx.Exec(m.sql); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return current, fmt.Errorf("apply migration %q: %w", m.name, err)
|
||||
}
|
||||
if _, err := tx.Exec(`INSERT INTO schema_migrations(version) VALUES (?)`, m.version); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return current, fmt.Errorf("record migration %d: %w", m.version, err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return current, fmt.Errorf("commit migration %d: %w", m.version, err)
|
||||
}
|
||||
current = m.version
|
||||
}
|
||||
return current, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
-- ©AngelaMos | 2026
|
||||
-- 0001_init.sql
|
||||
|
||||
CREATE TABLE sources (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
url TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
weight REAL NOT NULL DEFAULT 1.0,
|
||||
tags TEXT NOT NULL DEFAULT '',
|
||||
enabled INTEGER NOT NULL DEFAULT 1
|
||||
);
|
||||
|
||||
CREATE TABLE fetch_state (
|
||||
source_id INTEGER PRIMARY KEY REFERENCES sources(id) ON DELETE CASCADE,
|
||||
etag TEXT NOT NULL DEFAULT '',
|
||||
last_modified TEXT NOT NULL DEFAULT '',
|
||||
last_fetched INTEGER NOT NULL DEFAULT 0,
|
||||
last_status INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE articles (
|
||||
id INTEGER PRIMARY KEY,
|
||||
source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
|
||||
canonical_url TEXT NOT NULL UNIQUE,
|
||||
content_hash TEXT NOT NULL UNIQUE,
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
summary TEXT NOT NULL DEFAULT '',
|
||||
body TEXT NOT NULL DEFAULT '',
|
||||
author TEXT NOT NULL DEFAULT '',
|
||||
published_at INTEGER NOT NULL DEFAULT 0,
|
||||
fetched_at INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX idx_articles_published ON articles(published_at);
|
||||
CREATE INDEX idx_articles_source ON articles(source_id);
|
||||
|
||||
CREATE TABLE cves (
|
||||
id TEXT PRIMARY KEY,
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
cvss_score REAL,
|
||||
cvss_version TEXT NOT NULL DEFAULT '',
|
||||
cvss_severity TEXT NOT NULL DEFAULT '',
|
||||
cvss_vector TEXT NOT NULL DEFAULT '',
|
||||
cwe TEXT NOT NULL DEFAULT '',
|
||||
is_kev INTEGER NOT NULL DEFAULT 0,
|
||||
kev_date_added TEXT NOT NULL DEFAULT '',
|
||||
kev_ransomware INTEGER NOT NULL DEFAULT 0,
|
||||
epss REAL,
|
||||
epss_percentile REAL,
|
||||
nvd_published TEXT NOT NULL DEFAULT '',
|
||||
nvd_modified TEXT NOT NULL DEFAULT '',
|
||||
enriched_at INTEGER NOT NULL DEFAULT 0,
|
||||
enrich_status TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE TABLE article_cves (
|
||||
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
cve_id TEXT NOT NULL REFERENCES cves(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (article_id, cve_id)
|
||||
);
|
||||
|
||||
CREATE TABLE clusters (
|
||||
id INTEGER PRIMARY KEY,
|
||||
cluster_key TEXT NOT NULL UNIQUE,
|
||||
first_seen INTEGER NOT NULL DEFAULT 0,
|
||||
last_seen INTEGER NOT NULL DEFAULT 0,
|
||||
size INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE cluster_members (
|
||||
cluster_id INTEGER NOT NULL REFERENCES clusters(id) ON DELETE CASCADE,
|
||||
article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (cluster_id, article_id)
|
||||
);
|
||||
|
||||
CREATE TABLE ai_notes (
|
||||
id INTEGER PRIMARY KEY,
|
||||
cluster_id INTEGER NOT NULL REFERENCES clusters(id) ON DELETE CASCADE,
|
||||
provider TEXT NOT NULL DEFAULT '',
|
||||
summary TEXT NOT NULL DEFAULT '',
|
||||
why TEXT NOT NULL DEFAULT '',
|
||||
angles_json TEXT NOT NULL DEFAULT '',
|
||||
format TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
|
@ -0,0 +1,149 @@
|
|||
// ©AngelaMos | 2026
|
||||
// store.go
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"modernc.org/sqlite"
|
||||
sqlite3 "modernc.org/sqlite/lib"
|
||||
)
|
||||
|
||||
var ErrDuplicate = errors.New("store: article already exists")
|
||||
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
version int
|
||||
}
|
||||
|
||||
type SourceInput struct {
|
||||
Name string
|
||||
Title string
|
||||
URL string
|
||||
Type string
|
||||
Weight float64
|
||||
Tags []string
|
||||
Enabled bool
|
||||
}
|
||||
|
||||
type SourceRow struct {
|
||||
ID int64
|
||||
Name string
|
||||
Title string
|
||||
URL string
|
||||
Type string
|
||||
Weight float64
|
||||
Tags []string
|
||||
Enabled bool
|
||||
}
|
||||
|
||||
type Article struct {
|
||||
SourceID int64
|
||||
CanonicalURL string
|
||||
ContentHash string
|
||||
Title string
|
||||
Summary string
|
||||
Body string
|
||||
Author string
|
||||
PublishedAt int64
|
||||
FetchedAt int64
|
||||
}
|
||||
|
||||
func Open(path string) (*Store, error) {
|
||||
dsn := fmt.Sprintf("file:%s?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)", path)
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open sqlite %s: %w", path, err)
|
||||
}
|
||||
if err := db.Ping(); err != nil {
|
||||
_ = db.Close()
|
||||
return nil, fmt.Errorf("ping sqlite %s: %w", path, err)
|
||||
}
|
||||
version, err := migrate(db)
|
||||
if err != nil {
|
||||
_ = db.Close()
|
||||
return nil, err
|
||||
}
|
||||
return &Store{db: db, version: version}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
func (s *Store) Version() int { return s.version }
|
||||
func (s *Store) DB() *sql.DB { return s.db }
|
||||
|
||||
func (s *Store) UpsertSource(in SourceInput) (int64, error) {
|
||||
tags := strings.Join(in.Tags, ",")
|
||||
var id int64
|
||||
err := s.db.QueryRow(`
|
||||
INSERT INTO sources (name, title, url, type, weight, tags, enabled)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
title = excluded.title, url = excluded.url, type = excluded.type,
|
||||
weight = excluded.weight, tags = excluded.tags, enabled = excluded.enabled
|
||||
RETURNING id`,
|
||||
in.Name, in.Title, in.URL, in.Type, in.Weight, tags, boolToInt(in.Enabled),
|
||||
).Scan(&id)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("upsert source %q: %w", in.Name, err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetSourceByName(name string) (SourceRow, error) {
|
||||
var r SourceRow
|
||||
var tags string
|
||||
var enabled int
|
||||
err := s.db.QueryRow(`
|
||||
SELECT id, name, title, url, type, weight, tags, enabled
|
||||
FROM sources WHERE name = ?`, name,
|
||||
).Scan(&r.ID, &r.Name, &r.Title, &r.URL, &r.Type, &r.Weight, &tags, &enabled)
|
||||
if err != nil {
|
||||
return SourceRow{}, fmt.Errorf("get source %q: %w", name, err)
|
||||
}
|
||||
if tags != "" {
|
||||
r.Tags = strings.Split(tags, ",")
|
||||
}
|
||||
r.Enabled = enabled != 0
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (s *Store) InsertArticle(a Article) (int64, error) {
|
||||
res, err := s.db.Exec(`
|
||||
INSERT INTO articles
|
||||
(source_id, canonical_url, content_hash, title, summary, body, author, published_at, fetched_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
a.SourceID, a.CanonicalURL, a.ContentHash, a.Title, a.Summary, a.Body,
|
||||
a.Author, a.PublishedAt, a.FetchedAt,
|
||||
)
|
||||
if err != nil {
|
||||
var se *sqlite.Error
|
||||
if errors.As(err, &se) && se.Code() == sqlite3.SQLITE_CONSTRAINT_UNIQUE {
|
||||
return 0, ErrDuplicate
|
||||
}
|
||||
return 0, fmt.Errorf("insert article %q: %w", a.CanonicalURL, err)
|
||||
}
|
||||
id, err := res.LastInsertId()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("insert article %q: last insert id: %w", a.CanonicalURL, err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (s *Store) CountArticles() (int, error) {
|
||||
var n int
|
||||
if err := s.db.QueryRow(`SELECT COUNT(*) FROM articles`).Scan(&n); err != nil {
|
||||
return 0, fmt.Errorf("count articles: %w", err)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func boolToInt(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
// ©AngelaMos | 2026
|
||||
// store_test.go
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func openTemp(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
s, err := Open(filepath.Join(t.TempDir(), "test.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
func TestMigrateAppliesLatest(t *testing.T) {
|
||||
s := openTemp(t)
|
||||
if s.Version() < 1 {
|
||||
t.Fatalf("schema version = %d, want >= 1", s.Version())
|
||||
}
|
||||
var n int
|
||||
if err := s.DB().QueryRow(
|
||||
`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='articles'`,
|
||||
).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Error("articles table not created by migration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateIsIdempotent(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "idem.db")
|
||||
s1, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v1 := s1.Version()
|
||||
_ = s1.Close()
|
||||
|
||||
s2, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reopen: %v", err)
|
||||
}
|
||||
defer s2.Close()
|
||||
if s2.Version() != v1 {
|
||||
t.Errorf("reopen version = %d, want %d", s2.Version(), v1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSourceRoundTrip(t *testing.T) {
|
||||
s := openTemp(t)
|
||||
id, err := s.UpsertSource(SourceInput{
|
||||
Name: "krebs", Title: "Krebs", URL: "https://krebsonsecurity.com/feed/",
|
||||
Type: "rss", Weight: 1.0, Tags: []string{"news"}, Enabled: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UpsertSource: %v", err)
|
||||
}
|
||||
row, err := s.GetSourceByName("krebs")
|
||||
if err != nil {
|
||||
t.Fatalf("GetSourceByName: %v", err)
|
||||
}
|
||||
if row.ID != id || row.Weight != 1.0 || !row.Enabled || len(row.Tags) != 1 {
|
||||
t.Errorf("round trip mismatch: %+v", row)
|
||||
}
|
||||
|
||||
id2, err := s.UpsertSource(SourceInput{
|
||||
Name: "krebs", Title: "Krebs Updated", URL: "https://krebsonsecurity.com/feed/",
|
||||
Type: "rss", Weight: 0.9, Tags: []string{"news"}, Enabled: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("re-upsert: %v", err)
|
||||
}
|
||||
if id2 != id {
|
||||
t.Errorf("upsert should keep id %d, got %d", id, id2)
|
||||
}
|
||||
row2, _ := s.GetSourceByName("krebs")
|
||||
if row2.Title != "Krebs Updated" || row2.Weight != 0.9 {
|
||||
t.Errorf("upsert did not update fields: %+v", row2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArticleUniqueConstraint(t *testing.T) {
|
||||
s := openTemp(t)
|
||||
srcID, err := s.UpsertSource(SourceInput{
|
||||
Name: "krebs", URL: "https://krebsonsecurity.com/feed/", Type: "rss", Enabled: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := Article{
|
||||
SourceID: srcID, CanonicalURL: "https://krebsonsecurity.com/post-1",
|
||||
ContentHash: "hash-1", Title: "Post 1", PublishedAt: 100, FetchedAt: 200,
|
||||
}
|
||||
if _, err := s.InsertArticle(a); err != nil {
|
||||
t.Fatalf("first insert: %v", err)
|
||||
}
|
||||
if _, err := s.InsertArticle(a); !errors.Is(err, ErrDuplicate) {
|
||||
t.Errorf("duplicate insert: got %v, want ErrDuplicate", err)
|
||||
}
|
||||
|
||||
n, err := s.CountArticles()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("article count = %d, want 1 (dup rejected)", n)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
// ©AngelaMos | 2026
|
||||
// version.go
|
||||
|
||||
package version
|
||||
|
||||
const (
|
||||
Name = "nadezhda"
|
||||
Version = "0.1.0-dev"
|
||||
)
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
# ©AngelaMos | 2026
|
||||
# justfile
|
||||
|
||||
set shell := ["bash", "-cu"]
|
||||
|
||||
binary := "nadezhda"
|
||||
|
||||
default:
|
||||
@just --list
|
||||
|
||||
build:
|
||||
go build -o {{binary}} ./cmd/nadezhda
|
||||
|
||||
run *args:
|
||||
go run ./cmd/nadezhda {{args}}
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
fmt:
|
||||
gofmt -w .
|
||||
|
||||
lint: vet
|
||||
test -z "$(gofmt -l .)"
|
||||
|
||||
tidy:
|
||||
go mod tidy
|
||||
|
||||
clean:
|
||||
rm -f {{binary}} *.db *.db-wal *.db-shm
|
||||
Loading…
Reference in New Issue