diff --git a/PROJECTS/intermediate/security-news-scraper/.gitignore b/PROJECTS/intermediate/security-news-scraper/.gitignore new file mode 100644 index 00000000..38335067 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/.gitignore @@ -0,0 +1,24 @@ +# ©AngelaMos | 2026 +# .gitignore + +# private dev + research material (never public) +docs/ + +# Go build artifacts +/nadezhda +/bin/ +*.test +*.out +*.prof + +# local runtime data (SQLite store + WAL sidecars) +*.db +*.db-wal +*.db-shm +*.sqlite +/data/ + +# env / secrets / local overrides +.env +*.local.yaml +config.local.yaml diff --git a/PROJECTS/intermediate/security-news-scraper/README.md b/PROJECTS/intermediate/security-news-scraper/README.md new file mode 100644 index 00000000..f7e62dff --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/README.md @@ -0,0 +1,35 @@ +# Nadezhda + +A concurrent security-news and CVE aggregation engine, written in Go. + +Nadezhda ingests cybersecurity news from reliable RSS feeds, enriches every +referenced CVE with authoritative exploit intelligence (NVD, CISA KEV, FIRST +EPSS), clusters the same story across outlets, ranks items by real-world +significance, and surfaces content angles. It ships as a single static binary +with a local SQLite store, a colorful terminal UI, and Markdown/JSON export. + +## Status + +Early development. The scaffold is in place: configuration, source registry, +SQLite store with forward-only migrations, and the command skeleton. + +``` +nadezhda version # print version +nadezhda sources # list configured feeds and persist them to the store +``` + +Ingestion, CVE enrichment, ranking, the TUI, and the AI ideation layer land in +subsequent milestones. + +## Build + +``` +just build # -> ./nadezhda +just test +``` + +Requires Go 1.25+. + +--- + +Full documentation lands in `learn/` as the project matures. diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go new file mode 100644 index 00000000..a4c7e183 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/main.go @@ -0,0 +1,12 @@ +// ©AngelaMos | 2026 +// main.go + +package main + +import "os" + +func main() { + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/root.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/root.go new file mode 100644 index 00000000..50cd452d --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/root.go @@ -0,0 +1,38 @@ +// ©AngelaMos | 2026 +// root.go + +package main + +import ( + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/config" +) + +var ( + flagConfig string + flagDB string +) + +var rootCmd = &cobra.Command{ + Use: "nadezhda", + Short: "Security news and CVE aggregation engine", + Long: "Nadezhda aggregates cybersecurity news, enriches CVEs with NVD, CISA KEV, and EPSS intelligence, clusters stories across outlets, and ranks what matters.", + SilenceUsage: true, +} + +func init() { + rootCmd.PersistentFlags().StringVar(&flagConfig, "config", "", "path to config.yaml (optional)") + rootCmd.PersistentFlags().StringVar(&flagDB, "db", "", "override database path") +} + +func loadConfig() (config.Config, error) { + cfg, err := config.Load(flagConfig) + if err != nil { + return config.Config{}, err + } + if flagDB != "" { + cfg.DBPath = flagDB + } + return cfg, nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/sources.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/sources.go new file mode 100644 index 00000000..0c902d8f --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/sources.go @@ -0,0 +1,60 @@ +// ©AngelaMos | 2026 +// sources.go + +package main + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/source" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +var sourcesCmd = &cobra.Command{ + Use: "sources", + Short: "List configured sources and persist them to the store", + RunE: runSources, +} + +func init() { + rootCmd.AddCommand(sourcesCmd) +} + +func runSources(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + srcs, err := source.Load(cfg.SourcesPath) + if err != nil { + return err + } + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + + for _, s := range srcs { + if _, err := st.UpsertSource(store.SourceInput{ + Name: s.Name, Title: s.Title, URL: s.URL, Type: string(s.Type), + Weight: s.Weight, Tags: s.Tags, Enabled: s.Enabled, + }); err != nil { + return err + } + } + + fmt.Printf("%-18s %-8s %-7s %s\n", "NAME", "ENABLED", "WEIGHT", "URL") + for _, s := range srcs { + enabled := "no" + if s.Enabled { + enabled = "yes" + } + fmt.Printf("%-18s %-8s %-7.2f %s\n", s.Name, enabled, s.Weight, s.URL) + } + fmt.Printf("\n%d sources (%d enabled) persisted to %s\n", + len(srcs), len(source.Enabled(srcs)), cfg.DBPath) + return nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go new file mode 100644 index 00000000..2f9eb03e --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go @@ -0,0 +1,39 @@ +// ©AngelaMos | 2026 +// stubs.go + +package main + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +func notImplemented(milestone string) func(*cobra.Command, []string) error { + return func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("%s is not implemented yet (%s)", cmd.Name(), milestone) + } +} + +func init() { + stubs := []struct { + use string + short string + milestone string + }{ + {"scrape", "Ingest all enabled sources once", "milestone M1"}, + {"list", "List stored articles with filters", "milestone M3"}, + {"cve", "Show an enriched CVE and articles mentioning it", "milestone M3"}, + {"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"}, + {"tui", "Browse aggregated news in an interactive terminal UI", "milestone M5"}, + {"ideate", "Generate content angles from ranked clusters via an AI provider", "milestone M6"}, + {"watch", "Run as a daemon, re-ingesting on an interval", "milestone M7"}, + } + for _, s := range stubs { + rootCmd.AddCommand(&cobra.Command{ + Use: s.use, + Short: s.short, + RunE: notImplemented(s.milestone), + }) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/version.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/version.go new file mode 100644 index 00000000..7192b63e --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/version.go @@ -0,0 +1,25 @@ +// ©AngelaMos | 2026 +// version.go + +package main + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/version" +) + +var versionCmd = &cobra.Command{ + Use: "version", + Short: "Print version", + RunE: func(cmd *cobra.Command, args []string) error { + fmt.Printf("%s %s\n", version.Name, version.Version) + return nil + }, +} + +func init() { + rootCmd.AddCommand(versionCmd) +} diff --git a/PROJECTS/intermediate/security-news-scraper/go.mod b/PROJECTS/intermediate/security-news-scraper/go.mod new file mode 100644 index 00000000..0f7c5876 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/go.mod @@ -0,0 +1,23 @@ +module github.com/CarterPerez-dev/nadezhda + +go 1.25.0 + +require ( + github.com/spf13/cobra v1.10.2 + gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.53.0 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/spf13/pflag v1.0.9 // indirect + golang.org/x/sys v0.44.0 // indirect + modernc.org/libc v1.73.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect +) diff --git a/PROJECTS/intermediate/security-news-scraper/go.sum b/PROJECTS/intermediate/security-news-scraper/go.sum new file mode 100644 index 00000000..7c4811a2 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/go.sum @@ -0,0 +1,64 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= +golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= +modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= +modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc= +modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= +modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= +modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go new file mode 100644 index 00000000..cbe18b30 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go @@ -0,0 +1,206 @@ +// ©AngelaMos | 2026 +// config.go + +package config + +import ( + "fmt" + "os" + + "gopkg.in/yaml.v3" +) + +const ( + defaultDBPath = "nadezhda.db" + + defaultUserAgent = "nadezhda/0.1 (+https://github.com/CarterPerez-dev/nadezhda)" + defaultPerHostRate = 0.5 + defaultPerHostBurst = 1 + defaultTimeoutSeconds = 25 + defaultWorkers = 8 + defaultMaxRetries = 3 + + defaultCacheTTLHours = 24 + defaultNegativeTTLHours = 3 + + defaultTitleJaccard = 0.6 + defaultWindowHours = 72 + + defaultHalfLifeHours = 48 + defaultVelocityNorm = 0.5 + + defaultWeightRecency = 0.20 + defaultWeightCVSS = 0.15 + defaultWeightKEV = 0.25 + defaultWeightEPSS = 0.15 + defaultWeightVelocity = 0.15 + defaultWeightSource = 0.05 + defaultWeightKeyword = 0.05 + + defaultAIProvider = "qwen" + defaultQwenBaseURL = "http://localhost:11434/v1" + defaultQwenModel = "qwen2.5:7b" + defaultOpenAIURL = "https://api.openai.com/v1" + defaultOpenAIModel = "gpt-4o-mini" + defaultGeminiURL = "https://generativelanguage.googleapis.com/v1beta/openai/" + defaultGeminiModel = "gemini-2.5-flash" + defaultAnthropicURL = "https://api.anthropic.com/v1" + defaultClaudeModel = "claude-opus-4-8" +) + +type Fetch struct { + UserAgent string `yaml:"user_agent"` + PerHostRate float64 `yaml:"per_host_rate"` + PerHostBurst int `yaml:"per_host_burst"` + TimeoutSeconds int `yaml:"timeout_seconds"` + Workers int `yaml:"workers"` + MaxRetries int `yaml:"max_retries"` +} + +type Enrich struct { + CacheTTLHours int `yaml:"cache_ttl_hours"` + NegativeTTLHours int `yaml:"negative_ttl_hours"` +} + +type Cluster struct { + TitleJaccard float64 `yaml:"title_jaccard_threshold"` + WindowHours int `yaml:"window_hours"` +} + +type Weights struct { + Recency float64 `yaml:"recency"` + CVSS float64 `yaml:"cvss"` + KEV float64 `yaml:"kev"` + EPSS float64 `yaml:"epss"` + Velocity float64 `yaml:"velocity"` + Source float64 `yaml:"source"` + Keyword float64 `yaml:"keyword"` +} + +type Rank struct { + HalfLifeHours int `yaml:"half_life_hours"` + VelocityNorm float64 `yaml:"velocity_norm"` + Weights Weights `yaml:"weights"` +} + +type Provider struct { + BaseURL string `yaml:"base_url"` + Model string `yaml:"model"` +} + +type AI struct { + Enabled bool `yaml:"enabled"` + Provider string `yaml:"provider"` + Qwen Provider `yaml:"qwen"` + OpenAI Provider `yaml:"openai"` + Gemini Provider `yaml:"gemini"` + Anthropic Provider `yaml:"anthropic"` +} + +type Config struct { + DBPath string `yaml:"db_path"` + SourcesPath string `yaml:"sources_path"` + Watchlist []string `yaml:"watchlist"` + Fetch Fetch `yaml:"fetch"` + Enrich Enrich `yaml:"enrich"` + Cluster Cluster `yaml:"cluster"` + Rank Rank `yaml:"rank"` + AI AI `yaml:"ai"` +} + +func Default() Config { + return Config{ + DBPath: defaultDBPath, + Fetch: Fetch{ + UserAgent: defaultUserAgent, + PerHostRate: defaultPerHostRate, + PerHostBurst: defaultPerHostBurst, + TimeoutSeconds: defaultTimeoutSeconds, + Workers: defaultWorkers, + MaxRetries: defaultMaxRetries, + }, + Enrich: Enrich{ + CacheTTLHours: defaultCacheTTLHours, + NegativeTTLHours: defaultNegativeTTLHours, + }, + Cluster: Cluster{ + TitleJaccard: defaultTitleJaccard, + WindowHours: defaultWindowHours, + }, + Rank: Rank{ + HalfLifeHours: defaultHalfLifeHours, + VelocityNorm: defaultVelocityNorm, + Weights: Weights{ + Recency: defaultWeightRecency, + CVSS: defaultWeightCVSS, + KEV: defaultWeightKEV, + EPSS: defaultWeightEPSS, + Velocity: defaultWeightVelocity, + Source: defaultWeightSource, + Keyword: defaultWeightKeyword, + }, + }, + AI: AI{ + Enabled: false, + Provider: defaultAIProvider, + Qwen: Provider{BaseURL: defaultQwenBaseURL, Model: defaultQwenModel}, + OpenAI: Provider{BaseURL: defaultOpenAIURL, Model: defaultOpenAIModel}, + Gemini: Provider{BaseURL: defaultGeminiURL, Model: defaultGeminiModel}, + Anthropic: Provider{BaseURL: defaultAnthropicURL, Model: defaultClaudeModel}, + }, + } +} + +func Load(path string) (Config, error) { + cfg := Default() + if path == "" { + return cfg, nil + } + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return cfg, nil + } + return Config{}, fmt.Errorf("read config %s: %w", path, err) + } + if err := yaml.Unmarshal(raw, &cfg); err != nil { + return Config{}, fmt.Errorf("parse config %s: %w", path, err) + } + if err := cfg.validate(); err != nil { + return Config{}, err + } + return cfg, nil +} + +func (c Config) validate() error { + if c.DBPath == "" { + return fmt.Errorf("config: db_path must not be empty") + } + if c.Fetch.Workers < 1 { + return fmt.Errorf("config: fetch.workers must be >= 1, got %d", c.Fetch.Workers) + } + if c.Fetch.PerHostRate <= 0 { + return fmt.Errorf("config: fetch.per_host_rate must be > 0, got %v", c.Fetch.PerHostRate) + } + if c.Fetch.PerHostBurst < 1 { + return fmt.Errorf("config: fetch.per_host_burst must be >= 1, got %d", c.Fetch.PerHostBurst) + } + if c.Enrich.CacheTTLHours < 0 || c.Enrich.NegativeTTLHours < 0 { + return fmt.Errorf("config: enrich TTLs must be >= 0, got cache=%d negative=%d", c.Enrich.CacheTTLHours, c.Enrich.NegativeTTLHours) + } + if c.Cluster.TitleJaccard < 0 || c.Cluster.TitleJaccard > 1 { + return fmt.Errorf("config: cluster.title_jaccard_threshold must be in [0,1], got %v", c.Cluster.TitleJaccard) + } + if c.Rank.HalfLifeHours < 1 { + return fmt.Errorf("config: rank.half_life_hours must be >= 1, got %d", c.Rank.HalfLifeHours) + } + if c.Rank.VelocityNorm <= 0 { + return fmt.Errorf("config: rank.velocity_norm must be > 0, got %v", c.Rank.VelocityNorm) + } + switch c.AI.Provider { + case "qwen", "openai", "gemini", "anthropic": + default: + return fmt.Errorf("config: ai.provider must be one of qwen|openai|gemini|anthropic, got %q", c.AI.Provider) + } + return nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config_test.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config_test.go new file mode 100644 index 00000000..9a490a58 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config_test.go @@ -0,0 +1,94 @@ +// ©AngelaMos | 2026 +// config_test.go + +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDefaultHasSaneValues(t *testing.T) { + c := Default() + if c.DBPath != defaultDBPath { + t.Errorf("db_path = %q, want %q", c.DBPath, defaultDBPath) + } + if c.AI.Enabled { + t.Error("ai.enabled must default to false (opt-in)") + } + if c.AI.Provider != "qwen" { + t.Errorf("ai.provider = %q, want qwen", c.AI.Provider) + } + if c.AI.Qwen.Model != defaultQwenModel { + t.Errorf("qwen model = %q, want %q", c.AI.Qwen.Model, defaultQwenModel) + } + sum := c.Rank.Weights.Recency + c.Rank.Weights.CVSS + c.Rank.Weights.KEV + + c.Rank.Weights.EPSS + c.Rank.Weights.Velocity + c.Rank.Weights.Source + + c.Rank.Weights.Keyword + if sum < 0.99 || sum > 1.01 { + t.Errorf("rank weights sum = %v, want ~1.0", sum) + } +} + +func TestLoadMissingFileReturnsDefaults(t *testing.T) { + c, err := Load(filepath.Join(t.TempDir(), "nope.yaml")) + if err != nil { + t.Fatalf("Load missing file: %v", err) + } + if c.DBPath != defaultDBPath { + t.Errorf("missing file should yield defaults, got db_path %q", c.DBPath) + } +} + +func TestLoadOverridesDefaults(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + body := ` +db_path: /tmp/custom.db +watchlist: [fortinet, "cisco ios"] +fetch: + workers: 4 +ai: + enabled: true + provider: anthropic + anthropic: + model: claude-sonnet-4-6 +` + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + c, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if c.DBPath != "/tmp/custom.db" { + t.Errorf("db_path = %q, want /tmp/custom.db", c.DBPath) + } + if len(c.Watchlist) != 2 || c.Watchlist[0] != "fortinet" { + t.Errorf("watchlist = %v", c.Watchlist) + } + if c.Fetch.Workers != 4 { + t.Errorf("workers = %d, want 4", c.Fetch.Workers) + } + if !c.AI.Enabled || c.AI.Provider != "anthropic" { + t.Errorf("ai override failed: %+v", c.AI) + } + if c.AI.Anthropic.Model != "claude-sonnet-4-6" { + t.Errorf("anthropic model = %q, want claude-sonnet-4-6", c.AI.Anthropic.Model) + } + if c.AI.Qwen.Model != defaultQwenModel { + t.Errorf("unset qwen model should keep default, got %q", c.AI.Qwen.Model) + } +} + +func TestValidateRejectsBadProvider(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + if err := os.WriteFile(path, []byte("ai:\n provider: llama\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err == nil { + t.Error("expected error for invalid ai.provider") + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/source/source.go b/PROJECTS/intermediate/security-news-scraper/internal/source/source.go new file mode 100644 index 00000000..864dd3ea --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/source/source.go @@ -0,0 +1,92 @@ +// ©AngelaMos | 2026 +// source.go + +package source + +import ( + _ "embed" + "fmt" + "net/url" + "os" + + "gopkg.in/yaml.v3" +) + +//go:embed sources.yaml +var embedded []byte + +type Kind string + +const ( + KindRSS Kind = "rss" + KindAtom Kind = "atom" + KindHTML Kind = "html" +) + +type Source struct { + Name string `yaml:"name"` + Title string `yaml:"title"` + URL string `yaml:"url"` + Type Kind `yaml:"type"` + Extractor string `yaml:"extractor"` + Weight float64 `yaml:"weight"` + Tags []string `yaml:"tags"` + Enabled bool `yaml:"enabled"` +} + +func Defaults() ([]Source, error) { + return parse(embedded) +} + +func Load(path string) ([]Source, error) { + if path == "" { + return Defaults() + } + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return Defaults() + } + return nil, fmt.Errorf("read sources %s: %w", path, err) + } + return parse(raw) +} + +func parse(raw []byte) ([]Source, error) { + var out []Source + if err := yaml.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("parse sources: %w", err) + } + if len(out) == 0 { + return nil, fmt.Errorf("sources: no entries") + } + seen := make(map[string]struct{}, len(out)) + for i, s := range out { + if s.Name == "" { + return nil, fmt.Errorf("sources[%d]: name is required", i) + } + if _, dup := seen[s.Name]; dup { + return nil, fmt.Errorf("sources: duplicate name %q", s.Name) + } + seen[s.Name] = struct{}{} + if _, err := url.ParseRequestURI(s.URL); err != nil { + return nil, fmt.Errorf("sources[%s]: invalid url %q: %w", s.Name, s.URL, err) + } + switch s.Type { + case KindRSS, KindAtom, KindHTML: + default: + return nil, fmt.Errorf("sources[%s]: type must be rss|atom|html, got %q", s.Name, s.Type) + } + } + return out, nil +} + +func Enabled(all []Source) []Source { + out := make([]Source, 0, len(all)) + for _, s := range all { + if s.Enabled { + out = append(out, s) + } + } + return out +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/source/source_test.go b/PROJECTS/intermediate/security-news-scraper/internal/source/source_test.go new file mode 100644 index 00000000..54137e1d --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/source/source_test.go @@ -0,0 +1,103 @@ +// ©AngelaMos | 2026 +// source_test.go + +package source + +import ( + "os" + "path/filepath" + "testing" +) + +func TestEmbeddedDefaultsParse(t *testing.T) { + got, err := Defaults() + if err != nil { + t.Fatalf("Defaults: %v", err) + } + want := map[string]bool{ + "krebs": false, "thehackernews": false, "bleepingcomputer": false, + "securityweek": false, "darkreading": false, "theregister": false, + "cisa": false, + } + for _, s := range got { + if _, ok := want[s.Name]; !ok { + continue + } + want[s.Name] = true + if s.Type != KindRSS { + t.Errorf("%s: type = %q, want rss", s.Name, s.Type) + } + if s.Weight <= 0 || s.Weight > 1 { + t.Errorf("%s: weight %v out of (0,1]", s.Name, s.Weight) + } + if !s.Enabled { + t.Errorf("%s: expected enabled by default", s.Name) + } + } + for name, seen := range want { + if !seen { + t.Errorf("seed source %q missing from embedded defaults", name) + } + } +} + +func TestExternalOverride(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "sources.yaml") + body := ` +- name: custom + title: Custom Feed + url: https://example.com/feed.xml + type: rss + weight: 0.5 + enabled: true +` + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + got, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(got) != 1 || got[0].Name != "custom" { + t.Errorf("external override = %+v", got) + } +} + +func TestRejectsBadURL(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + body := "- name: x\n url: not-a-url\n type: rss\n enabled: true\n" + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err == nil { + t.Error("expected error for invalid url") + } +} + +func TestRejectsDuplicateName(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "dup.yaml") + body := ` +- {name: a, url: "https://a.com/f", type: rss, enabled: true} +- {name: a, url: "https://b.com/f", type: rss, enabled: true} +` + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Load(path); err == nil { + t.Error("expected error for duplicate source name") + } +} + +func TestEnabledFilter(t *testing.T) { + all := []Source{ + {Name: "on", Enabled: true}, + {Name: "off", Enabled: false}, + } + got := Enabled(all) + if len(got) != 1 || got[0].Name != "on" { + t.Errorf("Enabled filter = %+v", got) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/source/sources.yaml b/PROJECTS/intermediate/security-news-scraper/internal/source/sources.yaml new file mode 100644 index 00000000..98e5ccbf --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/source/sources.yaml @@ -0,0 +1,58 @@ +# ©AngelaMos | 2026 +# sources.yaml + +- name: krebs + title: Krebs on Security + url: https://krebsonsecurity.com/feed/ + type: rss + weight: 1.0 + tags: [news] + enabled: true + +- name: thehackernews + title: The Hacker News + url: https://feeds.feedburner.com/TheHackersNews + type: rss + weight: 0.8 + tags: [news] + enabled: true + +- name: bleepingcomputer + title: BleepingComputer + url: https://www.bleepingcomputer.com/feed/ + type: rss + weight: 0.9 + tags: [news] + enabled: true + +- name: securityweek + title: SecurityWeek + url: https://www.securityweek.com/feed/ + type: rss + weight: 0.8 + tags: [news] + enabled: true + +- name: darkreading + title: Dark Reading + url: https://www.darkreading.com/rss.xml + type: rss + weight: 0.8 + tags: [news] + enabled: true + +- name: theregister + title: The Register (Security) + url: https://www.theregister.com/security/headlines.atom + type: rss + weight: 0.8 + tags: [news] + enabled: true + +- name: cisa + title: CISA Advisories + url: https://www.cisa.gov/cybersecurity-advisories/all.xml + type: rss + weight: 1.0 + tags: [advisory, gov] + enabled: true diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/migrate.go b/PROJECTS/intermediate/security-news-scraper/internal/store/migrate.go new file mode 100644 index 00000000..db1f0ac6 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/migrate.go @@ -0,0 +1,103 @@ +// ©AngelaMos | 2026 +// migrate.go + +package store + +import ( + "database/sql" + "embed" + "fmt" + "io/fs" + "sort" + "strconv" + "strings" +) + +//go:embed migrations/*.sql +var migrationsFS embed.FS + +type migration struct { + version int + name string + sql string +} + +func loadMigrations() ([]migration, error) { + entries, err := fs.ReadDir(migrationsFS, "migrations") + if err != nil { + return nil, fmt.Errorf("read migrations dir: %w", err) + } + out := make([]migration, 0, len(entries)) + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".sql") { + continue + } + verStr, _, ok := strings.Cut(e.Name(), "_") + if !ok { + return nil, fmt.Errorf("migration %q: expected NNNN_name.sql", e.Name()) + } + ver, err := strconv.Atoi(verStr) + if err != nil { + return nil, fmt.Errorf("migration %q: bad version prefix: %w", e.Name(), err) + } + body, err := migrationsFS.ReadFile("migrations/" + e.Name()) + if err != nil { + return nil, fmt.Errorf("read migration %q: %w", e.Name(), err) + } + out = append(out, migration{version: ver, name: e.Name(), sql: string(body)}) + } + sort.Slice(out, func(i, j int) bool { return out[i].version < out[j].version }) + return out, nil +} + +func migrate(db *sql.DB) (int, error) { + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at INTEGER NOT NULL DEFAULT (strftime('%s','now')) + )`); err != nil { + return 0, fmt.Errorf("create schema_migrations: %w", err) + } + + var current int + if err := db.QueryRow(`SELECT COALESCE(MAX(version), 0) FROM schema_migrations`).Scan(¤t); err != nil { + return 0, fmt.Errorf("read current version: %w", err) + } + + migrations, err := loadMigrations() + if err != nil { + return current, err + } + + maxKnown := 0 + for _, m := range migrations { + if m.version > maxKnown { + maxKnown = m.version + } + } + if current > maxKnown { + return current, fmt.Errorf("store schema version %d is newer than this binary supports (max %d); refusing to run against a store written by a newer build", current, maxKnown) + } + + for _, m := range migrations { + if m.version <= current { + continue + } + tx, err := db.Begin() + if err != nil { + return current, fmt.Errorf("begin migration %d: %w", m.version, err) + } + if _, err := tx.Exec(m.sql); err != nil { + _ = tx.Rollback() + return current, fmt.Errorf("apply migration %q: %w", m.name, err) + } + if _, err := tx.Exec(`INSERT INTO schema_migrations(version) VALUES (?)`, m.version); err != nil { + _ = tx.Rollback() + return current, fmt.Errorf("record migration %d: %w", m.version, err) + } + if err := tx.Commit(); err != nil { + return current, fmt.Errorf("commit migration %d: %w", m.version, err) + } + current = m.version + } + return current, nil +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0001_init.sql b/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0001_init.sql new file mode 100644 index 00000000..c0dc5cf8 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0001_init.sql @@ -0,0 +1,87 @@ +-- ©AngelaMos | 2026 +-- 0001_init.sql + +CREATE TABLE sources ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + title TEXT NOT NULL DEFAULT '', + url TEXT NOT NULL, + type TEXT NOT NULL, + weight REAL NOT NULL DEFAULT 1.0, + tags TEXT NOT NULL DEFAULT '', + enabled INTEGER NOT NULL DEFAULT 1 +); + +CREATE TABLE fetch_state ( + source_id INTEGER PRIMARY KEY REFERENCES sources(id) ON DELETE CASCADE, + etag TEXT NOT NULL DEFAULT '', + last_modified TEXT NOT NULL DEFAULT '', + last_fetched INTEGER NOT NULL DEFAULT 0, + last_status INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE articles ( + id INTEGER PRIMARY KEY, + source_id INTEGER NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + canonical_url TEXT NOT NULL UNIQUE, + content_hash TEXT NOT NULL UNIQUE, + title TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + body TEXT NOT NULL DEFAULT '', + author TEXT NOT NULL DEFAULT '', + published_at INTEGER NOT NULL DEFAULT 0, + fetched_at INTEGER NOT NULL DEFAULT 0 +); + +CREATE INDEX idx_articles_published ON articles(published_at); +CREATE INDEX idx_articles_source ON articles(source_id); + +CREATE TABLE cves ( + id TEXT PRIMARY KEY, + description TEXT NOT NULL DEFAULT '', + cvss_score REAL, + cvss_version TEXT NOT NULL DEFAULT '', + cvss_severity TEXT NOT NULL DEFAULT '', + cvss_vector TEXT NOT NULL DEFAULT '', + cwe TEXT NOT NULL DEFAULT '', + is_kev INTEGER NOT NULL DEFAULT 0, + kev_date_added TEXT NOT NULL DEFAULT '', + kev_ransomware INTEGER NOT NULL DEFAULT 0, + epss REAL, + epss_percentile REAL, + nvd_published TEXT NOT NULL DEFAULT '', + nvd_modified TEXT NOT NULL DEFAULT '', + enriched_at INTEGER NOT NULL DEFAULT 0, + enrich_status TEXT NOT NULL DEFAULT '' +); + +CREATE TABLE article_cves ( + article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + cve_id TEXT NOT NULL REFERENCES cves(id) ON DELETE CASCADE, + PRIMARY KEY (article_id, cve_id) +); + +CREATE TABLE clusters ( + id INTEGER PRIMARY KEY, + cluster_key TEXT NOT NULL UNIQUE, + first_seen INTEGER NOT NULL DEFAULT 0, + last_seen INTEGER NOT NULL DEFAULT 0, + size INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE cluster_members ( + cluster_id INTEGER NOT NULL REFERENCES clusters(id) ON DELETE CASCADE, + article_id INTEGER NOT NULL REFERENCES articles(id) ON DELETE CASCADE, + PRIMARY KEY (cluster_id, article_id) +); + +CREATE TABLE ai_notes ( + id INTEGER PRIMARY KEY, + cluster_id INTEGER NOT NULL REFERENCES clusters(id) ON DELETE CASCADE, + provider TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + why TEXT NOT NULL DEFAULT '', + angles_json TEXT NOT NULL DEFAULT '', + format TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL DEFAULT 0 +); diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go new file mode 100644 index 00000000..dc874de7 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go @@ -0,0 +1,149 @@ +// ©AngelaMos | 2026 +// store.go + +package store + +import ( + "database/sql" + "errors" + "fmt" + "strings" + + "modernc.org/sqlite" + sqlite3 "modernc.org/sqlite/lib" +) + +var ErrDuplicate = errors.New("store: article already exists") + +type Store struct { + db *sql.DB + version int +} + +type SourceInput struct { + Name string + Title string + URL string + Type string + Weight float64 + Tags []string + Enabled bool +} + +type SourceRow struct { + ID int64 + Name string + Title string + URL string + Type string + Weight float64 + Tags []string + Enabled bool +} + +type Article struct { + SourceID int64 + CanonicalURL string + ContentHash string + Title string + Summary string + Body string + Author string + PublishedAt int64 + FetchedAt int64 +} + +func Open(path string) (*Store, error) { + dsn := fmt.Sprintf("file:%s?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)", path) + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("open sqlite %s: %w", path, err) + } + if err := db.Ping(); err != nil { + _ = db.Close() + return nil, fmt.Errorf("ping sqlite %s: %w", path, err) + } + version, err := migrate(db) + if err != nil { + _ = db.Close() + return nil, err + } + return &Store{db: db, version: version}, nil +} + +func (s *Store) Close() error { return s.db.Close() } +func (s *Store) Version() int { return s.version } +func (s *Store) DB() *sql.DB { return s.db } + +func (s *Store) UpsertSource(in SourceInput) (int64, error) { + tags := strings.Join(in.Tags, ",") + var id int64 + err := s.db.QueryRow(` + INSERT INTO sources (name, title, url, type, weight, tags, enabled) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(name) DO UPDATE SET + title = excluded.title, url = excluded.url, type = excluded.type, + weight = excluded.weight, tags = excluded.tags, enabled = excluded.enabled + RETURNING id`, + in.Name, in.Title, in.URL, in.Type, in.Weight, tags, boolToInt(in.Enabled), + ).Scan(&id) + if err != nil { + return 0, fmt.Errorf("upsert source %q: %w", in.Name, err) + } + return id, nil +} + +func (s *Store) GetSourceByName(name string) (SourceRow, error) { + var r SourceRow + var tags string + var enabled int + err := s.db.QueryRow(` + SELECT id, name, title, url, type, weight, tags, enabled + FROM sources WHERE name = ?`, name, + ).Scan(&r.ID, &r.Name, &r.Title, &r.URL, &r.Type, &r.Weight, &tags, &enabled) + if err != nil { + return SourceRow{}, fmt.Errorf("get source %q: %w", name, err) + } + if tags != "" { + r.Tags = strings.Split(tags, ",") + } + r.Enabled = enabled != 0 + return r, nil +} + +func (s *Store) InsertArticle(a Article) (int64, error) { + res, err := s.db.Exec(` + INSERT INTO articles + (source_id, canonical_url, content_hash, title, summary, body, author, published_at, fetched_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + a.SourceID, a.CanonicalURL, a.ContentHash, a.Title, a.Summary, a.Body, + a.Author, a.PublishedAt, a.FetchedAt, + ) + if err != nil { + var se *sqlite.Error + if errors.As(err, &se) && se.Code() == sqlite3.SQLITE_CONSTRAINT_UNIQUE { + return 0, ErrDuplicate + } + return 0, fmt.Errorf("insert article %q: %w", a.CanonicalURL, err) + } + id, err := res.LastInsertId() + if err != nil { + return 0, fmt.Errorf("insert article %q: last insert id: %w", a.CanonicalURL, err) + } + return id, nil +} + +func (s *Store) CountArticles() (int, error) { + var n int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM articles`).Scan(&n); err != nil { + return 0, fmt.Errorf("count articles: %w", err) + } + return n, nil +} + +func boolToInt(b bool) int { + if b { + return 1 + } + return 0 +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store_test.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store_test.go new file mode 100644 index 00000000..0bdec31c --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store_test.go @@ -0,0 +1,116 @@ +// ©AngelaMos | 2026 +// store_test.go + +package store + +import ( + "errors" + "path/filepath" + "testing" +) + +func openTemp(t *testing.T) *Store { + t.Helper() + s, err := Open(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + return s +} + +func TestMigrateAppliesLatest(t *testing.T) { + s := openTemp(t) + if s.Version() < 1 { + t.Fatalf("schema version = %d, want >= 1", s.Version()) + } + var n int + if err := s.DB().QueryRow( + `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='articles'`, + ).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Error("articles table not created by migration") + } +} + +func TestMigrateIsIdempotent(t *testing.T) { + path := filepath.Join(t.TempDir(), "idem.db") + s1, err := Open(path) + if err != nil { + t.Fatal(err) + } + v1 := s1.Version() + _ = s1.Close() + + s2, err := Open(path) + if err != nil { + t.Fatalf("reopen: %v", err) + } + defer s2.Close() + if s2.Version() != v1 { + t.Errorf("reopen version = %d, want %d", s2.Version(), v1) + } +} + +func TestSourceRoundTrip(t *testing.T) { + s := openTemp(t) + id, err := s.UpsertSource(SourceInput{ + Name: "krebs", Title: "Krebs", URL: "https://krebsonsecurity.com/feed/", + Type: "rss", Weight: 1.0, Tags: []string{"news"}, Enabled: true, + }) + if err != nil { + t.Fatalf("UpsertSource: %v", err) + } + row, err := s.GetSourceByName("krebs") + if err != nil { + t.Fatalf("GetSourceByName: %v", err) + } + if row.ID != id || row.Weight != 1.0 || !row.Enabled || len(row.Tags) != 1 { + t.Errorf("round trip mismatch: %+v", row) + } + + id2, err := s.UpsertSource(SourceInput{ + Name: "krebs", Title: "Krebs Updated", URL: "https://krebsonsecurity.com/feed/", + Type: "rss", Weight: 0.9, Tags: []string{"news"}, Enabled: true, + }) + if err != nil { + t.Fatalf("re-upsert: %v", err) + } + if id2 != id { + t.Errorf("upsert should keep id %d, got %d", id, id2) + } + row2, _ := s.GetSourceByName("krebs") + if row2.Title != "Krebs Updated" || row2.Weight != 0.9 { + t.Errorf("upsert did not update fields: %+v", row2) + } +} + +func TestArticleUniqueConstraint(t *testing.T) { + s := openTemp(t) + srcID, err := s.UpsertSource(SourceInput{ + Name: "krebs", URL: "https://krebsonsecurity.com/feed/", Type: "rss", Enabled: true, + }) + if err != nil { + t.Fatal(err) + } + a := Article{ + SourceID: srcID, CanonicalURL: "https://krebsonsecurity.com/post-1", + ContentHash: "hash-1", Title: "Post 1", PublishedAt: 100, FetchedAt: 200, + } + if _, err := s.InsertArticle(a); err != nil { + t.Fatalf("first insert: %v", err) + } + if _, err := s.InsertArticle(a); !errors.Is(err, ErrDuplicate) { + t.Errorf("duplicate insert: got %v, want ErrDuplicate", err) + } + + n, err := s.CountArticles() + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("article count = %d, want 1 (dup rejected)", n) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/version/version.go b/PROJECTS/intermediate/security-news-scraper/internal/version/version.go new file mode 100644 index 00000000..6d79e67b --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/version/version.go @@ -0,0 +1,9 @@ +// ©AngelaMos | 2026 +// version.go + +package version + +const ( + Name = "nadezhda" + Version = "0.1.0-dev" +) diff --git a/PROJECTS/intermediate/security-news-scraper/justfile b/PROJECTS/intermediate/security-news-scraper/justfile new file mode 100644 index 00000000..85d7f3c8 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/justfile @@ -0,0 +1,33 @@ +# ©AngelaMos | 2026 +# justfile + +set shell := ["bash", "-cu"] + +binary := "nadezhda" + +default: + @just --list + +build: + go build -o {{binary}} ./cmd/nadezhda + +run *args: + go run ./cmd/nadezhda {{args}} + +test: + go test ./... + +vet: + go vet ./... + +fmt: + gofmt -w . + +lint: vet + test -z "$(gofmt -l .)" + +tidy: + go mod tidy + +clean: + rm -f {{binary}} *.db *.db-wal *.db-shm