From b41276004fb25ad984bc8b134e3f7771c29b2b56 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Sun, 5 Jul 2026 14:51:57 -0400 Subject: [PATCH] feat(nadezhda): M1 ingestion core (fetch, parse, normalize, dedup) Concurrent, rate-limited feed ingestion wired end to end: 'scrape' pulls live security news into the SQLite store, idempotent across re-runs. - internal/fetch: per-host token-bucket rate limiting, conditional GET (ETag/Last-Modified with 304 handling), retry+backoff that honors Retry-After and treats timeouts/cancellation as non-retryable, per-source deadline, honest User-Agent, 16MB body cap. robots.txt is enforced only via Client.Allowed() for the future HTML article-scrape path, not on subscribed feed fetches (user-directed retrieval; several publishers blanket-disallow generic bots yet serve a public feed). - internal/parse: gofeed RSS/Atom to a normalized Item; RFC1123Z/RFC822Z/ RFC3339 time fallbacks (CISA emits RFC822Z, not RFC1123Z as first researched). - internal/normalize: canonical URL (lowercase scheme+host, drop fragment, strip utm_*/gclid/fbclid/ref/mc_cid/mc_eid, drop trailing slash), title normalization, goquery HTML strip, sha256 content+title hashes. - internal/ingest: errgroup fan-out over sources, fail-soft per source, fetch -> parse -> normalize -> InsertArticle dedup via store.ErrDuplicate, plus fetch_state upsert for conditional GET. - store: migration 0002 adds articles.title_hash; GetFetchState/UpsertFetchState. - Trimmed testdata/feeds fixtures (3 items each) back golden parse tests; full suite is offline and passes under -race. Proven live: cold run 215 new across 7 sources, warm run 0 new (304s + dedup). Ctrl-C aborts scrape cleanly via signal-aware context. --- .../cmd/nadezhda/scrape.go | 127 ++++ .../cmd/nadezhda/stubs.go | 1 - .../intermediate/security-news-scraper/go.mod | 12 + .../intermediate/security-news-scraper/go.sum | 102 ++- .../internal/config/config.go | 61 +- .../internal/fetch/fetch.go | 235 +++++++ .../internal/fetch/fetch_test.go | 244 +++++++ .../internal/fetch/robots.go | 88 +++ .../internal/ingest/ingest.go | 162 +++++ .../internal/ingest/ingest_test.go | 199 ++++++ .../internal/normalize/normalize.go | 111 ++++ .../internal/normalize/normalize_test.go | 111 ++++ .../internal/parse/parse.go | 74 +++ .../internal/parse/parse_test.go | 105 +++ .../internal/store/fetch_state_test.go | 81 +++ .../migrations/0002_article_title_hash.sql | 6 + .../internal/store/store.go | 44 +- .../testdata/feeds/bleepingcomputer.xml | 62 ++ .../testdata/feeds/cisa.xml | 598 ++++++++++++++++++ .../testdata/feeds/darkreading.xml | 52 ++ .../testdata/feeds/krebs.xml | 245 +++++++ .../testdata/feeds/securityweek.xml | 90 +++ .../testdata/feeds/thehackernews.xml | 9 + .../testdata/feeds/theregister.xml | 48 ++ 24 files changed, 2839 insertions(+), 28 deletions(-) create mode 100644 PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/fetch/robots.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/parse/parse.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/parse/parse_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/fetch_state_test.go create mode 100644 PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0002_article_title_hash.sql create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/feeds/bleepingcomputer.xml create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/feeds/cisa.xml create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/feeds/darkreading.xml create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/feeds/krebs.xml create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/feeds/securityweek.xml create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/feeds/thehackernews.xml create mode 100644 PROJECTS/intermediate/security-news-scraper/testdata/feeds/theregister.xml diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go new file mode 100644 index 00000000..6d3c5587 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/scrape.go @@ -0,0 +1,127 @@ +// ©AngelaMos | 2026 +// scrape.go + +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "time" + + "github.com/spf13/cobra" + + "github.com/CarterPerez-dev/nadezhda/internal/fetch" + "github.com/CarterPerez-dev/nadezhda/internal/ingest" + "github.com/CarterPerez-dev/nadezhda/internal/source" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const ( + statusNotModified = "304" + statusError = "error" + statusOK = "ok" + dash = "-" +) + +var scrapeSource string + +var scrapeCmd = &cobra.Command{ + Use: "scrape", + Short: "Ingest all enabled sources once", + RunE: runScrape, +} + +func init() { + scrapeCmd.Flags().StringVar(&scrapeSource, "source", "", "ingest only this source by name") + rootCmd.AddCommand(scrapeCmd) +} + +func runScrape(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + srcs, err := source.Load(cfg.SourcesPath) + if err != nil { + return err + } + + targets, err := selectTargets(srcs, scrapeSource) + if err != nil { + return err + } + + st, err := store.Open(cfg.DBPath) + if err != nil { + return err + } + defer st.Close() + + fc := fetch.New(fetch.Options{ + UserAgent: cfg.Fetch.UserAgent, + PerHostRate: cfg.Fetch.PerHostRate, + PerHostBurst: cfg.Fetch.PerHostBurst, + Timeout: time.Duration(cfg.Fetch.TimeoutSeconds) * time.Second, + MaxRetries: cfg.Fetch.MaxRetries, + }) + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + summary, err := ingest.Run(ctx, fc, st, cfg, targets, time.Now()) + if err != nil { + return err + } + + printSummary(cmd, summary) + return nil +} + +func selectTargets(srcs []source.Source, only string) ([]source.Source, error) { + if only != "" { + for _, s := range srcs { + if s.Name == only { + return []source.Source{s}, nil + } + } + return nil, fmt.Errorf("scrape: unknown source %q", only) + } + return source.Enabled(srcs), nil +} + +func printSummary(cmd *cobra.Command, summary ingest.Summary) { + out := cmd.OutOrStdout() + fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s\n", "SOURCE", "STATUS", "PARSED", "NEW", "DUP", "ERR") + for _, r := range summary.Results { + fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s\n", + r.Name, status(r), count(r, r.Parsed), count(r, r.New), count(r, r.Duplicates), count(r, r.ItemErrors)) + } + newArticles, duplicates, failed := summary.Totals() + fmt.Fprintf(out, "\n%d new, %d duplicate across %d sources (%d failed)\n", + newArticles, duplicates, len(summary.Results), failed) + for _, r := range summary.Results { + if r.Err != nil { + fmt.Fprintf(out, " %s: %v\n", r.Name, r.Err) + } + } +} + +func status(r ingest.SourceResult) string { + switch { + case r.Err != nil: + return statusError + case r.NotModified: + return statusNotModified + default: + return statusOK + } +} + +func count(r ingest.SourceResult, n int) string { + if r.Err != nil || r.NotModified { + return dash + } + return fmt.Sprintf("%d", n) +} diff --git a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go index 2f9eb03e..3b9f1431 100644 --- a/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go +++ b/PROJECTS/intermediate/security-news-scraper/cmd/nadezhda/stubs.go @@ -21,7 +21,6 @@ func init() { short string milestone string }{ - {"scrape", "Ingest all enabled sources once", "milestone M1"}, {"list", "List stored articles with filters", "milestone M3"}, {"cve", "Show an enriched CVE and articles mentioning it", "milestone M3"}, {"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"}, diff --git a/PROJECTS/intermediate/security-news-scraper/go.mod b/PROJECTS/intermediate/security-news-scraper/go.mod index 0f7c5876..7a74c19b 100644 --- a/PROJECTS/intermediate/security-news-scraper/go.mod +++ b/PROJECTS/intermediate/security-news-scraper/go.mod @@ -3,20 +3,32 @@ module github.com/CarterPerez-dev/nadezhda go 1.25.0 require ( + github.com/PuerkitoBio/goquery v1.12.0 + github.com/mmcdole/gofeed v1.3.0 github.com/spf13/cobra v1.10.2 + github.com/temoto/robotstxt v1.1.2 + golang.org/x/sync v0.21.0 + golang.org/x/time v0.15.0 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.53.0 ) require ( + github.com/andybalholm/cascadia v1.3.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/spf13/pflag v1.0.9 // indirect + golang.org/x/net v0.52.0 // indirect golang.org/x/sys v0.44.0 // indirect + golang.org/x/text v0.35.0 // indirect modernc.org/libc v1.73.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/PROJECTS/intermediate/security-news-scraper/go.sum b/PROJECTS/intermediate/security-news-scraper/go.sum index 7c4811a2..d61e7395 100644 --- a/PROJECTS/intermediate/security-news-scraper/go.sum +++ b/PROJECTS/intermediate/security-news-scraper/go.sum @@ -1,6 +1,15 @@ +github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo= +github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -9,10 +18,23 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= +github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -20,16 +42,92 @@ github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= +github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ= golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go index cbe18b30..a8692f5f 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/config/config.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/config/config.go @@ -13,12 +13,13 @@ import ( const ( defaultDBPath = "nadezhda.db" - defaultUserAgent = "nadezhda/0.1 (+https://github.com/CarterPerez-dev/nadezhda)" - defaultPerHostRate = 0.5 - defaultPerHostBurst = 1 - defaultTimeoutSeconds = 25 - defaultWorkers = 8 - defaultMaxRetries = 3 + defaultUserAgent = "nadezhda/0.1 (+https://github.com/CarterPerez-dev/nadezhda)" + defaultPerHostRate = 0.5 + defaultPerHostBurst = 1 + defaultTimeoutSeconds = 25 + defaultSourceTimeoutSeconds = 90 + defaultWorkers = 8 + defaultMaxRetries = 3 defaultCacheTTLHours = 24 defaultNegativeTTLHours = 3 @@ -26,6 +27,8 @@ const ( defaultTitleJaccard = 0.6 defaultWindowHours = 72 + trackingUTMPrefix = "utm_*" + defaultHalfLifeHours = 48 defaultVelocityNorm = 0.5 @@ -48,13 +51,18 @@ const ( defaultClaudeModel = "claude-opus-4-8" ) +var defaultTrackingParams = []string{ + trackingUTMPrefix, "gclid", "fbclid", "ref", "mc_cid", "mc_eid", +} + type Fetch struct { - UserAgent string `yaml:"user_agent"` - PerHostRate float64 `yaml:"per_host_rate"` - PerHostBurst int `yaml:"per_host_burst"` - TimeoutSeconds int `yaml:"timeout_seconds"` - Workers int `yaml:"workers"` - MaxRetries int `yaml:"max_retries"` + UserAgent string `yaml:"user_agent"` + PerHostRate float64 `yaml:"per_host_rate"` + PerHostBurst int `yaml:"per_host_burst"` + TimeoutSeconds int `yaml:"timeout_seconds"` + SourceTimeoutSeconds int `yaml:"source_timeout_seconds"` + Workers int `yaml:"workers"` + MaxRetries int `yaml:"max_retries"` } type Enrich struct { @@ -63,8 +71,9 @@ type Enrich struct { } type Cluster struct { - TitleJaccard float64 `yaml:"title_jaccard_threshold"` - WindowHours int `yaml:"window_hours"` + TitleJaccard float64 `yaml:"title_jaccard_threshold"` + WindowHours int `yaml:"window_hours"` + TrackingParams []string `yaml:"tracking_params"` } type Weights struct { @@ -112,20 +121,22 @@ func Default() Config { return Config{ DBPath: defaultDBPath, Fetch: Fetch{ - UserAgent: defaultUserAgent, - PerHostRate: defaultPerHostRate, - PerHostBurst: defaultPerHostBurst, - TimeoutSeconds: defaultTimeoutSeconds, - Workers: defaultWorkers, - MaxRetries: defaultMaxRetries, + UserAgent: defaultUserAgent, + PerHostRate: defaultPerHostRate, + PerHostBurst: defaultPerHostBurst, + TimeoutSeconds: defaultTimeoutSeconds, + SourceTimeoutSeconds: defaultSourceTimeoutSeconds, + Workers: defaultWorkers, + MaxRetries: defaultMaxRetries, }, Enrich: Enrich{ CacheTTLHours: defaultCacheTTLHours, NegativeTTLHours: defaultNegativeTTLHours, }, Cluster: Cluster{ - TitleJaccard: defaultTitleJaccard, - WindowHours: defaultWindowHours, + TitleJaccard: defaultTitleJaccard, + WindowHours: defaultWindowHours, + TrackingParams: defaultTrackingParams, }, Rank: Rank{ HalfLifeHours: defaultHalfLifeHours, @@ -185,6 +196,12 @@ func (c Config) validate() error { if c.Fetch.PerHostBurst < 1 { return fmt.Errorf("config: fetch.per_host_burst must be >= 1, got %d", c.Fetch.PerHostBurst) } + if c.Fetch.TimeoutSeconds < 1 { + return fmt.Errorf("config: fetch.timeout_seconds must be >= 1, got %d", c.Fetch.TimeoutSeconds) + } + if c.Fetch.SourceTimeoutSeconds < 1 { + return fmt.Errorf("config: fetch.source_timeout_seconds must be >= 1, got %d", c.Fetch.SourceTimeoutSeconds) + } if c.Enrich.CacheTTLHours < 0 || c.Enrich.NegativeTTLHours < 0 { return fmt.Errorf("config: enrich TTLs must be >= 0, got cache=%d negative=%d", c.Enrich.CacheTTLHours, c.Enrich.NegativeTTLHours) } diff --git a/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch.go b/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch.go new file mode 100644 index 00000000..42419123 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch.go @@ -0,0 +1,235 @@ +// ©AngelaMos | 2026 +// fetch.go + +package fetch + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "sync" + "time" + + "golang.org/x/time/rate" +) + +const ( + headerUserAgent = "User-Agent" + headerAccept = "Accept" + headerIfNoneMatch = "If-None-Match" + headerIfModifiedSince = "If-Modified-Since" + headerETag = "ETag" + headerLastModified = "Last-Modified" + headerRetryAfter = "Retry-After" + + acceptFeed = "application/rss+xml, application/atom+xml, application/xml;q=0.9, */*;q=0.8" + + defaultBackoffBase = 500 * time.Millisecond + maxBodyBytes = 16 << 20 + maxRetryAfter = 60 * time.Second +) + +type Options struct { + UserAgent string + PerHostRate float64 + PerHostBurst int + Timeout time.Duration + MaxRetries int +} + +type Request struct { + URL string + ETag string + LastModified string +} + +type Result struct { + Status int + Body []byte + ETag string + LastModified string + NotModified bool +} + +type Client struct { + http *http.Client + ua string + rate rate.Limit + burst int + maxRetries int + backoffBase time.Duration + + mu sync.Mutex + limiters map[string]*rate.Limiter + + robots *robotsCache +} + +func New(opts Options) *Client { + c := &Client{ + http: &http.Client{Timeout: opts.Timeout}, + ua: opts.UserAgent, + rate: rate.Limit(opts.PerHostRate), + burst: opts.PerHostBurst, + maxRetries: opts.MaxRetries, + backoffBase: defaultBackoffBase, + limiters: make(map[string]*rate.Limiter), + } + c.robots = newRobotsCache(c) + return c +} + +func (c *Client) limiterFor(host string) *rate.Limiter { + c.mu.Lock() + defer c.mu.Unlock() + l, ok := c.limiters[host] + if !ok { + l = rate.NewLimiter(c.rate, c.burst) + c.limiters[host] = l + } + return l +} + +func (c *Client) Fetch(ctx context.Context, req Request) (Result, error) { + u, err := url.Parse(req.URL) + if err != nil { + return Result{}, fmt.Errorf("fetch: parse url %q: %w", req.URL, err) + } + + var lastErr error + var override time.Duration + for attempt := 0; attempt <= c.maxRetries; attempt++ { + if attempt > 0 { + delay := c.backoff(attempt) + if override > 0 { + delay = override + } + if err := sleep(ctx, delay); err != nil { + return Result{}, err + } + } + if err := c.limiterFor(u.Host).Wait(ctx); err != nil { + return Result{}, err + } + res, retryAfter, retry, err := c.do(ctx, req) + if err == nil { + return res, nil + } + lastErr = err + override = retryAfter + if !retry { + return Result{}, err + } + } + return Result{}, fmt.Errorf("fetch %s: exhausted %d retries: %w", req.URL, c.maxRetries, lastErr) +} + +func (c *Client) Allowed(ctx context.Context, rawURL string) (bool, error) { + u, err := url.Parse(rawURL) + if err != nil { + return false, fmt.Errorf("fetch: parse url %q: %w", rawURL, err) + } + return c.robots.allowed(ctx, u) +} + +func (c *Client) do(ctx context.Context, req Request) (Result, time.Duration, bool, error) { + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, req.URL, nil) + if err != nil { + return Result{}, 0, false, fmt.Errorf("fetch %s: build request: %w", req.URL, err) + } + httpReq.Header.Set(headerUserAgent, c.ua) + httpReq.Header.Set(headerAccept, acceptFeed) + if req.ETag != "" { + httpReq.Header.Set(headerIfNoneMatch, req.ETag) + } + if req.LastModified != "" { + httpReq.Header.Set(headerIfModifiedSince, req.LastModified) + } + + resp, err := c.http.Do(httpReq) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return Result{}, 0, false, fmt.Errorf("fetch %s: %w", req.URL, err) + } + return Result{}, 0, true, fmt.Errorf("fetch %s: %w", req.URL, err) + } + defer resp.Body.Close() + + switch { + case resp.StatusCode == http.StatusNotModified: + return Result{ + Status: resp.StatusCode, + NotModified: true, + ETag: req.ETag, + LastModified: req.LastModified, + }, 0, false, nil + case resp.StatusCode == http.StatusOK: + body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes)) + if err != nil { + return Result{}, 0, true, fmt.Errorf("fetch %s: read body: %w", req.URL, err) + } + return Result{ + Status: resp.StatusCode, + Body: body, + ETag: resp.Header.Get(headerETag), + LastModified: resp.Header.Get(headerLastModified), + }, 0, false, nil + case resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= http.StatusInternalServerError: + drain(resp.Body) + return Result{}, retryAfter(resp), true, fmt.Errorf("fetch %s: server status %d", req.URL, resp.StatusCode) + default: + drain(resp.Body) + return Result{}, 0, false, fmt.Errorf("fetch %s: status %d", req.URL, resp.StatusCode) + } +} + +func drain(body io.Reader) { + _, _ = io.Copy(io.Discard, io.LimitReader(body, maxBodyBytes)) +} + +func retryAfter(resp *http.Response) time.Duration { + v := resp.Header.Get(headerRetryAfter) + if v == "" { + return 0 + } + if secs, err := strconv.Atoi(v); err == nil { + d := time.Duration(secs) * time.Second + if d > maxRetryAfter { + return maxRetryAfter + } + if d < 0 { + return 0 + } + return d + } + if t, err := http.ParseTime(v); err == nil { + d := time.Until(t) + if d <= 0 { + return 0 + } + if d > maxRetryAfter { + return maxRetryAfter + } + return d + } + return 0 +} + +func (c *Client) backoff(attempt int) time.Duration { + return c.backoffBase * time.Duration(1<<(attempt-1)) +} + +func sleep(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch_test.go b/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch_test.go new file mode 100644 index 00000000..a94beefa --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/fetch/fetch_test.go @@ -0,0 +1,244 @@ +// ©AngelaMos | 2026 +// fetch_test.go + +package fetch + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" +) + +func testClient() *Client { + c := New(Options{ + UserAgent: "nadezhda-test/1.0", + PerHostRate: 1e6, + PerHostBurst: 1, + Timeout: 5 * time.Second, + MaxRetries: 3, + }) + c.backoffBase = time.Millisecond + return c +} + +func TestFetchOKCapturesValidators(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(headerETag, `"v1"`) + w.Header().Set(headerLastModified, "Wed, 01 Jul 2026 00:00:00 GMT") + _, _ = w.Write([]byte("")) + })) + defer srv.Close() + + res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"}) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if res.Status != http.StatusOK { + t.Errorf("status = %d, want 200", res.Status) + } + if string(res.Body) != "" { + t.Errorf("body = %q", res.Body) + } + if res.ETag != `"v1"` { + t.Errorf("etag = %q, want \"v1\"", res.ETag) + } + if res.LastModified == "" { + t.Error("last-modified not captured") + } +} + +func TestFetchConditionalGET(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get(headerIfNoneMatch) == `"v1"` { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set(headerETag, `"v1"`) + _, _ = w.Write([]byte("body")) + })) + defer srv.Close() + + c := testClient() + first, err := c.Fetch(context.Background(), Request{URL: srv.URL + "/feed"}) + if err != nil || first.Status != http.StatusOK { + t.Fatalf("first fetch: status=%d err=%v", first.Status, err) + } + + second, err := c.Fetch(context.Background(), Request{URL: srv.URL + "/feed", ETag: first.ETag}) + if err != nil { + t.Fatalf("second fetch: %v", err) + } + if !second.NotModified { + t.Error("expected NotModified on matching ETag") + } + if second.ETag != `"v1"` { + t.Errorf("304 should retain etag, got %q", second.ETag) + } +} + +func TestFetchRetriesServerError(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 { + w.WriteHeader(http.StatusInternalServerError) + return + } + _, _ = w.Write([]byte("ok")) + })) + defer srv.Close() + + res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"}) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if string(res.Body) != "ok" { + t.Errorf("body = %q, want ok", res.Body) + } + if calls.Load() != 2 { + t.Errorf("calls = %d, want 2 (one 500, one 200)", calls.Load()) + } +} + +func TestFetchDoesNotRetryClientError(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + _, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"}) + if err == nil { + t.Fatal("expected error on 404") + } + if calls.Load() != 1 { + t.Errorf("calls = %d, want 1 (no retry on 4xx)", calls.Load()) + } +} + +func TestFetch429IsRetried(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 { + w.WriteHeader(http.StatusTooManyRequests) + return + } + _, _ = w.Write([]byte("ok")) + })) + defer srv.Close() + + res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"}) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if string(res.Body) != "ok" || calls.Load() != 2 { + t.Errorf("body=%q calls=%d, want ok/2", res.Body, calls.Load()) + } +} + +func TestFetchTimeoutNotRetried(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + time.Sleep(100 * time.Millisecond) + })) + defer srv.Close() + + c := New(Options{ + UserAgent: "nadezhda-test/1.0", PerHostRate: 1e6, PerHostBurst: 1, + Timeout: 15 * time.Millisecond, MaxRetries: 3, + }) + c.backoffBase = time.Millisecond + + _, err := c.Fetch(context.Background(), Request{URL: srv.URL + "/feed"}) + if err == nil { + t.Fatal("expected timeout error") + } + if calls.Load() != 1 { + t.Errorf("calls = %d, want 1 (timeout must not be retried)", calls.Load()) + } +} + +func TestRetryAfterParsing(t *testing.T) { + cases := []struct { + header string + want time.Duration + }{ + {"", 0}, + {"5", 5 * time.Second}, + {"0", 0}, + {"-3", 0}, + {"9999", maxRetryAfter}, + {"garbage", 0}, + } + for _, tc := range cases { + resp := &http.Response{Header: http.Header{}} + if tc.header != "" { + resp.Header.Set(headerRetryAfter, tc.header) + } + if got := retryAfter(resp); got != tc.want { + t.Errorf("retryAfter(%q) = %s, want %s", tc.header, got, tc.want) + } + } +} + +func TestFeedFetchIgnoresRobots(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == robotsPath { + t.Error("Fetch must not request robots.txt on the feed path") + _, _ = w.Write([]byte("User-agent: *\nDisallow: /\n")) + return + } + _, _ = w.Write([]byte("feed body")) + })) + defer srv.Close() + + res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"}) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if string(res.Body) != "feed body" { + t.Errorf("body = %q, want feed body", res.Body) + } +} + +func TestAllowedRespectsDisallow(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == robotsPath { + _, _ = w.Write([]byte("User-agent: *\nDisallow: /article\n")) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + ok, err := testClient().Allowed(context.Background(), srv.URL+"/article/123") + if err != nil { + t.Fatalf("Allowed: %v", err) + } + if ok { + t.Error("expected disallowed for /article path") + } +} + +func TestAllowedPermitsOtherPaths(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == robotsPath { + _, _ = w.Write([]byte("User-agent: *\nDisallow: /admin\n")) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + ok, err := testClient().Allowed(context.Background(), srv.URL+"/article/123") + if err != nil { + t.Fatalf("Allowed: %v", err) + } + if !ok { + t.Error("expected allowed for non-admin path") + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/fetch/robots.go b/PROJECTS/intermediate/security-news-scraper/internal/fetch/robots.go new file mode 100644 index 00000000..f8aee1be --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/fetch/robots.go @@ -0,0 +1,88 @@ +// ©AngelaMos | 2026 +// robots.go + +package fetch + +import ( + "context" + "io" + "net/http" + "net/url" + "sync" + + "github.com/temoto/robotstxt" +) + +const ( + robotsPath = "/robots.txt" + maxRobotsBytes = 512 << 10 + robotsRootPath = "/" +) + +type robotsEntry struct { + once sync.Once + data *robotstxt.RobotsData +} + +type robotsCache struct { + client *Client + + mu sync.Mutex + entries map[string]*robotsEntry +} + +func newRobotsCache(client *Client) *robotsCache { + return &robotsCache{ + client: client, + entries: make(map[string]*robotsEntry), + } +} + +func (rc *robotsCache) allowed(ctx context.Context, u *url.URL) (bool, error) { + rc.mu.Lock() + e, ok := rc.entries[u.Host] + if !ok { + e = &robotsEntry{} + rc.entries[u.Host] = e + } + rc.mu.Unlock() + + e.once.Do(func() { + data, err := rc.load(ctx, u) + if err != nil { + data, _ = robotstxt.FromStatusAndBytes(http.StatusOK, nil) + } + e.data = data + }) + + path := u.EscapedPath() + if path == "" { + path = robotsRootPath + } + return e.data.FindGroup(rc.client.ua).Test(path), nil +} + +func (rc *robotsCache) load(ctx context.Context, u *url.URL) (*robotstxt.RobotsData, error) { + robotsURL := u.Scheme + "://" + u.Host + robotsPath + if err := rc.client.limiterFor(u.Host).Wait(ctx); err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, robotsURL, nil) + if err != nil { + return nil, err + } + req.Header.Set(headerUserAgent, rc.client.ua) + + resp, err := rc.client.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(io.LimitReader(resp.Body, maxRobotsBytes)) + if err != nil { + return nil, err + } + return robotstxt.FromStatusAndBytes(resp.StatusCode, body) +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go b/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go new file mode 100644 index 00000000..215dc79b --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest.go @@ -0,0 +1,162 @@ +// ©AngelaMos | 2026 +// ingest.go + +package ingest + +import ( + "bytes" + "context" + "errors" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/fetch" + "github.com/CarterPerez-dev/nadezhda/internal/normalize" + "github.com/CarterPerez-dev/nadezhda/internal/parse" + "github.com/CarterPerez-dev/nadezhda/internal/source" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +type SourceResult struct { + Name string + Parsed int + New int + Duplicates int + ItemErrors int + NotModified bool + Err error +} + +type Summary struct { + Results []SourceResult +} + +func (s Summary) Totals() (newArticles, duplicates, failed int) { + for _, r := range s.Results { + newArticles += r.New + duplicates += r.Duplicates + if r.Err != nil { + failed++ + } + } + return newArticles, duplicates, failed +} + +func Run(ctx context.Context, fc *fetch.Client, st *store.Store, cfg config.Config, targets []source.Source, now time.Time) (Summary, error) { + results := make([]SourceResult, len(targets)) + + workers := cfg.Fetch.Workers + if workers < 1 { + workers = 1 + } + + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(workers) + + for i, src := range targets { + results[i].Name = src.Name + g.Go(func() error { + process(gctx, fc, st, cfg, src, now, &results[i]) + return nil + }) + } + if err := g.Wait(); err != nil { + return Summary{Results: results}, err + } + return Summary{Results: results}, nil +} + +func process(ctx context.Context, fc *fetch.Client, st *store.Store, cfg config.Config, src source.Source, now time.Time, out *SourceResult) { + ctx, cancel := context.WithTimeout(ctx, time.Duration(cfg.Fetch.SourceTimeoutSeconds)*time.Second) + defer cancel() + + id, err := st.UpsertSource(store.SourceInput{ + Name: src.Name, Title: src.Title, URL: src.URL, Type: string(src.Type), + Weight: src.Weight, Tags: src.Tags, Enabled: src.Enabled, + }) + if err != nil { + out.Err = err + return + } + + prev, _, err := st.GetFetchState(id) + if err != nil { + out.Err = err + return + } + + res, err := fc.Fetch(ctx, fetch.Request{ + URL: src.URL, ETag: prev.ETag, LastModified: prev.LastModified, + }) + if err != nil { + out.Err = err + return + } + + if res.NotModified { + out.NotModified = true + out.Err = st.UpsertFetchState(id, store.FetchState{ + ETag: prev.ETag, LastModified: prev.LastModified, + LastFetched: now.Unix(), LastStatus: int64(res.Status), + }) + return + } + + items, err := parse.Feed(bytes.NewReader(res.Body)) + if err != nil { + out.Err = err + return + } + out.Parsed = len(items) + + for _, it := range items { + storeItem(st, cfg, id, it, now, out) + } + + if err := st.UpsertFetchState(id, store.FetchState{ + ETag: res.ETag, LastModified: res.LastModified, + LastFetched: now.Unix(), LastStatus: int64(res.Status), + }); err != nil && out.Err == nil { + out.Err = err + } +} + +func storeItem(st *store.Store, cfg config.Config, sourceID int64, it parse.Item, now time.Time, out *SourceResult) { + if it.Link == "" { + out.ItemErrors++ + return + } + canonical, err := normalize.CanonicalURL(it.Link, cfg.Cluster.TrackingParams) + if err != nil { + out.ItemErrors++ + return + } + + var publishedAt int64 + if !it.Published.IsZero() { + publishedAt = it.Published.Unix() + } + + _, err = st.InsertArticle(store.Article{ + SourceID: sourceID, + CanonicalURL: canonical, + ContentHash: normalize.ContentHash(canonical), + TitleHash: normalize.TitleHash(normalize.NormalizeTitle(it.Title)), + Title: it.Title, + Summary: normalize.StripHTML(it.Summary), + Body: normalize.StripHTML(it.Body), + Author: it.Author, + PublishedAt: publishedAt, + FetchedAt: now.Unix(), + }) + switch { + case err == nil: + out.New++ + case errors.Is(err, store.ErrDuplicate): + out.Duplicates++ + default: + out.ItemErrors++ + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest_test.go b/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest_test.go new file mode 100644 index 00000000..0aa0f3e1 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest_test.go @@ -0,0 +1,199 @@ +// ©AngelaMos | 2026 +// ingest_test.go + +package ingest + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/CarterPerez-dev/nadezhda/internal/config" + "github.com/CarterPerez-dev/nadezhda/internal/fetch" + "github.com/CarterPerez-dev/nadezhda/internal/normalize" + "github.com/CarterPerez-dev/nadezhda/internal/source" + "github.com/CarterPerez-dev/nadezhda/internal/store" +) + +const fixture = "../../testdata/feeds/thehackernews.xml" + +func loadFixture(t *testing.T) []byte { + t.Helper() + b, err := os.ReadFile(fixture) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + return b +} + +func newStore(t *testing.T) *store.Store { + t.Helper() + st, err := store.Open(filepath.Join(t.TempDir(), "ingest.db")) + if err != nil { + t.Fatalf("open store: %v", err) + } + t.Cleanup(func() { _ = st.Close() }) + return st +} + +func newClient() *fetch.Client { + return fetch.New(fetch.Options{ + UserAgent: "nadezhda-test/1.0", + PerHostRate: 1e6, + PerHostBurst: 1, + Timeout: 5 * time.Second, + MaxRetries: 2, + }) +} + +func target(url string) []source.Source { + return []source.Source{{ + Name: "test", Title: "Test Feed", URL: url, + Type: source.KindRSS, Weight: 1.0, Tags: []string{"news"}, Enabled: true, + }} +} + +func TestRunIngestsAndDedups(t *testing.T) { + body := loadFixture(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(body) + })) + defer srv.Close() + + st := newStore(t) + cfg := config.Default() + targets := target(srv.URL + "/feed") + now := time.Unix(1_800_000_000, 0) + + first, err := Run(context.Background(), newClient(), st, cfg, targets, now) + if err != nil { + t.Fatalf("first Run: %v", err) + } + r0 := first.Results[0] + if r0.Err != nil { + t.Fatalf("source error: %v", r0.Err) + } + if r0.New != 3 || r0.Parsed != 3 { + t.Fatalf("first run: parsed=%d new=%d, want 3/3", r0.Parsed, r0.New) + } + + count, err := st.CountArticles() + if err != nil { + t.Fatal(err) + } + if count != 3 { + t.Fatalf("stored articles = %d, want 3", count) + } + + var title, canonical, contentHash, titleHash string + if err := st.DB().QueryRow( + `SELECT title, canonical_url, content_hash, title_hash FROM articles LIMIT 1`, + ).Scan(&title, &canonical, &contentHash, &titleHash); err != nil { + t.Fatal(err) + } + if contentHash != normalize.ContentHash(canonical) { + t.Errorf("content_hash = %q, want sha256(canonical)", contentHash) + } + if titleHash != normalize.TitleHash(normalize.NormalizeTitle(title)) { + t.Errorf("title_hash = %q, want sha256(normalized title)", titleHash) + } + + second, err := Run(context.Background(), newClient(), st, cfg, targets, now) + if err != nil { + t.Fatalf("second Run: %v", err) + } + r1 := second.Results[0] + if r1.New != 0 || r1.Duplicates != 3 { + t.Fatalf("second run: new=%d dup=%d, want 0/3", r1.New, r1.Duplicates) + } + + count, _ = st.CountArticles() + if count != 3 { + t.Fatalf("after re-run stored = %d, want 3 (idempotent)", count) + } +} + +func TestRunConditionalNotModified(t *testing.T) { + body := loadFixture(t) + const etag = `"abc123"` + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("If-None-Match") == etag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", etag) + _, _ = w.Write(body) + })) + defer srv.Close() + + st := newStore(t) + cfg := config.Default() + targets := target(srv.URL + "/feed") + now := time.Unix(1_800_000_000, 0) + + first, err := Run(context.Background(), newClient(), st, cfg, targets, now) + if err != nil { + t.Fatalf("first Run: %v", err) + } + if first.Results[0].New != 3 { + t.Fatalf("first run new = %d, want 3", first.Results[0].New) + } + var storedETag string + if err := st.DB().QueryRow(`SELECT etag FROM fetch_state LIMIT 1`).Scan(&storedETag); err != nil { + t.Fatal(err) + } + if storedETag != etag { + t.Errorf("persisted etag = %q, want %q", storedETag, etag) + } + + second, err := Run(context.Background(), newClient(), st, cfg, targets, now) + if err != nil { + t.Fatalf("second Run: %v", err) + } + r := second.Results[0] + if !r.NotModified { + t.Errorf("expected NotModified on second run, got %+v", r) + } + if r.New != 0 { + t.Errorf("new = %d on 304, want 0", r.New) + } +} + +func TestRunFailsSoftOnBadSource(t *testing.T) { + body := loadFixture(t) + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(body) + })) + defer good.Close() + bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer bad.Close() + + st := newStore(t) + cfg := config.Default() + targets := []source.Source{ + {Name: "bad", URL: bad.URL + "/feed", Type: source.KindRSS, Weight: 1, Enabled: true}, + {Name: "good", URL: good.URL + "/feed", Type: source.KindRSS, Weight: 1, Enabled: true}, + } + now := time.Unix(1_800_000_000, 0) + + summary, err := Run(context.Background(), newClient(), st, cfg, targets, now) + if err != nil { + t.Fatalf("Run returned error, should fail soft: %v", err) + } + byName := map[string]SourceResult{} + for _, r := range summary.Results { + byName[r.Name] = r + } + if byName["bad"].Err == nil { + t.Error("bad source should record an error") + } + if byName["good"].New != 3 { + t.Errorf("good source new = %d, want 3 (bad source must not abort run)", byName["good"].New) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize.go b/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize.go new file mode 100644 index 00000000..47fd8c7e --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize.go @@ -0,0 +1,111 @@ +// ©AngelaMos | 2026 +// normalize.go + +package normalize + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "net/url" + "strings" + "unicode" + + "github.com/PuerkitoBio/goquery" +) + +const ( + wildcardSuffix = "*" + trailingSlash = "/" +) + +func CanonicalURL(raw string, trackingParams []string) (string, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil { + return "", fmt.Errorf("normalize: parse url %q: %w", raw, err) + } + if u.Host == "" { + return "", fmt.Errorf("normalize: url %q has no host", raw) + } + + u.Scheme = strings.ToLower(u.Scheme) + u.Host = strings.ToLower(u.Host) + u.Fragment = "" + u.RawFragment = "" + + if q := u.Query(); len(q) > 0 { + for key := range q { + if isTracking(key, trackingParams) { + q.Del(key) + } + } + u.RawQuery = q.Encode() + } + + u.Path = strings.TrimRight(u.Path, trailingSlash) + if u.RawPath != "" { + u.RawPath = strings.TrimRight(u.RawPath, trailingSlash) + } + + return u.String(), nil +} + +func isTracking(key string, trackingParams []string) bool { + lowered := strings.ToLower(key) + for _, p := range trackingParams { + if strings.HasSuffix(p, wildcardSuffix) { + if strings.HasPrefix(lowered, strings.TrimSuffix(p, wildcardSuffix)) { + return true + } + continue + } + if lowered == p { + return true + } + } + return false +} + +func NormalizeTitle(title string) string { + var b strings.Builder + b.Grow(len(title)) + prevSpace := false + for _, r := range strings.ToLower(title) { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + b.WriteRune(r) + prevSpace = false + continue + } + if !prevSpace { + b.WriteByte(' ') + prevSpace = true + } + } + return strings.TrimSpace(b.String()) +} + +func StripHTML(s string) string { + doc, err := goquery.NewDocumentFromReader(strings.NewReader(s)) + if err != nil { + return collapseWhitespace(s) + } + doc.Find("script,style").Remove() + return collapseWhitespace(doc.Text()) +} + +func collapseWhitespace(s string) string { + return strings.Join(strings.Fields(s), " ") +} + +func ContentHash(canonicalURL string) string { + return sha256Hex(canonicalURL) +} + +func TitleHash(normalizedTitle string) string { + return sha256Hex(normalizedTitle) +} + +func sha256Hex(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize_test.go b/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize_test.go new file mode 100644 index 00000000..364e5039 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/normalize/normalize_test.go @@ -0,0 +1,111 @@ +// ©AngelaMos | 2026 +// normalize_test.go + +package normalize + +import "testing" + +var params = []string{"utm_*", "gclid", "fbclid", "ref", "mc_cid", "mc_eid"} + +func TestCanonicalURL(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"lowercase host+scheme", "HTTPS://Example.COM/Path", "https://example.com/Path"}, + {"drop fragment", "https://example.com/a#section", "https://example.com/a"}, + {"strip utm params", "https://example.com/a?utm_source=x&utm_medium=y&id=7", "https://example.com/a?id=7"}, + {"strip gclid fbclid ref", "https://example.com/a?gclid=1&fbclid=2&ref=z&keep=1", "https://example.com/a?keep=1"}, + {"strip mailchimp", "https://example.com/a?mc_cid=1&mc_eid=2", "https://example.com/a"}, + {"drop trailing slash", "https://example.com/a/b/", "https://example.com/a/b"}, + {"root trailing slash", "https://example.com/", "https://example.com"}, + {"sorted query", "https://example.com/a?b=2&a=1", "https://example.com/a?a=1&b=2"}, + {"tracking case insensitive", "https://example.com/a?UTM_Source=x&id=1", "https://example.com/a?id=1"}, + {"path preserved case", "https://example.com/Foo/Bar", "https://example.com/Foo/Bar"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := CanonicalURL(tc.in, params) + if err != nil { + t.Fatalf("CanonicalURL(%q): %v", tc.in, err) + } + if got != tc.want { + t.Errorf("CanonicalURL(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +func TestCanonicalURLCollapsesTracking(t *testing.T) { + a, _ := CanonicalURL("https://example.com/story?utm_source=twitter", params) + b, _ := CanonicalURL("https://example.com/story", params) + if a != b { + t.Errorf("tracking variants did not collapse: %q vs %q", a, b) + } + if ContentHash(a) != ContentHash(b) { + t.Error("content hashes differ for tracking variants") + } +} + +func TestCanonicalURLErrors(t *testing.T) { + for _, in := range []string{"", "not a url", "/relative/only"} { + if _, err := CanonicalURL(in, params); err == nil { + t.Errorf("CanonicalURL(%q) expected error", in) + } + } +} + +func TestNormalizeTitle(t *testing.T) { + cases := []struct { + in string + want string + }{ + {"CVE-2021-44228: Log4Shell!", "cve 2021 44228 log4shell"}, + {" Multiple Spaces ", "multiple spaces"}, + {"Punctuation, and; stuff.", "punctuation and stuff"}, + {"Café déjà vu", "café déjà vu"}, + {"", ""}, + } + for _, tc := range cases { + if got := NormalizeTitle(tc.in); got != tc.want { + t.Errorf("NormalizeTitle(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestNormalizeTitleStableHash(t *testing.T) { + a := TitleHash(NormalizeTitle("Breaking: Big Hack!!!")) + b := TitleHash(NormalizeTitle("breaking big hack")) + if a != b { + t.Error("normalized-title hashes differ for equivalent titles") + } +} + +func TestStripHTML(t *testing.T) { + cases := []struct { + in string + want string + }{ + {"

Hello world

", "Hello world"}, + {"
a
\n
b
", "a b"}, + {"visible", "visible"}, + {"plain text", "plain text"}, + {"link and text", "link and text"}, + } + for _, tc := range cases { + if got := StripHTML(tc.in); got != tc.want { + t.Errorf("StripHTML(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestContentHashDeterministic(t *testing.T) { + const u = "https://example.com/a" + if ContentHash(u) != ContentHash(u) { + t.Error("ContentHash not deterministic") + } + if len(ContentHash(u)) != 64 { + t.Errorf("ContentHash length = %d, want 64 hex chars", len(ContentHash(u))) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/parse/parse.go b/PROJECTS/intermediate/security-news-scraper/internal/parse/parse.go new file mode 100644 index 00000000..c8d02b8a --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/parse/parse.go @@ -0,0 +1,74 @@ +// ©AngelaMos | 2026 +// parse.go + +package parse + +import ( + "fmt" + "io" + "strings" + "time" + + "github.com/mmcdole/gofeed" +) + +type Item struct { + Title string + Link string + Summary string + Body string + Author string + Published time.Time + Categories []string +} + +var timeLayouts = []string{time.RFC1123Z, time.RFC1123, time.RFC822Z, time.RFC822, time.RFC3339} + +func Feed(r io.Reader) ([]Item, error) { + feed, err := gofeed.NewParser().Parse(r) + if err != nil { + return nil, fmt.Errorf("parse: %w", err) + } + items := make([]Item, 0, len(feed.Items)) + for _, it := range feed.Items { + items = append(items, Item{ + Title: strings.TrimSpace(it.Title), + Link: strings.TrimSpace(it.Link), + Summary: it.Description, + Body: it.Content, + Author: author(it), + Published: published(it), + Categories: it.Categories, + }) + } + return items, nil +} + +func author(it *gofeed.Item) string { + if it.Author != nil && it.Author.Name != "" { + return it.Author.Name + } + if len(it.Authors) > 0 { + return it.Authors[0].Name + } + return "" +} + +func published(it *gofeed.Item) time.Time { + if it.PublishedParsed != nil { + return it.PublishedParsed.UTC() + } + if it.UpdatedParsed != nil { + return it.UpdatedParsed.UTC() + } + raw := strings.TrimSpace(it.Published) + if raw == "" { + raw = strings.TrimSpace(it.Updated) + } + for _, layout := range timeLayouts { + if t, err := time.Parse(layout, raw); err == nil { + return t.UTC() + } + } + return time.Time{} +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/parse/parse_test.go b/PROJECTS/intermediate/security-news-scraper/internal/parse/parse_test.go new file mode 100644 index 00000000..4c21c79e --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/parse/parse_test.go @@ -0,0 +1,105 @@ +// ©AngelaMos | 2026 +// parse_test.go + +package parse + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +const feedsDir = "../../testdata/feeds" + +func loadFeed(t *testing.T, name string) []Item { + t.Helper() + f, err := os.Open(filepath.Join(feedsDir, name)) + if err != nil { + t.Fatalf("open fixture %s: %v", name, err) + } + t.Cleanup(func() { _ = f.Close() }) + items, err := Feed(f) + if err != nil { + t.Fatalf("Feed(%s): %v", name, err) + } + return items +} + +func TestFeedGolden(t *testing.T) { + cases := []struct { + file string + wantItems int + fullBody bool + }{ + {"krebs.xml", 3, true}, + {"theregister.xml", 3, true}, + {"thehackernews.xml", 3, false}, + {"bleepingcomputer.xml", 3, false}, + {"securityweek.xml", 3, false}, + {"darkreading.xml", 3, false}, + {"cisa.xml", 3, false}, + } + for _, tc := range cases { + t.Run(tc.file, func(t *testing.T) { + items := loadFeed(t, tc.file) + if len(items) != tc.wantItems { + t.Fatalf("items = %d, want %d", len(items), tc.wantItems) + } + for i, it := range items { + if it.Title == "" { + t.Errorf("item %d: empty title", i) + } + if it.Link == "" { + t.Errorf("item %d: empty link", i) + } + if it.Published.IsZero() { + t.Errorf("item %d (%q): published time did not parse", i, it.Title) + } + if tc.fullBody && it.Body == "" { + t.Errorf("item %d (%q): expected full body, got empty", i, it.Title) + } + } + }) + } +} + +func TestFeedExactPublishedTime(t *testing.T) { + cases := []struct { + file string + raw string + }{ + {"thehackernews.xml", "Sat, 04 Jul 2026 18:17:53 +0530"}, + {"krebs.xml", "Thu, 02 Jul 2026 19:27:33 +0000"}, + } + for _, tc := range cases { + t.Run(tc.file, func(t *testing.T) { + want, err := time.Parse(time.RFC1123Z, tc.raw) + if err != nil { + t.Fatalf("parse expected time: %v", err) + } + items := loadFeed(t, tc.file) + if !items[0].Published.Equal(want) { + t.Errorf("published = %s, want %s", items[0].Published, want.UTC()) + } + }) + } +} + +func TestFeedExactTitles(t *testing.T) { + cases := []struct { + file string + title string + }{ + {"krebs.xml", "FBI Seizes NetNut Proxy Platform, Popa Botnet"}, + {"thehackernews.xml", "U.S. Government Entity Paid Kairos $1 Million in Data-Theft Extortion Case"}, + } + for _, tc := range cases { + t.Run(tc.file, func(t *testing.T) { + items := loadFeed(t, tc.file) + if items[0].Title != tc.title { + t.Errorf("first title = %q, want %q", items[0].Title, tc.title) + } + }) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/fetch_state_test.go b/PROJECTS/intermediate/security-news-scraper/internal/store/fetch_state_test.go new file mode 100644 index 00000000..1e89e2f1 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/fetch_state_test.go @@ -0,0 +1,81 @@ +// ©AngelaMos | 2026 +// fetch_state_test.go + +package store + +import "testing" + +func seedSource(t *testing.T, s *Store) int64 { + t.Helper() + id, err := s.UpsertSource(SourceInput{ + Name: "krebs", Title: "Krebs", URL: "https://krebsonsecurity.com/feed/", + Type: "rss", Weight: 1.0, Tags: []string{"news"}, Enabled: true, + }) + if err != nil { + t.Fatalf("UpsertSource: %v", err) + } + return id +} + +func TestFetchStateMissing(t *testing.T) { + s := openTemp(t) + id := seedSource(t, s) + fs, ok, err := s.GetFetchState(id) + if err != nil { + t.Fatalf("GetFetchState: %v", err) + } + if ok { + t.Error("expected no fetch_state for fresh source") + } + if fs != (FetchState{}) { + t.Errorf("expected zero FetchState, got %+v", fs) + } +} + +func TestFetchStateRoundTrip(t *testing.T) { + s := openTemp(t) + id := seedSource(t, s) + + want := FetchState{ETag: `"v1"`, LastModified: "Wed, 01 Jul 2026 00:00:00 GMT", LastFetched: 1700, LastStatus: 200} + if err := s.UpsertFetchState(id, want); err != nil { + t.Fatalf("UpsertFetchState: %v", err) + } + + got, ok, err := s.GetFetchState(id) + if err != nil { + t.Fatalf("GetFetchState: %v", err) + } + if !ok { + t.Fatal("expected fetch_state to exist") + } + if got != want { + t.Errorf("round-trip = %+v, want %+v", got, want) + } + + updated := FetchState{ETag: `"v2"`, LastModified: "Thu, 02 Jul 2026 00:00:00 GMT", LastFetched: 1800, LastStatus: 304} + if err := s.UpsertFetchState(id, updated); err != nil { + t.Fatalf("second UpsertFetchState: %v", err) + } + got, _, _ = s.GetFetchState(id) + if got != updated { + t.Errorf("after update = %+v, want %+v", got, updated) + } +} + +func TestInsertArticleStoresTitleHash(t *testing.T) { + s := openTemp(t) + id := seedSource(t, s) + if _, err := s.InsertArticle(Article{ + SourceID: id, CanonicalURL: "https://example.com/a", ContentHash: "hash1", + TitleHash: "thash1", Title: "A", + }); err != nil { + t.Fatalf("InsertArticle: %v", err) + } + var th string + if err := s.DB().QueryRow(`SELECT title_hash FROM articles WHERE content_hash = ?`, "hash1").Scan(&th); err != nil { + t.Fatal(err) + } + if th != "thash1" { + t.Errorf("title_hash = %q, want thash1", th) + } +} diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0002_article_title_hash.sql b/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0002_article_title_hash.sql new file mode 100644 index 00000000..85d0c17f --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/migrations/0002_article_title_hash.sql @@ -0,0 +1,6 @@ +-- ©AngelaMos | 2026 +-- 0002_article_title_hash.sql + +ALTER TABLE articles ADD COLUMN title_hash TEXT NOT NULL DEFAULT ''; + +CREATE INDEX idx_articles_title_hash ON articles(title_hash); diff --git a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go index dc874de7..a19dddc7 100644 --- a/PROJECTS/intermediate/security-news-scraper/internal/store/store.go +++ b/PROJECTS/intermediate/security-news-scraper/internal/store/store.go @@ -45,6 +45,7 @@ type Article struct { SourceID int64 CanonicalURL string ContentHash string + TitleHash string Title string Summary string Body string @@ -53,6 +54,13 @@ type Article struct { FetchedAt int64 } +type FetchState struct { + ETag string + LastModified string + LastFetched int64 + LastStatus int64 +} + func Open(path string) (*Store, error) { dsn := fmt.Sprintf("file:%s?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)", path) db, err := sql.Open("sqlite", dsn) @@ -114,9 +122,9 @@ func (s *Store) GetSourceByName(name string) (SourceRow, error) { func (s *Store) InsertArticle(a Article) (int64, error) { res, err := s.db.Exec(` INSERT INTO articles - (source_id, canonical_url, content_hash, title, summary, body, author, published_at, fetched_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, - a.SourceID, a.CanonicalURL, a.ContentHash, a.Title, a.Summary, a.Body, + (source_id, canonical_url, content_hash, title_hash, title, summary, body, author, published_at, fetched_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + a.SourceID, a.CanonicalURL, a.ContentHash, a.TitleHash, a.Title, a.Summary, a.Body, a.Author, a.PublishedAt, a.FetchedAt, ) if err != nil { @@ -141,6 +149,36 @@ func (s *Store) CountArticles() (int, error) { return n, nil } +func (s *Store) GetFetchState(sourceID int64) (FetchState, bool, error) { + var fs FetchState + err := s.db.QueryRow(` + SELECT etag, last_modified, last_fetched, last_status + FROM fetch_state WHERE source_id = ?`, sourceID, + ).Scan(&fs.ETag, &fs.LastModified, &fs.LastFetched, &fs.LastStatus) + if errors.Is(err, sql.ErrNoRows) { + return FetchState{}, false, nil + } + if err != nil { + return FetchState{}, false, fmt.Errorf("get fetch_state %d: %w", sourceID, err) + } + return fs, true, nil +} + +func (s *Store) UpsertFetchState(sourceID int64, fs FetchState) error { + _, err := s.db.Exec(` + INSERT INTO fetch_state (source_id, etag, last_modified, last_fetched, last_status) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(source_id) DO UPDATE SET + etag = excluded.etag, last_modified = excluded.last_modified, + last_fetched = excluded.last_fetched, last_status = excluded.last_status`, + sourceID, fs.ETag, fs.LastModified, fs.LastFetched, fs.LastStatus, + ) + if err != nil { + return fmt.Errorf("upsert fetch_state %d: %w", sourceID, err) + } + return nil +} + func boolToInt(b bool) int { if b { return 1 diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/bleepingcomputer.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/bleepingcomputer.xml new file mode 100644 index 00000000..311dc127 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/bleepingcomputer.xml @@ -0,0 +1,62 @@ + + + + + BleepingComputer + + https://www.bleepingcomputer.com/ + BleepingComputer - All Stories + Sun, 05 Jul 2026 16:55:00 GMT + https://www.bleepingcomputer.com/ + en + + + + Flipper Zero firmware development continues with community help + https://www.bleepingcomputer.com/news/security/flipper-zero-firmware-development-continues-with-community-help/ + Sun, 05 Jul 2026 10:14:52 -0400 + Bill Toulas + + + + + + https://www.bleepingcomputer.com/news/security/flipper-zero-firmware-development-continues-with-community-help/ + + + + JadePuffer ransomware used AI agent to automate entire attack + https://www.bleepingcomputer.com/news/security/jadepuffer-ransomware-used-ai-agent-to-automate-entire-attack/ + Sat, 04 Jul 2026 10:16:38 -0400 + Bill Toulas + + + + + + + + https://www.bleepingcomputer.com/news/security/jadepuffer-ransomware-used-ai-agent-to-automate-entire-attack/ + + + + NetNut proxy network disrupted, 2 million infected devices cut off + https://www.bleepingcomputer.com/news/security/netnut-proxy-network-disrupted-2-million-infected-devices-cut-off/ + Fri, 03 Jul 2026 13:50:04 -0400 + Ionut Ilascu + + + + + + https://www.bleepingcomputer.com/news/security/netnut-proxy-network-disrupted-2-million-infected-devices-cut-off/ + + + + + diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/cisa.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/cisa.xml new file mode 100644 index 00000000..3c9971e4 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/cisa.xml @@ -0,0 +1,598 @@ + + + + All CISA Advisories + https://www.cisa.gov/ + + en + + + CubeSpace CW0057 Reaction Wheel + https://www.cisa.gov/news-events/ics-advisories/icsa-26-183-02 + <p><a href="https://github.com/cisagov/CSAF/blob/develop/csaf_files/OT/white/2026/icsa-26-183-02.json"><strong>View CSAF</strong></a></p> +<h2>Summary</h2> +<p><strong>Successful exploitation of this vulnerability could allow an attacker to upload arbitrary malicious firmware to the device.</strong></p> +<p>The following versions of CubeSpace CW0057 Reaction Wheel are affected:</p> +<ul> +<li>CW0057 Reaction Wheel</li> +</ul> +<div class="csaf-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS</th> +<th role="columnheader">Vendor</th> +<th role="columnheader">Equipment</th> +<th role="columnheader">Vulnerabilities</th> +</tr> +</thead> +<tbody> +<tr> +<td>v3 6.1</td> +<td>CubeSpace</td> +<td>CubeSpace CW0057 Reaction Wheel</td> +<td>Improper Verification of Cryptographic Signature</td> +</tr> +</tbody> +</table> +</div> +<h3>Background</h3> +<ul> +<li><strong>Critical Infrastructure Sectors: </strong>Communications</li> +<li><strong>Countries/Areas Deployed: </strong>Worldwide</li> +<li><strong>Company Headquarters Location: </strong>South Africa</li> +</ul> +<hr> +<h2>Vulnerabilities</h2> +<div class="csaf-accordion"> +<p><a class="csaf-accordion-toggle-all" href="#">Expand All +</a></p> +<div class="csaf-accordion-item"> +<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-13743</a></h3> +<div class="csaf-accordion-content"> +<p>CubeSpace CW0057 Reaction Wheel firmware versions prior to 5.0.20 are vulnerable to an Improper Verification of Cryptographic Signature vulnerability. This could allow an attacker with physical access to the product to upload arbitrary malicious firmware to the device without authentication.</p> +<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-13743">View CVE Details</a></p> +<hr> +<h4>Affected Products</h4> +<h5>CubeSpace CW0057 Reaction Wheel</h5> +<div class="ics-vendor-version-status"> +<div class="ics-vendor"><strong>Vendor:</strong><br>CubeSpace</div> +<div class="ics-version"><strong>Product Version:</strong><br>CubeSpace CW0057 Reaction Wheel: &lt;firmware_5.0.20</div> +<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div> +</div> +<div class="ics-remediations"> +<h6>Remediations</h6> +<p><strong>Vendor fix</strong><br>CubeSpace has released the following firmware versions for users to enable: Firmware version 5.0.20. Firmware version 5.0.20 introduces the capability for cryptographically verified secure boot; however, this protection is not enabled by default. Users must activate signed‑boot functionality, particularly the fully immutable mode, to achieve full security.</p> +<p><strong>Mitigation</strong><br>CubeSpace acknowledges the finding. The CW0057 reaction wheel authenticates firmware updates with a CRC-32 integrity check, which confirms image integrity but does not verify the source of an image. Exploitation requires direct physical access to the device and is not exploitable remotely. A device affected by this method remains recoverable: the bootloader operates independently of the application firmware and can reload known-good, CubeSpace-supplied images, so an affected unit cannot be permanently disabled by this method. Starting with firmware version 5.0.20, CubeSpace offers optional cryptographic secure boot of varying security levels which customers can enable. Given the physical-access prerequisite and the availability of recovery, CubeSpace assesses the practical risk as low.</p> +</div> +<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/347.html">CWE-347 Improper Verification of Cryptographic Signature</a></p> +<hr> +<h4>Metrics</h4> +<div class="csaf-table csaf-metrics-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th> +<th role="columnheader">Base Score</th> +<th role="columnheader">Base Severity</th> +<th role="columnheader">Vector String</th> +</tr> +</thead> +<tbody> +<tr> +<td>3.1</td> +<td>6.1</td> +<td>MEDIUM</td> +<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:P/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:H">CVSS:3.1/AV:P/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:H</a></td> +</tr> +<tr> +<td>4.0</td> +<td>3.3</td> +<td>LOW</td> +<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:P/AC:L/AT:N/PR:N/UI:N/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N/E:P">CVSS:4.0/AV:P/AC:L/AT:N/PR:N/UI:N/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N/E:P</a></td> +</tr> +</tbody> +</table> +</div> +</div> +</div> +</div> +<hr> +<h2>Acknowledgments</h2> +<ul> +<li>Anthony Rose reported this vulnerability to CISA</li> +</ul> +<hr> +<h2>Legal Notice and Terms of Use</h2> +<p>This product is provided subject to this Notification (https://www.cisa.gov/notification) and this Privacy &amp; Use policy (https://www.cisa.gov/privacy-policy).</p> +<hr> +<h2>Recommended Practices</h2> +<p>CISA recommends users take defensive measures to minimize the risk of exploitation of this vulnerability.</p> +<p>Minimize network exposure for all control system devices and/or systems, ensuring they are not accessible from the internet.</p> +<p>Locate control system networks and remote devices behind firewalls and isolating them from business networks.</p> +<p>When remote access is required, use more secure methods, such as Virtual Private Networks (VPNs), recognizing VPNs may have vulnerabilities and should be updated to the most current version available. Also recognize VPN is only as secure as the connected devices.</p> +<p>CISA reminds organizations to perform proper impact analysis and risk assessment prior to deploying defensive measures.</p> +<p>CISA also provides a section for control systems security recommended practices on the ICS webpage on cisa.gov/ics. Several CISA products detailing cyber defense best practices are available for reading and download, including Improving Industrial Control Systems Cybersecurity with Defense-in-Depth Strategies.</p> +<p>CISA encourages organizations to implement recommended cybersecurity strategies for proactive defense of ICS assets.</p> +<p>Additional mitigation guidance and recommended practices are publicly available on the ICS webpage at cisa.gov/ics in the technical information paper, ICS-TIP-12-146-01B--Targeted Cyber Intrusion Detection and Mitigation Strategies.</p> +<p>Organizations observing suspected malicious activity should follow established internal procedures and report findings to CISA for tracking and correlation against other incidents.</p> +<p>CISA also recommends users take the following measures to protect themselves from social engineering attacks:</p> +<p>Do not click web links or open attachments in unsolicited email messages.</p> +<p>Refer to Recognizing and Avoiding Email Scams for more information on avoiding email scams.</p> +<p>Refer to Avoiding Social Engineering and Phishing Attacks for more information on social engineering attacks.</p> +<p>No known public exploitation specifically targeting this vulnerability has been reported to CISA at this time. This vulnerability is not exploitable remotely.</p> +<hr> +<h2>Revision History</h2> +<ul> +<li><strong>Initial Release Date: </strong>2026-07-02</li> +</ul> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">Date</th> +<th role="columnheader">Revision</th> +<th role="columnheader">Summary</th> +</tr> +</thead> +<tbody> +<tr> +<td>2026-07-02</td> +<td>1</td> +<td>Initial Publication</td> +</tr> +</tbody> +</table> +<hr> +<h2>Legal Notice and Terms of Use</h2> + + Thu, 02 Jul 26 12:00:00 +0000 + CISA + /node/25107 + + + Gardyn IoT Hub + https://www.cisa.gov/news-events/ics-advisories/icsa-26-183-03 + <p><a href="https://github.com/cisagov/CSAF/blob/develop/csaf_files/OT/white/2026/icsa-26-183-03.json"><strong>View CSAF</strong></a></p> +<h2>Summary</h2> +<p><strong>Successful exploitation of these vulnerabilities could allow unauthenticated users to access and control IoT Hub managed devices.</strong></p> +<p>The following versions of Gardyn IoT Hub are affected:</p> +<ul> +<li>Home Firmware</li> +<li>Studio Firmware</li> +<li>Cloud API &lt;2.12.2026 (CVE-2026-13768, CVE-2026-55726, CVE-2026-54477)</li> +</ul> +<div class="csaf-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS</th> +<th role="columnheader">Vendor</th> +<th role="columnheader">Equipment</th> +<th role="columnheader">Vulnerabilities</th> +</tr> +</thead> +<tbody> +<tr> +<td>v3 10</td> +<td>Gardyn</td> +<td>Gardyn IoT Hub</td> +<td>Use of Hard-coded Credentials, Exposure of Sensitive System Information to an Unauthorized Control Sphere, Improper Neutralization of HTTP Headers for Scripting Syntax</td> +</tr> +</tbody> +</table> +</div> +<h3>Background</h3> +<ul> +<li><strong>Critical Infrastructure Sectors: </strong>Food and Agriculture</li> +<li><strong>Countries/Areas Deployed: </strong>United States</li> +<li><strong>Company Headquarters Location: </strong>United States</li> +</ul> +<hr> +<h2>Vulnerabilities</h2> +<div class="csaf-accordion"> +<p><a class="csaf-accordion-toggle-all" href="#">Expand All +</a></p> +<div class="csaf-accordion-item"> +<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-13768</a></h3> +<div class="csaf-accordion-content"> +<p>Gardyn devices expose a privileged iothubowner key. Access to this key will allow a malicious user to invoke an IoTHub Registry Manager function which returns connection information for all Gardyn Home Kit and Studio devices. Access to this key also allows a malicious user to execute arbitrary commands on a specific connected device and may allow the malicious user to pivot to other devices on the user's network.</p> +<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-13768">View CVE Details</a></p> +<hr> +<h4>Affected Products</h4> +<h5>Gardyn IoT Hub</h5> +<div class="ics-vendor-version-status"> +<div class="ics-vendor"><strong>Vendor:</strong><br>Gardyn</div> +<div class="ics-version"><strong>Product Version:</strong><br>Gardyn Home Firmware: &lt;master.627, Gardyn Studio Firmware: &lt;master.627, Gardyn Cloud API: &lt;2.12.2026</div> +<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div> +</div> +<div class="ics-remediations"> +<h6>Remediations</h6> +<p><strong>Mitigation</strong><br>Gardyn states that IoT Hub deployed infrastructure has been updated to fix the listed vulnerabilities.</p> +<p><strong>Mitigation</strong><br>Gardyn requests that users ensure their devices have Internet connectivity in order to automatically download needed firmware updates. Unconnected devices will automatically update when configured with a working Internet connection. Gardyn also recommends that users update their mobile application to the most recent version. The current versions of the Gardyn App and the Gardyn Home firmware can be checked in the Gardyn App.</p> +<p><strong>Mitigation</strong><br>Further information on Gardyn security can be found here: https://mygardyn.com/security/<br><a href="https://mygardyn.com/security/">https://mygardyn.com/security/</a></p> +<p><strong>Mitigation</strong><br>Further customer support can be obtained from Gardyn at: support@mygardyn.com<br><a href="mailto:support@mygardyn.com">mailto:support@mygardyn.com</a></p> +</div> +<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/798.html">CWE-798 Use of Hard-coded Credentials</a></p> +<hr> +<h4>Metrics</h4> +<div class="csaf-table csaf-metrics-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th> +<th role="columnheader">Base Score</th> +<th role="columnheader">Base Severity</th> +<th role="columnheader">Vector String</th> +</tr> +</thead> +<tbody> +<tr> +<td>3.1</td> +<td>10</td> +<td>CRITICAL</td> +<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:L">CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:L</a></td> +</tr> +<tr> +<td>4.0</td> +<td>9.5</td> +<td>CRITICAL</td> +<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:N/VC:H/VI:H/VA:L/SC:H/SI:H/SA:L">CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:N/VC:H/VI:H/VA:L/SC:H/SI:H/SA:L</a></td> +</tr> +</tbody> +</table> +</div> +</div> +</div> +<div class="csaf-accordion-item"> +<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-55726</a></h3> +<div class="csaf-accordion-content"> +<p>The Azure Blob Storage container used for Gardyn device logs is publicly listable without authentication. A malicious user would be able to access any device log file available in the blob storage container.</p> +<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-55726">View CVE Details</a></p> +<hr> +<h4>Affected Products</h4> +<h5>Gardyn IoT Hub</h5> +<div class="ics-vendor-version-status"> +<div class="ics-vendor"><strong>Vendor:</strong><br>Gardyn</div> +<div class="ics-version"><strong>Product Version:</strong><br>Gardyn Home Firmware: &lt;master.627, Gardyn Studio Firmware: &lt;master.627, Gardyn Cloud API: &lt;2.12.2026</div> +<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div> +</div> +<div class="ics-remediations"> +<h6>Remediations</h6> +<p><strong>Mitigation</strong><br>Gardyn states that IoT Hub deployed infrastructure has been updated to fix the listed vulnerabilities.</p> +<p><strong>Mitigation</strong><br>Gardyn requests that users ensure their devices have Internet connectivity in order to automatically download needed firmware updates. Unconnected devices will automatically update when configured with a working Internet connection. Gardyn also recommends that users update their mobile application to the most recent version. The current versions of the Gardyn App and the Gardyn Home firmware can be checked in the Gardyn App.</p> +<p><strong>Mitigation</strong><br>Further information on Gardyn security can be found here: https://mygardyn.com/security/<br><a href="https://mygardyn.com/security/">https://mygardyn.com/security/</a></p> +<p><strong>Mitigation</strong><br>Further customer support can be obtained from Gardyn at: support@mygardyn.com<br><a href="mailto:support@mygardyn.com">mailto:support@mygardyn.com</a></p> +</div> +<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/497.html">CWE-497 Exposure of Sensitive System Information to an Unauthorized Control Sphere</a></p> +<hr> +<h4>Metrics</h4> +<div class="csaf-table csaf-metrics-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th> +<th role="columnheader">Base Score</th> +<th role="columnheader">Base Severity</th> +<th role="columnheader">Vector String</th> +</tr> +</thead> +<tbody> +<tr> +<td>3.1</td> +<td>5.3</td> +<td>MEDIUM</td> +<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:N/A:N">CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:N/A:N</a></td> +</tr> +<tr> +<td>4.0</td> +<td>6.9</td> +<td>MEDIUM</td> +<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:L/VI:N/VA:N/SC:N/SI:N/SA:N">CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:L/VI:N/VA:N/SC:N/SI:N/SA:N</a></td> +</tr> +</tbody> +</table> +</div> +</div> +</div> +<div class="csaf-accordion-item"> +<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-54477</a></h3> +<div class="csaf-accordion-content"> +<p>The admin panel lacks standard security headers, enabling clickjacking and cross-site scripting attacks.</p> +<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-54477">View CVE Details</a></p> +<hr> +<h4>Affected Products</h4> +<h5>Gardyn IoT Hub</h5> +<div class="ics-vendor-version-status"> +<div class="ics-vendor"><strong>Vendor:</strong><br>Gardyn</div> +<div class="ics-version"><strong>Product Version:</strong><br>Gardyn Home Firmware: &lt;master.627, Gardyn Studio Firmware: &lt;master.627, Gardyn Cloud API: &lt;2.12.2026</div> +<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div> +</div> +<div class="ics-remediations"> +<h6>Remediations</h6> +<p><strong>Mitigation</strong><br>Gardyn states that IoT Hub deployed infrastructure has been updated to fix the listed vulnerabilities.</p> +<p><strong>Mitigation</strong><br>Gardyn requests that users ensure their devices have Internet connectivity in order to automatically download needed firmware updates. Unconnected devices will automatically update when configured with a working Internet connection. Gardyn also recommends that users update their mobile application to the most recent version. The current versions of the Gardyn App and the Gardyn Home firmware can be checked in the Gardyn App.</p> +<p><strong>Mitigation</strong><br>Further information on Gardyn security can be found here: https://mygardyn.com/security/<br><a href="https://mygardyn.com/security/">https://mygardyn.com/security/</a></p> +<p><strong>Mitigation</strong><br>Further customer support can be obtained from Gardyn at: support@mygardyn.com<br><a href="mailto:support@mygardyn.com">mailto:support@mygardyn.com</a></p> +</div> +<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/644.html">CWE-644 Improper Neutralization of HTTP Headers for Scripting Syntax</a></p> +<hr> +<h4>Metrics</h4> +<div class="csaf-table csaf-metrics-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th> +<th role="columnheader">Base Score</th> +<th role="columnheader">Base Severity</th> +<th role="columnheader">Vector String</th> +</tr> +</thead> +<tbody> +<tr> +<td>3.1</td> +<td>5.4</td> +<td>MEDIUM</td> +<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:L/I:L/A:N">CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:L/I:L/A:N</a></td> +</tr> +<tr> +<td>4.0</td> +<td>5.1</td> +<td>MEDIUM</td> +<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N">CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N</a></td> +</tr> +</tbody> +</table> +</div> +</div> +</div> +</div> +<hr> +<h2>Acknowledgments</h2> +<ul> +<li>Michael Groberman reported these vulnerabilities to CISA</li> +</ul> +<hr> +<h2>Legal Notice and Terms of Use</h2> +<p>This product is provided subject to this Notification (https://www.cisa.gov/notification) and this Privacy &amp; Use policy (https://www.cisa.gov/privacy-policy).</p> +<hr> +<h2>Recommended Practices</h2> +<p>CISA recommends users take defensive measures to minimize the risk of exploitation of these vulnerabilities.</p> +<p>Minimize network exposure for all control system devices and/or systems, ensuring they are not accessible from the Internet.</p> +<p>Locate control system networks and remote devices behind firewalls and isolating them from business networks.</p> +<p>When remote access is required, use more secure methods, such as Virtual Private Networks (VPNs), recognizing VPNs may have vulnerabilities and should be updated to the most current version available. Also recognize VPN is only as secure as the connected devices.</p> +<p>CISA reminds organizations to perform proper impact analysis and risk assessment prior to deploying defensive measures.</p> +<p>CISA also provides a section for control systems security recommended practices on the ICS webpage on cisa.gov/ics. Several CISA products detailing cyber defense best practices are available for reading and download, including Improving Industrial Control Systems Cybersecurity with Defense-in-Depth Strategies.</p> +<p>CISA encourages organizations to implement recommended cybersecurity strategies for proactive defense of ICS assets.</p> +<p>Additional mitigation guidance and recommended practices are publicly available on the ICS webpage at cisa.gov/ics in the technical information paper, ICS-TIP-12-146-01B--Targeted Cyber Intrusion Detection and Mitigation Strategies.</p> +<p>Organizations observing suspected malicious activity should follow established internal procedures and report findings to CISA for tracking and correlation against other incidents.</p> +<p>No known public exploitation specifically targeting these vulnerabilities has been reported to CISA at this time.</p> +<hr> +<h2>Revision History</h2> +<ul> +<li><strong>Initial Release Date: </strong>2026-07-02</li> +</ul> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">Date</th> +<th role="columnheader">Revision</th> +<th role="columnheader">Summary</th> +</tr> +</thead> +<tbody> +<tr> +<td>2026-07-02</td> +<td>1</td> +<td>Initial Publication</td> +</tr> +</tbody> +</table> +<hr> +<h2>Legal Notice and Terms of Use</h2> + + Thu, 02 Jul 26 12:00:00 +0000 + CISA + /node/25108 + + + ST Engineering iDirect iQ-Series Terminals + https://www.cisa.gov/news-events/ics-advisories/icsa-26-183-01 + <p><a href="https://github.com/cisagov/CSAF/blob/develop/csaf_files/OT/white/2026/icsa-26-183-01.json"><strong>View CSAF</strong></a></p> +<h2>Summary</h2> +<p><strong>Successful exploitation of these vulnerabilities could allow an attacker to gain unauthorized access to device information or cause a denial-of-service condition.</strong></p> +<p>The following versions of ST Engineering iDirect iQ-Series Terminals are affected:</p> +<ul> +<li>Evolution iQ‑Series terminals &lt;=4.5.2.1 (CVE-2026-38059, CVE-2026-38057)</li> +<li>3315‑Series terminals &lt;=4.5.2.1 (CVE-2026-38059, CVE-2026-38057)</li> +<li>9‑Series terminals &lt;=4.5.2.1 (CVE-2026-38059, CVE-2026-38057)</li> +</ul> +<div class="csaf-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS</th> +<th role="columnheader">Vendor</th> +<th role="columnheader">Equipment</th> +<th role="columnheader">Vulnerabilities</th> +</tr> +</thead> +<tbody> +<tr> +<td>v3 8.1</td> +<td>ST Engineering iDirect</td> +<td>ST Engineering iDirect iQ-Series Terminals</td> +<td>Missing Authentication for Critical Function, Cross-Site Request Forgery (CSRF)</td> +</tr> +</tbody> +</table> +</div> +<h3>Background</h3> +<ul> +<li><strong>Critical Infrastructure Sectors: </strong>Communications, Defense Industrial Base, Energy, Government Services and Facilities, Transportation Systems</li> +<li><strong>Countries/Areas Deployed: </strong>Worldwide</li> +<li><strong>Company Headquarters Location: </strong>United States</li> +</ul> +<hr> +<h2>Vulnerabilities</h2> +<div class="csaf-accordion"> +<p><a class="csaf-accordion-toggle-all" href="#">Expand All +</a></p> +<div class="csaf-accordion-item"> +<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-38059</a></h3> +<div class="csaf-accordion-content"> +<p>The iDirect iQ200 exposes the /api/identity and /api/ REST API endpoints without authentication. An unauthenticated attacker with network access can retrieve sensitive device information including the serial number, Device ID (DID), Terminal Private Key identifier (TPK), MAC address, and exact firmware version. The DID and TPK are used for satellite network authentication in the iDirect platform, potentially enabling terminal impersonation and network reconnaissance.</p> +<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-38059">View CVE Details</a></p> +<hr> +<h4>Affected Products</h4> +<h5>ST Engineering iDirect iQ-Series Terminals</h5> +<div class="ics-vendor-version-status"> +<div class="ics-vendor"><strong>Vendor:</strong><br>ST Engineering iDirect</div> +<div class="ics-version"><strong>Product Version:</strong><br>ST Engineering iDirect Evolution iQ‑Series terminals: &lt;=4.5.2.1, ST Engineering iDirect 3315‑Series terminals: &lt;=4.5.2.1, ST Engineering iDirect 9‑Series terminals: &lt;=4.5.2.1</div> +<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div> +</div> +<div class="ics-remediations"> +<h6>Remediations</h6> +<p><strong>Mitigation</strong><br>ST Engineering iDirect has fixed the vulnerabilities and recommend users update the software to version 4.5.2.2 or newer.</p> +<p><strong>Mitigation</strong><br>Registered users are able to download patches from the iDirect Support Portal https://support.idirect.net/s/login.<br><a href="https://support.idirect.net/s/login">https://support.idirect.net/s/login</a></p> +<p><strong>Mitigation</strong><br>Restrict management interfaces to trusted networks (e.g., VPN, ACLs).</p> +<p><strong>Mitigation</strong><br>Avoid exposing administrative APIs to the public internet.</p> +<p><strong>Mitigation</strong><br>Enforce strong authentication practices.</p> +<p><strong>Mitigation</strong><br>Monitor for anomalous API activity and unexpected device reboots.</p> +</div> +<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/306.html">CWE-306 Missing Authentication for Critical Function</a></p> +<hr> +<h4>Metrics</h4> +<div class="csaf-table csaf-metrics-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th> +<th role="columnheader">Base Score</th> +<th role="columnheader">Base Severity</th> +<th role="columnheader">Vector String</th> +</tr> +</thead> +<tbody> +<tr> +<td>3.1</td> +<td>7.5</td> +<td>HIGH</td> +<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N">CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N</a></td> +</tr> +<tr> +<td>4.0</td> +<td>8.7</td> +<td>HIGH</td> +<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:N/VA:N/SC:N/SI:N/SA:N">CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:N/VA:N/SC:N/SI:N/SA:N</a></td> +</tr> +</tbody> +</table> +</div> +</div> +</div> +<div class="csaf-accordion-item"> +<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-38057</a></h3> +<div class="csaf-accordion-content"> +<p>The iDirect iQ200 does not validate CSRF tokens on state-changing API endpoints after authentication. The /api/reboot endpoint accepts POST requests authenticated solely by a session cookie that lacks the SameSite attribute. A remote attacker can host a malicious web page that, when visited by an authenticated administrator, automatically submits a cross-site POST request causing an immediate device reboot and satellite link loss. Repeated attacks can sustain a denial-of-service condition.</p> +<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-38057">View CVE Details</a></p> +<hr> +<h4>Affected Products</h4> +<h5>ST Engineering iDirect iQ-Series Terminals</h5> +<div class="ics-vendor-version-status"> +<div class="ics-vendor"><strong>Vendor:</strong><br>ST Engineering iDirect</div> +<div class="ics-version"><strong>Product Version:</strong><br>ST Engineering iDirect Evolution iQ‑Series terminals: &lt;=4.5.2.1, ST Engineering iDirect 3315‑Series terminals: &lt;=4.5.2.1, ST Engineering iDirect 9‑Series terminals: &lt;=4.5.2.1</div> +<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div> +</div> +<div class="ics-remediations"> +<h6>Remediations</h6> +<p><strong>Mitigation</strong><br>ST Engineering iDirect has fixed the vulnerabilities and recommend users update the software to version 4.5.2.2 or newer.</p> +<p><strong>Mitigation</strong><br>Registered users are able to download patches from the iDirect Support Portal https://support.idirect.net/s/login.<br><a href="https://support.idirect.net/s/login">https://support.idirect.net/s/login</a></p> +<p><strong>Mitigation</strong><br>Restrict management interfaces to trusted networks (e.g., VPN, ACLs).</p> +<p><strong>Mitigation</strong><br>Avoid exposing administrative APIs to the public internet.</p> +<p><strong>Mitigation</strong><br>Enforce strong authentication practices.</p> +<p><strong>Mitigation</strong><br>Monitor for anomalous API activity and unexpected device reboots.</p> +</div> +<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/352.html">CWE-352 Cross-Site Request Forgery (CSRF)</a></p> +<hr> +<h4>Metrics</h4> +<div class="csaf-table csaf-metrics-table"> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th> +<th role="columnheader">Base Score</th> +<th role="columnheader">Base Severity</th> +<th role="columnheader">Vector String</th> +</tr> +</thead> +<tbody> +<tr> +<td>3.1</td> +<td>8.1</td> +<td>HIGH</td> +<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:H/A:H">CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:H/A:H</a></td> +</tr> +<tr> +<td>4.0</td> +<td>7</td> +<td>HIGH</td> +<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N">CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N</a></td> +</tr> +</tbody> +</table> +</div> +</div> +</div> +</div> +<hr> +<h2>Acknowledgments</h2> +<ul> +<li>Ahmed Alqahtani of Aramco reported these vulnerabilities to CISA</li> +</ul> +<hr> +<h2>Legal Notice and Terms of Use</h2> +<p>This product is provided subject to this Notification (https://www.cisa.gov/notification) and this Privacy &amp; Use policy (https://www.cisa.gov/privacy-policy).</p> +<hr> +<h2>Recommended Practices</h2> +<p>CISA recommends users take defensive measures to minimize the risk of exploitation of these vulnerabilities.</p> +<p>Minimize network exposure for all control system devices and/or systems, ensuring they are not accessible from the internet.</p> +<p>Locate control system networks and remote devices behind firewalls and isolating them from business networks.</p> +<p>When remote access is required, use more secure methods, such as Virtual Private Networks (VPNs), recognizing VPNs may have vulnerabilities and should be updated to the most current version available. Also recognize VPN is only as secure as the connected devices.</p> +<p>CISA reminds organizations to perform proper impact analysis and risk assessment prior to deploying defensive measures.</p> +<p>CISA also provides a section for control systems security recommended practices on the ICS webpage on cisa.gov/ics. Several CISA products detailing cyber defense best practices are available for reading and download, including Improving Industrial Control Systems Cybersecurity with Defense-in-Depth Strategies.</p> +<p>CISA encourages organizations to implement recommended cybersecurity strategies for proactive defense of ICS assets.</p> +<p>Additional mitigation guidance and recommended practices are publicly available on the ICS webpage at cisa.gov/ics in the technical information paper, ICS-TIP-12-146-01B--Targeted Cyber Intrusion Detection and Mitigation Strategies.</p> +<p>Organizations observing suspected malicious activity should follow established internal procedures and report findings to CISA for tracking and correlation against other incidents.</p> +<p>CISA also recommends users take the following measures to protect themselves from social engineering attacks:</p> +<p>Do not click web links or open attachments in unsolicited email messages.</p> +<p>Refer to Recognizing and Avoiding Email Scams for more information on avoiding email scams.</p> +<p>Refer to Avoiding Social Engineering and Phishing Attacks for more information on social engineering attacks.</p> +<p>No known public exploitation specifically targeting these vulnerabilities has been reported to CISA at this time.</p> +<hr> +<h2>Revision History</h2> +<ul> +<li><strong>Initial Release Date: </strong>2026-07-02</li> +</ul> +<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap> +<thead> +<tr> +<th role="columnheader" data-tablesaw-priority="persist">Date</th> +<th role="columnheader">Revision</th> +<th role="columnheader">Summary</th> +</tr> +</thead> +<tbody> +<tr> +<td>2026-07-02</td> +<td>1</td> +<td>Initial Publication</td> +</tr> +</tbody> +</table> +<hr> +<h2>Legal Notice and Terms of Use</h2> + + Thu, 02 Jul 26 12:00:00 +0000 + CISA + /node/25106 + + + + diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/darkreading.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/darkreading.xml new file mode 100644 index 00000000..ddbc13c3 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/darkreading.xml @@ -0,0 +1,52 @@ + + + + darkreading + https://www.darkreading.com + Public RSS feed + en + + + <![CDATA[Chinese LLMs Broaden the Gap Between Attackers & Defenders]]> + + + Fri, 03 Jul 2026 13:01:00 GMT + Robert Lemos + + + + + castle-of-the-moors-Gabriela_Beres-shutterstock.jpg + + + + <![CDATA[Aussies Face Reduced Cybercrime Risk, as Pressure Shifts to SMBs]]> + + + Thu, 02 Jul 2026 23:01:00 GMT + Nate Nelson + + + + + Australia-imaginima-Getty.jpg + + + + <![CDATA[Apple Reverses Age-Old Patch Policy to Keep Up With AI]]> + + + Thu, 02 Jul 2026 19:31:58 GMT + Nate Nelson + + + + + Apple_bandage-imtmphoto-Getty.jpg + + + diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/krebs.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/krebs.xml new file mode 100644 index 00000000..4b80de7a --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/krebs.xml @@ -0,0 +1,245 @@ + + + + Krebs on Security + + https://krebsonsecurity.com + In-depth security news and investigation + Sat, 04 Jul 2026 16:16:19 +0000 + en-US + + hourly + + 1 + https://wordpress.org/?v=6.2.2 + + FBI Seizes NetNut Proxy Platform, Popa Botnet + https://krebsonsecurity.com/2026/07/fbi-seizes-netnut-proxy-platform-popa-botnet/ + https://krebsonsecurity.com/2026/07/fbi-seizes-netnut-proxy-platform-popa-botnet/#comments + + + Thu, 02 Jul 2026 19:27:33 +0000 + + + + + + + + + + + + + + + + + + https://krebsonsecurity.com/?p=73923 + + + The Federal Bureau of Investigation (FBI) said today it worked with industry partners to seize hundreds of domains associated with NetNut, a sprawling residential proxy service operated by the publicly-traded Israeli company Alarum Technologies [NASDAQ: ALAR]. The action comes roughly two weeks after KrebsOnSecurity published findings from multiple security firms connecting NetNut to the Popa botnet, a collection of at least two million devices that have been compromised by malicious software with little or no consent from victims.

+

The NetNut homepage today was replaced by this seizure banner from the FBI.

+

On June 19, three different security firms issued similar findings: That NetNut is a residential proxy network which populates a botnet called Popa, and distributes software for devices commonly found in homes, such as smart TVs and streaming boxes. NetNut’s software turns those systems into always-on residential proxy nodes that are rented to others, who predominantly use them to relay abusive and intrusive Internet traffic, such as mass content scraping, advertising fraud, and account takeover activity.

+

Earlier today, NetNut’s homepage was replaced with a seizure notice from the FBI and the Internal Revenue Service Criminal Investigation division. The seizure notice thanked Google, Lumen, Shadowserver and other industry partners for their help in dismantling hundreds of domains tied to the Popa botnet, which experts say has long been synonymous with NetNut’s residential proxy infrastructure.

+

In a blog post published today, the Google Threat Intelligence Group (GTIG) said NetNut’s proxy network is widely resold and white-labeled by a number of third-party proxy providers, and that its services are heavily sought out by cybercriminals seeking to obfuscate the source of their malicious traffic. The GTIG said that in a single week during June 2026, they observed 316 distinct clusters of threat actors using suspected NetNut exit nodes, including cybercriminal and espionage groups.

+

“These bad actors can use NetNut to mask their origin IP address when accessing victim environments, accessing their own infrastructure, and conducting password spray attacks,” Google’s GTIG wrote. “Furthermore, when a consumer device becomes an exit node, unauthorized network traffic passes through it. This means bad actors can access other private devices on the same home network, effectively exposing them to Internet threats.”

+

Google said it disabled Google accounts and services used by NetNut for malware command and control, and that it shared technical intelligence on NetNut’s software development kits (SDKs) and backend infrastructure with platform providers, law enforcement and research firms. The company also disabled apps known to bundle NetNut’s various SDKs.

+

Omer Weiss, legal counsel for NetNut parent Alarum Technologies, said the company was aware of the FBI seizure and cooperating with investigators.

+

“Alarum takes this matter seriously and will fully cooperate with law enforcement to ensure any misuse of its infrastructure is thoroughly investigated and those responsible are held to account,” Weiss said in a written statement.
+

+

Benjamin Brundage is founder of the proxy tracking service Synthient, one of the companies that published evidence last month linking the Popa botnet to NetNut and Alarum Technologies. Brundage said the domain seizures appear to have disrupted both the Popa botnet and the NetNut proxy network that rides on top of it.

+

Brundage said NetNut’s apparent demise is likely to be a great disadvantage for the cybercrime community, which was already reeling from legal actions by Google earlier this year that seized infrastructure for NetNut’s biggest competitor — IPIDEA.

+

“I think this takedown is going to have a big impact, because NetNut gained significant popularity after the IPIDEA takedown,” he said. “Also NetNut has been incredibly common among resellers, and they were on par with IPIDEA in terms of their daily traffic, quality, size, price per gigabyte, all of it.”

+

NetNut’s infrastructure, in a nutshell. Image: Black Lotus Labs, Lumen.

+

The NetNut and Popa botnet takedown may have another added benefit, Brundage said: Lessening the impact of large distributed denial-of-service botnets that have been built on the backs of poorly configured residential proxy services. In January, Synthient revealed how cybercriminals had built the world’s largest DDoS botnet (Kimwolf) by tunneling through IPIDEA proxy connections into the local networks of TV box owners, and infecting other Android-based devices behind the victim’s firewall.

+

While many of the bigger proxy providers took steps to block this activity, resellers of the major proxy networks have been far slower to respond to the threat, Brundage said.

+

“In terms of all these TV box devices getting compromised from the proxy network, it will have an impact on the DDoS botnets out there,” he said.

+

For its part, Google reckons today’s actions have caused “significant degradation to NetNut’s proxy network and its business operations, reducing the available pool of devices for the proxy operator by millions.” But the company warns that proxy networks can rebuild themselves by effectively reselling other proxy services, as IPIDEA has done over the past few months.

+

“Google has high confidence that many popular residential proxy brands are in fact whitelabeling the NetNut botnet,” the GTIG report concludes. “While we expect this disruption to have a larger ripple effect across the residential proxy ecosystem, observations after the disruption of IPIDEA proved that individual networks can appear resilient. What we have observed is that when faced with the degradation of their own botnet, proxy operators begin buying capacity from their competitors, effectively becoming a reseller. We recognize that creating a lasting disruption in this fluid ecosystem means we must scale our efforts to target the infrastructure of several interconnected providers.”

+

As KrebsOnSecurity has warned repeatedly, most of the no-name TV streaming boxes for sale on the major e-commerce websites either come pre-installed with residential proxy software, or require the installation of proxy SDKs in order to use the device for its stated purpose (streaming pirated movies, sporting events and TV shows). Google’s advice here is sound: When it comes to TV boxes, stick to name brands from reputable manufacturers, and then be sparing and judicious with any apps you choose to install.

+

The sketchy TV boxes that are being commandeered by the Popa botnet and other threats all come with or require the user to install unofficial Android operating systems that do not operate within the confines of Google’s Official Play Protect store. Google says consumers can confirm whether or not a device is built with the official Android TV OS and Play Protect certification by following these instructions.

+

Even people without TV streaming boxes can find their smart TVs enrolled in residential proxy networks, just by installing one of thousands of apps available for download on Samsung and LG smart TVs. In a report released last month, the proxy tracking company Spur found 42 percent of apps available for download via the webOS operating system on LG smart TVs include SDKs that turn one’s television into an always-on residential proxy node. More than a quarter of the apps made for Samsung’s Tizen operating system had similar residential proxy components, Spur found.

+

Image: Spur.us.

+

Update, 4:24 p.m. ET: Included a statement shared post-publication from an attorney representing NetNut parent Alarum Technologies.

+]]> + + https://krebsonsecurity.com/2026/07/fbi-seizes-netnut-proxy-platform-popa-botnet/feed/ + 17 + + + + + Scattered Spider Hackers Plead Guilty on Day 1 of Trial + https://krebsonsecurity.com/2026/06/scattered-spider-hackers-plead-guilty-on-day-1-of-trial/ + https://krebsonsecurity.com/2026/06/scattered-spider-hackers-plead-guilty-on-day-1-of-trial/#comments + + + Tue, 23 Jun 2026 16:12:49 +0000 + + + + + + + + + + https://krebsonsecurity.com/?p=73876 + + + Two men pleaded guilty in the United Kingdom this week to criminal charges stemming from an August 2024 cyberattack that crippled Transport for London, the entity responsible for the public transport network in the Greater London area. The duo were key members of a prolific cybercrime group known as Scattered Spider, and their guilty pleas came on the first day of what was expected to be a six-week trial.

+

Owen Flowers (left) 18, and Thalha Jubair, 20. Image: UK National Crime Agency (NCA).

+

Thalha Jubair, 20, of East London and 18-year-old Owen Flowers of Walsall admitted conspiring to commit unauthorized acts against Transport for London computer systems and causing risk of serious damage to human welfare. According to a report from the BBC, Flowers alone admitted to being part of a conspiracy to hack into U.S. based healthcare providers SSM Health Care Corporation and Sutter Health in September 2024.

+

Jubair is also wanted by U.S. law enforcement agencies. In September 2025, prosecutors in New Jersey unsealed an indictment alleging Jubair and other Scattered Spider members committed computer fraud, wire fraud, and money laundering in relation to 120 computer network intrusions involving 47 U.S. entities between May 2022 and September 2025, and that the group’s victims paid at least $115 million in ransom payments.

+

In July 2025, KrebsOnSecurity reported that Flowers and Jubair were arrested in the United Kingdom in connection with Scattered Spider ransom attacks against the retailers Marks & Spencer and Harrods, and the British food retailer Co-op Group. Multiple sources familiar with those investigations said Flowers was the Scattered Spider member who anonymously gave interviews to the media in the days after the group’s September 2023 ransomware attacks disrupted operations at Las Vegas casinos operated by MGM Resorts and Caesars Entertainment.

+

According to prosecutors, Jubair co-ran a bustling Telegram channel called Star Chat, the home of a SIM-swapping group that used voice- and SMS-based phishing attacks to steal credentials from employees at the major wireless providers in the U.S. and U.K. The group would then use that access to sell a service that could redirect a target’s phone number to a device the attackers controlled and intercept the victim’s calls and text messages (including one-time codes for multi-factor authentication).

+

A receipt from Star Fraud Chat’s SIM-swapping service targeting a T-Mobile customer after the group gained access to internal T-Mobile employee tools. “Rocket Ace” was one of Jubair’s hacker handles, according to U.S. prosecutors.

+

New Jersey prosecutors also allege Jubair also was involved in a mass SMS phishing campaign during the summer of 2022 that stole single sign-on credentials from employees at hundreds of companies. That weeks-long SMS phishing campaign led to intrusions and data thefts at more than 130 organizations, including LastPass, DoorDash, Mailchimp, Plex and Signal.

+

KrebsOnSecurity reported last year that one of Jubair’s alter egos at age 15 was “Everlynn,” a hacker who sold fraudulent “emergency data requests” that used compromised police and government email addresses to demand subscriber data (e.g. username, IP/email address) from major tech companies, claiming the requests concerned urgent matters of life and death and could not wait for a court order.

+

In April 2026, 24-year-old British national and Scattered Spider member Tyler “Tylerb” Buchanan pleaded guilty to wire fraud conspiracy and aggravated identity theft for participating in the group’s SMS phishing spree in the summer of 2022. The government said Buchanan, Jubair and others used the credentials harvested in that phishing campaign to steal at least $8 million in cryptocurrency from victims throughout the United States. Buchanan is currently scheduled to be sentenced on October 2.

+

In August 2025, 20-year-old Scattered Spider member from Florida named Noah Michael Urban was sentenced to 10 years in federal prison and ordered to pay $13 million in restitution, after pleading guilty to charges of wire fraud and conspiracy.

+

The U.S. Department of Justice says three alleged Scattered Spider defendants indicted along with Buchanan still face charges, including Ahmed Hossam Eldin Elbadawy, 24, a.k.a. “AD,” of College Station, Texas; Evans Onyeaka Osiebo, 21, of Dallas, Texas; and Joel Martin Evans, 26, a.k.a. “joeleoli,” of Jacksonville, North Carolina.

+

Flowers and Jubair are slated to be sentenced in a London court on July 15, 2026.

+]]> + + https://krebsonsecurity.com/2026/06/scattered-spider-hackers-plead-guilty-on-day-1-of-trial/feed/ + 39 + + + + + ‘Popa’ Botnet Linked to Publicly-Traded Israeli Firm + https://krebsonsecurity.com/2026/06/popa-botnet-linked-to-publicly-traded-israeli-firm/ + https://krebsonsecurity.com/2026/06/popa-botnet-linked-to-publicly-traded-israeli-firm/#comments + + + Thu, 18 Jun 2026 17:37:58 +0000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + https://krebsonsecurity.com/?p=73832 + + + For the past four years, a sprawling Android-based botnet called Popa has forced millions of consumer TV boxes to relay Internet traffic linked to advertising fraud, account takeovers, and mass data-scraping efforts. This week, researchers from multiple security firms concluded that the Popa botnet is linked to NetNut, a “residential proxy” provider operated by the publicly-traded Israeli firm Alarum Technologies Ltd [NASDAQ: ALAR].

+
Malicious streaming devices sold online that enroll the user's home Internet address in a residential proxy service. Image: Synthient. Pictured are 8 different TV boxes, including the X96 Mini Box, stick, and other no-name brands.

Malicious streaming devices sold online that enroll the user’s home Internet address in a residential proxy service. Image: HUMAN Security.

+

Popa is a massive botnet, but by all accounts it is unlike traditional botnets that enlist compromised systems in destructive activities, such as coordinating huge distributed denial-of-service attacks. Rather, Popa appears designed with a singular purpose: Implementing a persistent communications layer capable of registering a device, maintaining long-lived encrypted connections, and opening communication tunnels on demand.

+

Experts say Popa is a plugin component associated with the Vo1d botnet, a large-scale malware campaign targeting unofficial Android-based TV boxes. These devices, which are marketed under thousands of brand names and model numbers and broadly available for purchase at top e-commerce destinations, all advertise the ability to stream hundreds of subscription video services for an up front one-time fee.

+

But as the FBI and security industry experts have warned repeatedly, these streaming boxes typically bundle or come pre-installed with software that turns the user’s TV into a “residential proxy” — allowing anyone to route their Internet traffic through that device for as long as it remains plugged into a wall socket and connected to a local network. More concerning, some of these proxy networks do little to stop malicious customers from communicating with and even compromising systems on the local network of the unsuspecting device owner.

+

The first clues about Popa’s origins came in a 2025 report from the Chinese security company XLAB, which flagged at least nine domain names that were used to register and direct the activities of compromised devices. In a report released today, the security firm Qurium described how it stumbled on some of those same domains while investigating a series of disruptive and expensive data scraping events targeting the company’s hosted organizations in May 2026, in which the scraping activity was scattered evenly across more than 1.4 million Internet addresses.

+

Qurium said it found several dozen domains used to control Popa that were all hosted in lockstep across multiple Internet addresses over time, including gmslb[.]net, safernetwork[.]io, tera-home[.]com, and ninjatech[.]io. Digging deeper, Qurium discovered gmslb[.]net was referenced in dozens of pirated or modded video content streaming apps, such as CRICFy, DooFlix, Sprozfy, RTS Tv, Flixoid, CyberFlix, Rapid Streamz, TvMob and HD/OceanStreams.

+

Qurium’s report notes that most of the domains long used to control the Popa botnet were seized or dismantled in July 2025, after Google, HUMAN Security and Trend Micro teamed up to disrupt Badbox 2.0, a botnet that is closely associated with Vo1d. Qurium said that immediately after that disruption, several dozen new domains were registered to serve as controllers for the Popa botnet, but that one of those control domains was not new: ninjatech[.]io.

+

Ninjatech is a company founded by Moishi Kramer, whose LinkedIn profile says he is vice president of research and development at NetNut. That resume credits Kramer for helping NetNut to build from the “ground up,” “designing the architecture,” and “scaling the NetNut” before the company was acquired by Alarum Technologies. A self-created listing at the job board F6S references Kramer as the sole owner of the Ninjatech domain (a screen capture of it is pictured below).

+

Image: F6S.com.

+

Responding via email, Mr. Kramer said Ninjatech ceased operations approximately five years ago, when the company sold a software development kit (SDK) called Popa that was designed to use a small portion of a device’s bandwidth and to run only after the host application obtained user consent.

+

“That code was sold and licensed to third parties including resellers years ago,” Kramer said. “Once software is distributed that way, the original developer has no control over how others later modify, rebrand, or deploy it.”

+

Kramer said neither he nor NetNut builds, operates or maintains the infrastructure being described as Popa, nor does he control the Ninjatech domain.

+

“I didn’t register the June 2025 domains you mention, and I don’t know who did,” he continued. “I have no control over, or visibility into, that infrastructure. I can only tell you it isn’t operated by me or by NetNut.”

+

But in a separate Popa research report released today, the proxy-tracking company Synthient said a recent analysis of the Popa SDK revealed outbound traffic clearly associated with NetNut.

+

“The research team assesses with high confidence that devices running Popa forward traffic from Netnut clients,” Synthient wrote. “This proves without a shadow of a doubt that Popa actively continues to be used by NetNut as part of their proxy pool.”

+

Synthient’s platform receiving outbound traffic from Popa. Image: Synthient.com.

+

Alarum Technologies, NetNut’s Tel Aviv-based parent company, said the reports by Synthient and Qurium contained “demonstrably inaccurate assertions and flawed deductions rather than verified facts.” Alarum shared a statement saying they reject the basic characterization of the SDKs and technologies discussed in the reports as a “botnet.”

+

“The SDKs at issue are designed to facilitate bandwidth-sharing functionality and do not transform user devices into malware-controlled systems or otherwise compromise the devices on which they operate,” the statement reads. “Netnut operates a commercial proxy network and maintains policies, procedures, and technological measures designed to promote lawful and responsible use of its services.”

+

Alarum said NetNut places “significant emphasis on appropriate notice and consent mechanisms, conducts customer due diligence, monitors for potential misuse, and takes steps intended to detect and mitigate suspicious or unauthorized activity.”

+

“This method of operation is supported both by internal procedures and policies, including performing KYC checks and additional due diligence of NetNut’s customers, as well as employing various technological measures, designed to assist in identifying and addressing suspected misuse of the network,” their statement continued.

+

However, in a report released on June 8, the proxy tracking service Spur asserted that NetNut does not require corporate verification or meaningful “know your customer” procedures before allowing customers to purchase proxy access.

+

“An individual can sign up, pay, and route traffic through partner address space, including space belonging to institutions whose users never opted in,” Spur wrote. “The ‘verified corporations only’ claim is simply marketing for bandwidth sellers, not an access control on who actually uses the proxies.”

+

“Nor is NetNut the only front door,” Spur continued. “A number of downstream white labelers and resellers repackage the same ISP proxy pool under their own brands. These outlets typically perform no KYC at all, less scrutiny than NetNut itself, who at the very least might assign an account manager to potential users. Anyone who knows where to look can buy access through a reseller with nothing more than a burner email address and $5 in crypto.”

+

Synthient found that although the most recent builds of Popa (as of three months ago) have added the ability to ask the user for consent before installing proxy components, not all variants or previous versions of Popa contain this functionality.

+

“Of the over 20 genuine Popa publishers analyzed, none of them were observed asking for user consent,” Sythient wrote.

+

THE PREVALENCE OF POPA

+

Chris Formosa is senior lead information security engineer for Black Lotus Labs, a division of the Internet backbone carrier Lumen Technologies.

+

“What especially makes Popa dangerous is just how widely used NetNut is for reselling and sharing,” Formosa said, explaining that many other proxy services simply resell NetNut proxies rather than building out their own far-flung proxy networks. “So these Popa IPs appear in tons of different services all over the ecosystem, which makes it one of the most problematic and dangerous proxy botnets on the market currently.”

+

Formosa said the Popa botnet averages between 1.5 million to 2.5 million distinct IP addresses each day, relying on between 250 and 300 Internet addresses that are used to direct its activities.

+

“That’s why Popa is so dangerous,” Formosa said. “It may not be the largest botnet we have seen, but it is spread all over the industry, making its power very amplified.”

+

Formosa said while that makes Popa one of the larger botnets out there today, its numbers pale in comparison to those previously boasted by IPIDEA, a China-based proxy provider that until recently operated a daily pool of nearly 10 million devices that they resold as proxies to anyone. In January 2026, Synthient published research showing that multiple new large DDoS botnets had grown rapidly by tunneling through IPIDEA proxies into the local networks of unsuspecting TV box owners and infecting other Android-based devices behind the user’s firewall.

+

IPIDEA is based largely on SDKs used to view pirated streaming content on a vast number of TV box devices, but the service’s numbers have dwindled since January, when Google and industry partners took legal action to seize domain names that IPIDEA used to control devices and proxy traffic through them.

+

Jérôme Meyer, a security researcher at Nokia Deepfield, said the total population of devices participating in the Popa botnet may be far higher than Lumen’s estimates. Meyer told KrebsOnSecurity that Nokia is monitoring 26 of at least 359 known relay nodes for the botnet, and estimates that each relay node handles between 35,000 and 60,000 clients simultaneously.

+

“On the relay node subset I am looking at (26 of them), 750,000 unique sources in 24 hours,” Meyer wrote in response to questions.

+

Nokia Deepfield released its own report today on RoboVPN, a VPN app tied to the Vo1d botnet’s Popa plugin that Qurium attributes to NetNut/Alarum Technologies.

+

THE SYMBIOSIS OF PROXIES AND DATA SCRAPING

+

Experts say many of the world’s largest proxy providers have updated their public-facing branding to highlight their utility for training AI platforms, implying it is a primary use case for their residential proxies. That’s because AI services tend to rely on constantly mass-scraping the Internet for new text, images and video content that can be used to train large language models (LLMs).

+

NetNut and other proxy services have recast themselves as critical infrastructure for the AI scraping economy. Image: Synthient.com.

+

“AI companies depend on web-scraped content: for pre-training, for retrieval, for agent grounding, for search,” reads a report this month from Include Security that examines the prevalence of proxy SDKs in smart TV apps. “But the modern web isn’t scrapeable from a datacenter. Cloudflare, DataDome, HUMAN, among others throttle or block requests from known cloud IPs. The workaround is residential proxies. A scraping job routed through a Comcast or T-Mobile subscriber’s connection arrives at the target site from an IP that belongs to a paying residential customer.”

+

This non-stop content scraping has spawned more than 70 copyright infringement lawsuits against major tech companies that have acknowledged large-scale data scraping as a major source of the “brains” behind their commercial AI offerings. Ironically, much of that scraping is being aided by proxy services that are intimately tied to unofficial Android TV boxes and associated SDKs whose stated purpose is streaming pirated content.

+

The scraping activity has become so aggressive that it often overwhelms the targeted websites, preventing them from being reachable by legitimate visitors. In many reported cases, nonprofit organizations, libraries and universities have complained of constantly battling to keep their services online in the face of relentless data-scraping firms hiding behind residential proxy services.

+

A survey conducted last year by the Confederation of Open Access Repositories (COAR) found while some content scraping bots are rather innocuous, “others are sufficiently aggressive that they are increasingly causing service disruptions in repositories and other scholarly communications infrastructures.” More than 90 percent of survey respondents indicated their repository is encountering aggressive bots, usually more than once a week, and often leading to slow downs and service outages.

+

“Automated web scraping is nothing new, and has been the key technology underlying search engines such as Google for over 30 years,” wrote Brendan O’Connell, platform manager at the Directory of Open Access Journals (DOAJ), a free, community-curated index of peer-reviewed academic journals. “However, the current investor-fueled AI startup craze means there are now thousands of well-funded companies developing and deploying their own scraping tools to train AI models, alongside existing major players like OpenAI and Google.”

+

DON’T TOUCH THAT DIAL!

+

Across the United States, local communities are pushing back against the proliferation of new data centers aimed primarily at improving the capabilities of AI. But security experts say the general public remains largely unaware that using one of these unsanctioned Android TV boxes means their “smart TV” is almost certainly using a significant amount of bandwidth each month to help train modern AI models.

+

Even households without these sketchy TV boxes can still have their smart TVs turned into residential proxy nodes, just by downloading one of thousands of apps made available on Samsung and LG smart TVs. Spur said it recently scraped the LG and Samsung app stores and found that each had approximately 3,000 apps available for download. Many of these apps are simple games or utilities that state in the fine print that the user’s Internet connection will be used to download data and that they can opt out at any time.

+

Spur said it found that more than 42 percent of apps available for download via the webOS operating system on LG smart TVs include SDKs that turn one’s television into an always-on residential proxy node. More than a quarter of the apps made for Samsung’s Tizen operating system had similar residential proxy components, Spur found.

+

Image: Spur.us.

+

Experts say it’s questionable whether TV apps with proxy SDKs can obtain meaningful consent from users for installing an always-on proxy connection, particularly when anyone in a household — including children — can effectively opt the family TV into a residential proxy network just by installing a simple game or app.

+

“Privacy-policy disclosure is the wrong control surface for a TV,” Include Security wrote. “It is hard to scroll through a legal document navigated by arrow keys on a remote, and the in-app consent dialog doesn’t convey that a paying customer is about to route their scraping traffic through the user’s home internet.”

+

Spur’s head of research Sean Simmons told KrebsOnSecurity that most people do not have a working mental model for what it means to sell access to their residential IP address, no matter what device they are using.

+

“And on a TV, the gap is even wider,” Simmons said. “A one-time prompt navigated with a remote can disappear into the setup flow, while the app keeps monetizing the connection long after anyone remembers what they accepted.”

+

Simmons said LG and Samsung should follow the lead of other TV platforms that have already drawn a line against residential proxy providers, pointing to policies by Amazon that prohibit apps facilitating proxy services for third parties. Likewise the TV streaming device maker Roku reportedly now bars developers from using proxy SDKs and has removed apps that bundled them.

+

Piracy related apps pushing proxy SDKs onto unconsenting users. Image: Synthient.

+

Apps that turn one’s device into a residential proxy node are not limited to smart TVs and no-name streaming boxes, of course. As noted by the security firm Infoblox, mobile app developers can embed SDKs provided by the residential proxy networks into their products to monetize their software, allowing them to receive a small amount of money on each installation.

+

The result, Infoblox said, is that devices are frequently enrolled without the owner’s knowledge, typically through free applications such as VPNs, streaming apps, screensavers and “productivity” apps such as PDF viewers and break reminders.

+

All too often, these proxy services are beaconing out from employee devices brought into the workplace, Infoblox found. In a blog post earlier this month, Infoblox said it discovered that fully 65% of its customer base was querying one or more residential proxy related domains.

+

“We saw steady growth in these queries in 2025, with a 25% increase over the year to over 500 billion per month,” Infoblox wrote. “Over 90% of our pharmaceutical and food & beverage customers have queried residential proxy indicators. Perhaps even more concerning is that over 60% of government and banking customers have as well.”

+

Infoblox researchers Nick Sundvall and David Brunsdon warned that with residential proxies in the corporate environment, external access is granted to an organization’s IP space.

+

“If threat actors were to abuse the residential proxy to attack a third party, the third party’s incident response would, correctly, identify your residential proxy as the source,” they wrote. “Untangling that, by proving that you were the conduit and not the threat actor, costs time, creates legal exposure, and can damage your reputation. The stunning prevalence of these services within customer environments warrants attention from both network defenders and policy makers who should consider how the risks posed by residential proxies could be impacting their security posture.”

+]]> + + https://krebsonsecurity.com/2026/06/popa-botnet-linked-to-publicly-traded-israeli-firm/feed/ + 16 + + + + + + + diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/securityweek.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/securityweek.xml new file mode 100644 index 00000000..d6580353 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/securityweek.xml @@ -0,0 +1,90 @@ + + + + SecurityWeek + + https://www.securityweek.com/ + Cybersecurity News, Insights & Analysis + Fri, 03 Jul 2026 15:10:44 +0000 + en-US + + hourly + + 1 + https://wordpress.org/?v=7.0 + + + https://www.securityweek.com/wp-content/uploads/2023/01/cropped-SecurityWeek-Icon-32x32.jpeg + SecurityWeek + https://www.securityweek.com/ + 32 + 32 + + + In Other News: Canadian Hacker Jailed, Open Source Zero-Days, Two Sentenced for ATM Jackpotting + https://www.securityweek.com/in-other-news-canadian-hacker-jailed-open-source-zero-days-two-sentenced-for-atm-jackpotting/ + + + Fri, 03 Jul 2026 15:10:42 +0000 + + + + + https://www.securityweek.com/?p=47625 + + Noteworthy stories that might have slipped under the radar: Anonymous-linked Canadian hacker jailed, researcher drops zero-days in open source projects, Venezuelans sentenced in the US over ATM jackpotting.

+

The post In Other News: Canadian Hacker Jailed, Open Source Zero-Days, Two Sentenced for ATM Jackpotting appeared first on SecurityWeek.

+]]> + + + + + + Agentic AI Used to Conduct Ransomware Attack via Langflow + https://www.securityweek.com/agentic-ai-used-to-conduct-ransomware-attack-via-langflow/ + + + Fri, 03 Jul 2026 11:00:00 +0000 + + + + + + https://www.securityweek.com/?p=47618 + + Attack demonstrates how LLM agents can combine known exploitation techniques with real-time reasoning to automate complex, multi-stage intrusions.

+

The post Agentic AI Used to Conduct Ransomware Attack via Langflow appeared first on SecurityWeek.

+]]> + + + + + + Medtronic Data Breach Impacts 3.8 Million People + https://www.securityweek.com/medtronic-data-breach-impacts-3-8-million-people/ + + + Fri, 03 Jul 2026 10:00:00 +0000 + + + + + + https://www.securityweek.com/?p=47616 + + In April, ShinyHunters accessed the company’s corporate IT systems and stole patients’ personal and medical information.

+

The post Medtronic Data Breach Impacts 3.8 Million People appeared first on SecurityWeek.

+]]> + + + + + + diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/thehackernews.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/thehackernews.xml new file mode 100644 index 00000000..771c4871 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/thehackernews.xml @@ -0,0 +1,9 @@ +The Hacker Newshttps://thehackernews.comMost trusted, widely-read independent cybersecurity news source for everyone; supported by hackers and IT professionals — Send TIPs to admin@thehackernews.comen-usSun, 05 Jul 2026 21:16:08 +0530hourly1U.S. Government Entity Paid Kairos $1 Million in Data-Theft Extortion Casehttps://thehackernews.com/2026/07/us-government-entity-paid-kairos-group.htmlhttps://thehackernews.com/2026/07/us-government-entity-paid-kairos-group.htmlSat, 04 Jul 2026 18:17:53 +0530info@thehackernews.com (The Hacker News) +North Korean Hackers Publish 108 Malicious Packages and Extensions in PolinRider Campaignhttps://thehackernews.com/2026/07/north-korean-hackers-publish-108.htmlhttps://thehackernews.com/2026/07/north-korean-hackers-publish-108.htmlSat, 04 Jul 2026 16:47:24 +0530info@thehackernews.com (The Hacker News) +Unpatched Flaws Disclosed in Filesystem Bundled Into Millions of Embedded Deviceshttps://thehackernews.com/2026/07/unpatched-flaws-disclosed-in-filesystem.htmlhttps://thehackernews.com/2026/07/unpatched-flaws-disclosed-in-filesystem.htmlSat, 04 Jul 2026 01:49:31 +0530info@thehackernews.com (The Hacker News) diff --git a/PROJECTS/intermediate/security-news-scraper/testdata/feeds/theregister.xml b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/theregister.xml new file mode 100644 index 00000000..e76ea427 --- /dev/null +++ b/PROJECTS/intermediate/security-news-scraper/testdata/feeds/theregister.xml @@ -0,0 +1,48 @@ + + + + www.theregister.com - Articles + https://www.theregister.com + Articles from www.theregister.com + + + https://www.theregister.com/a/5266161 + https://www.theregister.com/security/2026/07/05/mfa-optional-banks-leave-safe-doors-and-accounts-wide-open-for-thieves-to-pillage/5266161 + Sun, 05 Jul 2026 17:01:00 +0200 + MFA-optional banks leave safe doors (and accounts) wide open for thieves to pillage + + security + + Fri, 03 Jul 2026 00:01:12 +0000 + + + + + + https://www.theregister.com/a/5266056 + https://www.theregister.com/security/2026/07/04/confidential-computings-core-trust-mechanism-is-broken-the-fix-may-not-exist/5266056 + Sat, 04 Jul 2026 12:03:00 +0200 + Confidential computing's core trust mechanism is broken. The fix may not exist + + security + + Fri, 03 Jul 2026 07:47:18 +0000 + + + + + + https://www.theregister.com/a/5266512 + https://www.theregister.com/security/2026/07/03/adapthealth-crooks-stole-our-passwords-patient-health-data/5266512 + Fri, 03 Jul 2026 16:29:00 +0200 + AdaptHealth says attackers sweet-talked their way into cloud systems and stole patient data + + security + + Fri, 03 Jul 2026 13:50:25 +0000 + + + + + +