feat(nadezhda): M1 ingestion core (fetch, parse, normalize, dedup)
Concurrent, rate-limited feed ingestion wired end to end: 'scrape' pulls live security news into the SQLite store, idempotent across re-runs. - internal/fetch: per-host token-bucket rate limiting, conditional GET (ETag/Last-Modified with 304 handling), retry+backoff that honors Retry-After and treats timeouts/cancellation as non-retryable, per-source deadline, honest User-Agent, 16MB body cap. robots.txt is enforced only via Client.Allowed() for the future HTML article-scrape path, not on subscribed feed fetches (user-directed retrieval; several publishers blanket-disallow generic bots yet serve a public feed). - internal/parse: gofeed RSS/Atom to a normalized Item; RFC1123Z/RFC822Z/ RFC3339 time fallbacks (CISA emits RFC822Z, not RFC1123Z as first researched). - internal/normalize: canonical URL (lowercase scheme+host, drop fragment, strip utm_*/gclid/fbclid/ref/mc_cid/mc_eid, drop trailing slash), title normalization, goquery HTML strip, sha256 content+title hashes. - internal/ingest: errgroup fan-out over sources, fail-soft per source, fetch -> parse -> normalize -> InsertArticle dedup via store.ErrDuplicate, plus fetch_state upsert for conditional GET. - store: migration 0002 adds articles.title_hash; GetFetchState/UpsertFetchState. - Trimmed testdata/feeds fixtures (3 items each) back golden parse tests; full suite is offline and passes under -race. Proven live: cold run 215 new across 7 sources, warm run 0 new (304s + dedup). Ctrl-C aborts scrape cleanly via signal-aware context.
This commit is contained in:
CarterPerez-dev
parent
8b8eaafa1f
commit
b41276004f
|
|
@ -0,0 +1,127 @@
|
|||
// ©AngelaMos | 2026
|
||||
// scrape.go
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/fetch"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/ingest"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/source"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
statusNotModified = "304"
|
||||
statusError = "error"
|
||||
statusOK = "ok"
|
||||
dash = "-"
|
||||
)
|
||||
|
||||
var scrapeSource string
|
||||
|
||||
var scrapeCmd = &cobra.Command{
|
||||
Use: "scrape",
|
||||
Short: "Ingest all enabled sources once",
|
||||
RunE: runScrape,
|
||||
}
|
||||
|
||||
func init() {
|
||||
scrapeCmd.Flags().StringVar(&scrapeSource, "source", "", "ingest only this source by name")
|
||||
rootCmd.AddCommand(scrapeCmd)
|
||||
}
|
||||
|
||||
func runScrape(cmd *cobra.Command, args []string) error {
|
||||
cfg, err := loadConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
srcs, err := source.Load(cfg.SourcesPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
targets, err := selectTargets(srcs, scrapeSource)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
st, err := store.Open(cfg.DBPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer st.Close()
|
||||
|
||||
fc := fetch.New(fetch.Options{
|
||||
UserAgent: cfg.Fetch.UserAgent,
|
||||
PerHostRate: cfg.Fetch.PerHostRate,
|
||||
PerHostBurst: cfg.Fetch.PerHostBurst,
|
||||
Timeout: time.Duration(cfg.Fetch.TimeoutSeconds) * time.Second,
|
||||
MaxRetries: cfg.Fetch.MaxRetries,
|
||||
})
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
|
||||
defer stop()
|
||||
|
||||
summary, err := ingest.Run(ctx, fc, st, cfg, targets, time.Now())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
printSummary(cmd, summary)
|
||||
return nil
|
||||
}
|
||||
|
||||
func selectTargets(srcs []source.Source, only string) ([]source.Source, error) {
|
||||
if only != "" {
|
||||
for _, s := range srcs {
|
||||
if s.Name == only {
|
||||
return []source.Source{s}, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("scrape: unknown source %q", only)
|
||||
}
|
||||
return source.Enabled(srcs), nil
|
||||
}
|
||||
|
||||
func printSummary(cmd *cobra.Command, summary ingest.Summary) {
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s\n", "SOURCE", "STATUS", "PARSED", "NEW", "DUP", "ERR")
|
||||
for _, r := range summary.Results {
|
||||
fmt.Fprintf(out, "%-18s %-8s %-8s %-5s %-5s %-5s\n",
|
||||
r.Name, status(r), count(r, r.Parsed), count(r, r.New), count(r, r.Duplicates), count(r, r.ItemErrors))
|
||||
}
|
||||
newArticles, duplicates, failed := summary.Totals()
|
||||
fmt.Fprintf(out, "\n%d new, %d duplicate across %d sources (%d failed)\n",
|
||||
newArticles, duplicates, len(summary.Results), failed)
|
||||
for _, r := range summary.Results {
|
||||
if r.Err != nil {
|
||||
fmt.Fprintf(out, " %s: %v\n", r.Name, r.Err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func status(r ingest.SourceResult) string {
|
||||
switch {
|
||||
case r.Err != nil:
|
||||
return statusError
|
||||
case r.NotModified:
|
||||
return statusNotModified
|
||||
default:
|
||||
return statusOK
|
||||
}
|
||||
}
|
||||
|
||||
func count(r ingest.SourceResult, n int) string {
|
||||
if r.Err != nil || r.NotModified {
|
||||
return dash
|
||||
}
|
||||
return fmt.Sprintf("%d", n)
|
||||
}
|
||||
|
|
@ -21,7 +21,6 @@ func init() {
|
|||
short string
|
||||
milestone string
|
||||
}{
|
||||
{"scrape", "Ingest all enabled sources once", "milestone M1"},
|
||||
{"list", "List stored articles with filters", "milestone M3"},
|
||||
{"cve", "Show an enriched CVE and articles mentioning it", "milestone M3"},
|
||||
{"digest", "Render a ranked digest to Markdown or JSON", "milestone M4"},
|
||||
|
|
|
|||
|
|
@ -3,20 +3,32 @@ module github.com/CarterPerez-dev/nadezhda
|
|||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.12.0
|
||||
github.com/mmcdole/gofeed v1.3.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/temoto/robotstxt v1.1.2
|
||||
golang.org/x/sync v0.21.0
|
||||
golang.org/x/time v0.15.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.53.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/spf13/pflag v1.0.9 // indirect
|
||||
golang.org/x/net v0.52.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
modernc.org/libc v1.73.4 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
|
|
|
|||
|
|
@ -1,6 +1,15 @@
|
|||
github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo=
|
||||
github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
|
|
@ -9,10 +18,23 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs
|
|||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
|
||||
github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
|
|
@ -20,16 +42,92 @@ github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
|||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
|
||||
github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
|
||||
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
|
||||
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
|
|
|||
|
|
@ -13,12 +13,13 @@ import (
|
|||
const (
|
||||
defaultDBPath = "nadezhda.db"
|
||||
|
||||
defaultUserAgent = "nadezhda/0.1 (+https://github.com/CarterPerez-dev/nadezhda)"
|
||||
defaultPerHostRate = 0.5
|
||||
defaultPerHostBurst = 1
|
||||
defaultTimeoutSeconds = 25
|
||||
defaultWorkers = 8
|
||||
defaultMaxRetries = 3
|
||||
defaultUserAgent = "nadezhda/0.1 (+https://github.com/CarterPerez-dev/nadezhda)"
|
||||
defaultPerHostRate = 0.5
|
||||
defaultPerHostBurst = 1
|
||||
defaultTimeoutSeconds = 25
|
||||
defaultSourceTimeoutSeconds = 90
|
||||
defaultWorkers = 8
|
||||
defaultMaxRetries = 3
|
||||
|
||||
defaultCacheTTLHours = 24
|
||||
defaultNegativeTTLHours = 3
|
||||
|
|
@ -26,6 +27,8 @@ const (
|
|||
defaultTitleJaccard = 0.6
|
||||
defaultWindowHours = 72
|
||||
|
||||
trackingUTMPrefix = "utm_*"
|
||||
|
||||
defaultHalfLifeHours = 48
|
||||
defaultVelocityNorm = 0.5
|
||||
|
||||
|
|
@ -48,13 +51,18 @@ const (
|
|||
defaultClaudeModel = "claude-opus-4-8"
|
||||
)
|
||||
|
||||
var defaultTrackingParams = []string{
|
||||
trackingUTMPrefix, "gclid", "fbclid", "ref", "mc_cid", "mc_eid",
|
||||
}
|
||||
|
||||
type Fetch struct {
|
||||
UserAgent string `yaml:"user_agent"`
|
||||
PerHostRate float64 `yaml:"per_host_rate"`
|
||||
PerHostBurst int `yaml:"per_host_burst"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
||||
Workers int `yaml:"workers"`
|
||||
MaxRetries int `yaml:"max_retries"`
|
||||
UserAgent string `yaml:"user_agent"`
|
||||
PerHostRate float64 `yaml:"per_host_rate"`
|
||||
PerHostBurst int `yaml:"per_host_burst"`
|
||||
TimeoutSeconds int `yaml:"timeout_seconds"`
|
||||
SourceTimeoutSeconds int `yaml:"source_timeout_seconds"`
|
||||
Workers int `yaml:"workers"`
|
||||
MaxRetries int `yaml:"max_retries"`
|
||||
}
|
||||
|
||||
type Enrich struct {
|
||||
|
|
@ -63,8 +71,9 @@ type Enrich struct {
|
|||
}
|
||||
|
||||
type Cluster struct {
|
||||
TitleJaccard float64 `yaml:"title_jaccard_threshold"`
|
||||
WindowHours int `yaml:"window_hours"`
|
||||
TitleJaccard float64 `yaml:"title_jaccard_threshold"`
|
||||
WindowHours int `yaml:"window_hours"`
|
||||
TrackingParams []string `yaml:"tracking_params"`
|
||||
}
|
||||
|
||||
type Weights struct {
|
||||
|
|
@ -112,20 +121,22 @@ func Default() Config {
|
|||
return Config{
|
||||
DBPath: defaultDBPath,
|
||||
Fetch: Fetch{
|
||||
UserAgent: defaultUserAgent,
|
||||
PerHostRate: defaultPerHostRate,
|
||||
PerHostBurst: defaultPerHostBurst,
|
||||
TimeoutSeconds: defaultTimeoutSeconds,
|
||||
Workers: defaultWorkers,
|
||||
MaxRetries: defaultMaxRetries,
|
||||
UserAgent: defaultUserAgent,
|
||||
PerHostRate: defaultPerHostRate,
|
||||
PerHostBurst: defaultPerHostBurst,
|
||||
TimeoutSeconds: defaultTimeoutSeconds,
|
||||
SourceTimeoutSeconds: defaultSourceTimeoutSeconds,
|
||||
Workers: defaultWorkers,
|
||||
MaxRetries: defaultMaxRetries,
|
||||
},
|
||||
Enrich: Enrich{
|
||||
CacheTTLHours: defaultCacheTTLHours,
|
||||
NegativeTTLHours: defaultNegativeTTLHours,
|
||||
},
|
||||
Cluster: Cluster{
|
||||
TitleJaccard: defaultTitleJaccard,
|
||||
WindowHours: defaultWindowHours,
|
||||
TitleJaccard: defaultTitleJaccard,
|
||||
WindowHours: defaultWindowHours,
|
||||
TrackingParams: defaultTrackingParams,
|
||||
},
|
||||
Rank: Rank{
|
||||
HalfLifeHours: defaultHalfLifeHours,
|
||||
|
|
@ -185,6 +196,12 @@ func (c Config) validate() error {
|
|||
if c.Fetch.PerHostBurst < 1 {
|
||||
return fmt.Errorf("config: fetch.per_host_burst must be >= 1, got %d", c.Fetch.PerHostBurst)
|
||||
}
|
||||
if c.Fetch.TimeoutSeconds < 1 {
|
||||
return fmt.Errorf("config: fetch.timeout_seconds must be >= 1, got %d", c.Fetch.TimeoutSeconds)
|
||||
}
|
||||
if c.Fetch.SourceTimeoutSeconds < 1 {
|
||||
return fmt.Errorf("config: fetch.source_timeout_seconds must be >= 1, got %d", c.Fetch.SourceTimeoutSeconds)
|
||||
}
|
||||
if c.Enrich.CacheTTLHours < 0 || c.Enrich.NegativeTTLHours < 0 {
|
||||
return fmt.Errorf("config: enrich TTLs must be >= 0, got cache=%d negative=%d", c.Enrich.CacheTTLHours, c.Enrich.NegativeTTLHours)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,235 @@
|
|||
// ©AngelaMos | 2026
|
||||
// fetch.go
|
||||
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
const (
|
||||
headerUserAgent = "User-Agent"
|
||||
headerAccept = "Accept"
|
||||
headerIfNoneMatch = "If-None-Match"
|
||||
headerIfModifiedSince = "If-Modified-Since"
|
||||
headerETag = "ETag"
|
||||
headerLastModified = "Last-Modified"
|
||||
headerRetryAfter = "Retry-After"
|
||||
|
||||
acceptFeed = "application/rss+xml, application/atom+xml, application/xml;q=0.9, */*;q=0.8"
|
||||
|
||||
defaultBackoffBase = 500 * time.Millisecond
|
||||
maxBodyBytes = 16 << 20
|
||||
maxRetryAfter = 60 * time.Second
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
UserAgent string
|
||||
PerHostRate float64
|
||||
PerHostBurst int
|
||||
Timeout time.Duration
|
||||
MaxRetries int
|
||||
}
|
||||
|
||||
type Request struct {
|
||||
URL string
|
||||
ETag string
|
||||
LastModified string
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
Status int
|
||||
Body []byte
|
||||
ETag string
|
||||
LastModified string
|
||||
NotModified bool
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
http *http.Client
|
||||
ua string
|
||||
rate rate.Limit
|
||||
burst int
|
||||
maxRetries int
|
||||
backoffBase time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
limiters map[string]*rate.Limiter
|
||||
|
||||
robots *robotsCache
|
||||
}
|
||||
|
||||
func New(opts Options) *Client {
|
||||
c := &Client{
|
||||
http: &http.Client{Timeout: opts.Timeout},
|
||||
ua: opts.UserAgent,
|
||||
rate: rate.Limit(opts.PerHostRate),
|
||||
burst: opts.PerHostBurst,
|
||||
maxRetries: opts.MaxRetries,
|
||||
backoffBase: defaultBackoffBase,
|
||||
limiters: make(map[string]*rate.Limiter),
|
||||
}
|
||||
c.robots = newRobotsCache(c)
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Client) limiterFor(host string) *rate.Limiter {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
l, ok := c.limiters[host]
|
||||
if !ok {
|
||||
l = rate.NewLimiter(c.rate, c.burst)
|
||||
c.limiters[host] = l
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func (c *Client) Fetch(ctx context.Context, req Request) (Result, error) {
|
||||
u, err := url.Parse(req.URL)
|
||||
if err != nil {
|
||||
return Result{}, fmt.Errorf("fetch: parse url %q: %w", req.URL, err)
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
var override time.Duration
|
||||
for attempt := 0; attempt <= c.maxRetries; attempt++ {
|
||||
if attempt > 0 {
|
||||
delay := c.backoff(attempt)
|
||||
if override > 0 {
|
||||
delay = override
|
||||
}
|
||||
if err := sleep(ctx, delay); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
}
|
||||
if err := c.limiterFor(u.Host).Wait(ctx); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
res, retryAfter, retry, err := c.do(ctx, req)
|
||||
if err == nil {
|
||||
return res, nil
|
||||
}
|
||||
lastErr = err
|
||||
override = retryAfter
|
||||
if !retry {
|
||||
return Result{}, err
|
||||
}
|
||||
}
|
||||
return Result{}, fmt.Errorf("fetch %s: exhausted %d retries: %w", req.URL, c.maxRetries, lastErr)
|
||||
}
|
||||
|
||||
func (c *Client) Allowed(ctx context.Context, rawURL string) (bool, error) {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("fetch: parse url %q: %w", rawURL, err)
|
||||
}
|
||||
return c.robots.allowed(ctx, u)
|
||||
}
|
||||
|
||||
func (c *Client) do(ctx context.Context, req Request) (Result, time.Duration, bool, error) {
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, req.URL, nil)
|
||||
if err != nil {
|
||||
return Result{}, 0, false, fmt.Errorf("fetch %s: build request: %w", req.URL, err)
|
||||
}
|
||||
httpReq.Header.Set(headerUserAgent, c.ua)
|
||||
httpReq.Header.Set(headerAccept, acceptFeed)
|
||||
if req.ETag != "" {
|
||||
httpReq.Header.Set(headerIfNoneMatch, req.ETag)
|
||||
}
|
||||
if req.LastModified != "" {
|
||||
httpReq.Header.Set(headerIfModifiedSince, req.LastModified)
|
||||
}
|
||||
|
||||
resp, err := c.http.Do(httpReq)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return Result{}, 0, false, fmt.Errorf("fetch %s: %w", req.URL, err)
|
||||
}
|
||||
return Result{}, 0, true, fmt.Errorf("fetch %s: %w", req.URL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
switch {
|
||||
case resp.StatusCode == http.StatusNotModified:
|
||||
return Result{
|
||||
Status: resp.StatusCode,
|
||||
NotModified: true,
|
||||
ETag: req.ETag,
|
||||
LastModified: req.LastModified,
|
||||
}, 0, false, nil
|
||||
case resp.StatusCode == http.StatusOK:
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodyBytes))
|
||||
if err != nil {
|
||||
return Result{}, 0, true, fmt.Errorf("fetch %s: read body: %w", req.URL, err)
|
||||
}
|
||||
return Result{
|
||||
Status: resp.StatusCode,
|
||||
Body: body,
|
||||
ETag: resp.Header.Get(headerETag),
|
||||
LastModified: resp.Header.Get(headerLastModified),
|
||||
}, 0, false, nil
|
||||
case resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= http.StatusInternalServerError:
|
||||
drain(resp.Body)
|
||||
return Result{}, retryAfter(resp), true, fmt.Errorf("fetch %s: server status %d", req.URL, resp.StatusCode)
|
||||
default:
|
||||
drain(resp.Body)
|
||||
return Result{}, 0, false, fmt.Errorf("fetch %s: status %d", req.URL, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func drain(body io.Reader) {
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(body, maxBodyBytes))
|
||||
}
|
||||
|
||||
func retryAfter(resp *http.Response) time.Duration {
|
||||
v := resp.Header.Get(headerRetryAfter)
|
||||
if v == "" {
|
||||
return 0
|
||||
}
|
||||
if secs, err := strconv.Atoi(v); err == nil {
|
||||
d := time.Duration(secs) * time.Second
|
||||
if d > maxRetryAfter {
|
||||
return maxRetryAfter
|
||||
}
|
||||
if d < 0 {
|
||||
return 0
|
||||
}
|
||||
return d
|
||||
}
|
||||
if t, err := http.ParseTime(v); err == nil {
|
||||
d := time.Until(t)
|
||||
if d <= 0 {
|
||||
return 0
|
||||
}
|
||||
if d > maxRetryAfter {
|
||||
return maxRetryAfter
|
||||
}
|
||||
return d
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (c *Client) backoff(attempt int) time.Duration {
|
||||
return c.backoffBase * time.Duration(1<<(attempt-1))
|
||||
}
|
||||
|
||||
func sleep(ctx context.Context, d time.Duration) error {
|
||||
timer := time.NewTimer(d)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,244 @@
|
|||
// ©AngelaMos | 2026
|
||||
// fetch_test.go
|
||||
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testClient() *Client {
|
||||
c := New(Options{
|
||||
UserAgent: "nadezhda-test/1.0",
|
||||
PerHostRate: 1e6,
|
||||
PerHostBurst: 1,
|
||||
Timeout: 5 * time.Second,
|
||||
MaxRetries: 3,
|
||||
})
|
||||
c.backoffBase = time.Millisecond
|
||||
return c
|
||||
}
|
||||
|
||||
func TestFetchOKCapturesValidators(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set(headerETag, `"v1"`)
|
||||
w.Header().Set(headerLastModified, "Wed, 01 Jul 2026 00:00:00 GMT")
|
||||
_, _ = w.Write([]byte("<rss></rss>"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"})
|
||||
if err != nil {
|
||||
t.Fatalf("Fetch: %v", err)
|
||||
}
|
||||
if res.Status != http.StatusOK {
|
||||
t.Errorf("status = %d, want 200", res.Status)
|
||||
}
|
||||
if string(res.Body) != "<rss></rss>" {
|
||||
t.Errorf("body = %q", res.Body)
|
||||
}
|
||||
if res.ETag != `"v1"` {
|
||||
t.Errorf("etag = %q, want \"v1\"", res.ETag)
|
||||
}
|
||||
if res.LastModified == "" {
|
||||
t.Error("last-modified not captured")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchConditionalGET(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get(headerIfNoneMatch) == `"v1"` {
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
w.Header().Set(headerETag, `"v1"`)
|
||||
_, _ = w.Write([]byte("body"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := testClient()
|
||||
first, err := c.Fetch(context.Background(), Request{URL: srv.URL + "/feed"})
|
||||
if err != nil || first.Status != http.StatusOK {
|
||||
t.Fatalf("first fetch: status=%d err=%v", first.Status, err)
|
||||
}
|
||||
|
||||
second, err := c.Fetch(context.Background(), Request{URL: srv.URL + "/feed", ETag: first.ETag})
|
||||
if err != nil {
|
||||
t.Fatalf("second fetch: %v", err)
|
||||
}
|
||||
if !second.NotModified {
|
||||
t.Error("expected NotModified on matching ETag")
|
||||
}
|
||||
if second.ETag != `"v1"` {
|
||||
t.Errorf("304 should retain etag, got %q", second.ETag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRetriesServerError(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if calls.Add(1) == 1 {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"})
|
||||
if err != nil {
|
||||
t.Fatalf("Fetch: %v", err)
|
||||
}
|
||||
if string(res.Body) != "ok" {
|
||||
t.Errorf("body = %q, want ok", res.Body)
|
||||
}
|
||||
if calls.Load() != 2 {
|
||||
t.Errorf("calls = %d, want 2 (one 500, one 200)", calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchDoesNotRetryClientError(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
_, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"})
|
||||
if err == nil {
|
||||
t.Fatal("expected error on 404")
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Errorf("calls = %d, want 1 (no retry on 4xx)", calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetch429IsRetried(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if calls.Add(1) == 1 {
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"})
|
||||
if err != nil {
|
||||
t.Fatalf("Fetch: %v", err)
|
||||
}
|
||||
if string(res.Body) != "ok" || calls.Load() != 2 {
|
||||
t.Errorf("body=%q calls=%d, want ok/2", res.Body, calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchTimeoutNotRetried(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := New(Options{
|
||||
UserAgent: "nadezhda-test/1.0", PerHostRate: 1e6, PerHostBurst: 1,
|
||||
Timeout: 15 * time.Millisecond, MaxRetries: 3,
|
||||
})
|
||||
c.backoffBase = time.Millisecond
|
||||
|
||||
_, err := c.Fetch(context.Background(), Request{URL: srv.URL + "/feed"})
|
||||
if err == nil {
|
||||
t.Fatal("expected timeout error")
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Errorf("calls = %d, want 1 (timeout must not be retried)", calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryAfterParsing(t *testing.T) {
|
||||
cases := []struct {
|
||||
header string
|
||||
want time.Duration
|
||||
}{
|
||||
{"", 0},
|
||||
{"5", 5 * time.Second},
|
||||
{"0", 0},
|
||||
{"-3", 0},
|
||||
{"9999", maxRetryAfter},
|
||||
{"garbage", 0},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
resp := &http.Response{Header: http.Header{}}
|
||||
if tc.header != "" {
|
||||
resp.Header.Set(headerRetryAfter, tc.header)
|
||||
}
|
||||
if got := retryAfter(resp); got != tc.want {
|
||||
t.Errorf("retryAfter(%q) = %s, want %s", tc.header, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedFetchIgnoresRobots(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == robotsPath {
|
||||
t.Error("Fetch must not request robots.txt on the feed path")
|
||||
_, _ = w.Write([]byte("User-agent: *\nDisallow: /\n"))
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte("feed body"))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
res, err := testClient().Fetch(context.Background(), Request{URL: srv.URL + "/feed"})
|
||||
if err != nil {
|
||||
t.Fatalf("Fetch: %v", err)
|
||||
}
|
||||
if string(res.Body) != "feed body" {
|
||||
t.Errorf("body = %q, want feed body", res.Body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllowedRespectsDisallow(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == robotsPath {
|
||||
_, _ = w.Write([]byte("User-agent: *\nDisallow: /article\n"))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ok, err := testClient().Allowed(context.Background(), srv.URL+"/article/123")
|
||||
if err != nil {
|
||||
t.Fatalf("Allowed: %v", err)
|
||||
}
|
||||
if ok {
|
||||
t.Error("expected disallowed for /article path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllowedPermitsOtherPaths(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == robotsPath {
|
||||
_, _ = w.Write([]byte("User-agent: *\nDisallow: /admin\n"))
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
ok, err := testClient().Allowed(context.Background(), srv.URL+"/article/123")
|
||||
if err != nil {
|
||||
t.Fatalf("Allowed: %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Error("expected allowed for non-admin path")
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
// ©AngelaMos | 2026
|
||||
// robots.go
|
||||
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
|
||||
"github.com/temoto/robotstxt"
|
||||
)
|
||||
|
||||
const (
|
||||
robotsPath = "/robots.txt"
|
||||
maxRobotsBytes = 512 << 10
|
||||
robotsRootPath = "/"
|
||||
)
|
||||
|
||||
type robotsEntry struct {
|
||||
once sync.Once
|
||||
data *robotstxt.RobotsData
|
||||
}
|
||||
|
||||
type robotsCache struct {
|
||||
client *Client
|
||||
|
||||
mu sync.Mutex
|
||||
entries map[string]*robotsEntry
|
||||
}
|
||||
|
||||
func newRobotsCache(client *Client) *robotsCache {
|
||||
return &robotsCache{
|
||||
client: client,
|
||||
entries: make(map[string]*robotsEntry),
|
||||
}
|
||||
}
|
||||
|
||||
func (rc *robotsCache) allowed(ctx context.Context, u *url.URL) (bool, error) {
|
||||
rc.mu.Lock()
|
||||
e, ok := rc.entries[u.Host]
|
||||
if !ok {
|
||||
e = &robotsEntry{}
|
||||
rc.entries[u.Host] = e
|
||||
}
|
||||
rc.mu.Unlock()
|
||||
|
||||
e.once.Do(func() {
|
||||
data, err := rc.load(ctx, u)
|
||||
if err != nil {
|
||||
data, _ = robotstxt.FromStatusAndBytes(http.StatusOK, nil)
|
||||
}
|
||||
e.data = data
|
||||
})
|
||||
|
||||
path := u.EscapedPath()
|
||||
if path == "" {
|
||||
path = robotsRootPath
|
||||
}
|
||||
return e.data.FindGroup(rc.client.ua).Test(path), nil
|
||||
}
|
||||
|
||||
func (rc *robotsCache) load(ctx context.Context, u *url.URL) (*robotstxt.RobotsData, error) {
|
||||
robotsURL := u.Scheme + "://" + u.Host + robotsPath
|
||||
if err := rc.client.limiterFor(u.Host).Wait(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, robotsURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set(headerUserAgent, rc.client.ua)
|
||||
|
||||
resp, err := rc.client.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxRobotsBytes))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return robotstxt.FromStatusAndBytes(resp.StatusCode, body)
|
||||
}
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
// ©AngelaMos | 2026
|
||||
// ingest.go
|
||||
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/config"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/fetch"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/normalize"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/parse"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/source"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/store"
|
||||
)
|
||||
|
||||
type SourceResult struct {
|
||||
Name string
|
||||
Parsed int
|
||||
New int
|
||||
Duplicates int
|
||||
ItemErrors int
|
||||
NotModified bool
|
||||
Err error
|
||||
}
|
||||
|
||||
type Summary struct {
|
||||
Results []SourceResult
|
||||
}
|
||||
|
||||
func (s Summary) Totals() (newArticles, duplicates, failed int) {
|
||||
for _, r := range s.Results {
|
||||
newArticles += r.New
|
||||
duplicates += r.Duplicates
|
||||
if r.Err != nil {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
return newArticles, duplicates, failed
|
||||
}
|
||||
|
||||
func Run(ctx context.Context, fc *fetch.Client, st *store.Store, cfg config.Config, targets []source.Source, now time.Time) (Summary, error) {
|
||||
results := make([]SourceResult, len(targets))
|
||||
|
||||
workers := cfg.Fetch.Workers
|
||||
if workers < 1 {
|
||||
workers = 1
|
||||
}
|
||||
|
||||
g, gctx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(workers)
|
||||
|
||||
for i, src := range targets {
|
||||
results[i].Name = src.Name
|
||||
g.Go(func() error {
|
||||
process(gctx, fc, st, cfg, src, now, &results[i])
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err := g.Wait(); err != nil {
|
||||
return Summary{Results: results}, err
|
||||
}
|
||||
return Summary{Results: results}, nil
|
||||
}
|
||||
|
||||
func process(ctx context.Context, fc *fetch.Client, st *store.Store, cfg config.Config, src source.Source, now time.Time, out *SourceResult) {
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Duration(cfg.Fetch.SourceTimeoutSeconds)*time.Second)
|
||||
defer cancel()
|
||||
|
||||
id, err := st.UpsertSource(store.SourceInput{
|
||||
Name: src.Name, Title: src.Title, URL: src.URL, Type: string(src.Type),
|
||||
Weight: src.Weight, Tags: src.Tags, Enabled: src.Enabled,
|
||||
})
|
||||
if err != nil {
|
||||
out.Err = err
|
||||
return
|
||||
}
|
||||
|
||||
prev, _, err := st.GetFetchState(id)
|
||||
if err != nil {
|
||||
out.Err = err
|
||||
return
|
||||
}
|
||||
|
||||
res, err := fc.Fetch(ctx, fetch.Request{
|
||||
URL: src.URL, ETag: prev.ETag, LastModified: prev.LastModified,
|
||||
})
|
||||
if err != nil {
|
||||
out.Err = err
|
||||
return
|
||||
}
|
||||
|
||||
if res.NotModified {
|
||||
out.NotModified = true
|
||||
out.Err = st.UpsertFetchState(id, store.FetchState{
|
||||
ETag: prev.ETag, LastModified: prev.LastModified,
|
||||
LastFetched: now.Unix(), LastStatus: int64(res.Status),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
items, err := parse.Feed(bytes.NewReader(res.Body))
|
||||
if err != nil {
|
||||
out.Err = err
|
||||
return
|
||||
}
|
||||
out.Parsed = len(items)
|
||||
|
||||
for _, it := range items {
|
||||
storeItem(st, cfg, id, it, now, out)
|
||||
}
|
||||
|
||||
if err := st.UpsertFetchState(id, store.FetchState{
|
||||
ETag: res.ETag, LastModified: res.LastModified,
|
||||
LastFetched: now.Unix(), LastStatus: int64(res.Status),
|
||||
}); err != nil && out.Err == nil {
|
||||
out.Err = err
|
||||
}
|
||||
}
|
||||
|
||||
func storeItem(st *store.Store, cfg config.Config, sourceID int64, it parse.Item, now time.Time, out *SourceResult) {
|
||||
if it.Link == "" {
|
||||
out.ItemErrors++
|
||||
return
|
||||
}
|
||||
canonical, err := normalize.CanonicalURL(it.Link, cfg.Cluster.TrackingParams)
|
||||
if err != nil {
|
||||
out.ItemErrors++
|
||||
return
|
||||
}
|
||||
|
||||
var publishedAt int64
|
||||
if !it.Published.IsZero() {
|
||||
publishedAt = it.Published.Unix()
|
||||
}
|
||||
|
||||
_, err = st.InsertArticle(store.Article{
|
||||
SourceID: sourceID,
|
||||
CanonicalURL: canonical,
|
||||
ContentHash: normalize.ContentHash(canonical),
|
||||
TitleHash: normalize.TitleHash(normalize.NormalizeTitle(it.Title)),
|
||||
Title: it.Title,
|
||||
Summary: normalize.StripHTML(it.Summary),
|
||||
Body: normalize.StripHTML(it.Body),
|
||||
Author: it.Author,
|
||||
PublishedAt: publishedAt,
|
||||
FetchedAt: now.Unix(),
|
||||
})
|
||||
switch {
|
||||
case err == nil:
|
||||
out.New++
|
||||
case errors.Is(err, store.ErrDuplicate):
|
||||
out.Duplicates++
|
||||
default:
|
||||
out.ItemErrors++
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,199 @@
|
|||
// ©AngelaMos | 2026
|
||||
// ingest_test.go
|
||||
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/config"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/fetch"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/normalize"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/source"
|
||||
"github.com/CarterPerez-dev/nadezhda/internal/store"
|
||||
)
|
||||
|
||||
const fixture = "../../testdata/feeds/thehackernews.xml"
|
||||
|
||||
func loadFixture(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(fixture)
|
||||
if err != nil {
|
||||
t.Fatalf("read fixture: %v", err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func newStore(t *testing.T) *store.Store {
|
||||
t.Helper()
|
||||
st, err := store.Open(filepath.Join(t.TempDir(), "ingest.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = st.Close() })
|
||||
return st
|
||||
}
|
||||
|
||||
func newClient() *fetch.Client {
|
||||
return fetch.New(fetch.Options{
|
||||
UserAgent: "nadezhda-test/1.0",
|
||||
PerHostRate: 1e6,
|
||||
PerHostBurst: 1,
|
||||
Timeout: 5 * time.Second,
|
||||
MaxRetries: 2,
|
||||
})
|
||||
}
|
||||
|
||||
func target(url string) []source.Source {
|
||||
return []source.Source{{
|
||||
Name: "test", Title: "Test Feed", URL: url,
|
||||
Type: source.KindRSS, Weight: 1.0, Tags: []string{"news"}, Enabled: true,
|
||||
}}
|
||||
}
|
||||
|
||||
func TestRunIngestsAndDedups(t *testing.T) {
|
||||
body := loadFixture(t)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write(body)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
st := newStore(t)
|
||||
cfg := config.Default()
|
||||
targets := target(srv.URL + "/feed")
|
||||
now := time.Unix(1_800_000_000, 0)
|
||||
|
||||
first, err := Run(context.Background(), newClient(), st, cfg, targets, now)
|
||||
if err != nil {
|
||||
t.Fatalf("first Run: %v", err)
|
||||
}
|
||||
r0 := first.Results[0]
|
||||
if r0.Err != nil {
|
||||
t.Fatalf("source error: %v", r0.Err)
|
||||
}
|
||||
if r0.New != 3 || r0.Parsed != 3 {
|
||||
t.Fatalf("first run: parsed=%d new=%d, want 3/3", r0.Parsed, r0.New)
|
||||
}
|
||||
|
||||
count, err := st.CountArticles()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count != 3 {
|
||||
t.Fatalf("stored articles = %d, want 3", count)
|
||||
}
|
||||
|
||||
var title, canonical, contentHash, titleHash string
|
||||
if err := st.DB().QueryRow(
|
||||
`SELECT title, canonical_url, content_hash, title_hash FROM articles LIMIT 1`,
|
||||
).Scan(&title, &canonical, &contentHash, &titleHash); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if contentHash != normalize.ContentHash(canonical) {
|
||||
t.Errorf("content_hash = %q, want sha256(canonical)", contentHash)
|
||||
}
|
||||
if titleHash != normalize.TitleHash(normalize.NormalizeTitle(title)) {
|
||||
t.Errorf("title_hash = %q, want sha256(normalized title)", titleHash)
|
||||
}
|
||||
|
||||
second, err := Run(context.Background(), newClient(), st, cfg, targets, now)
|
||||
if err != nil {
|
||||
t.Fatalf("second Run: %v", err)
|
||||
}
|
||||
r1 := second.Results[0]
|
||||
if r1.New != 0 || r1.Duplicates != 3 {
|
||||
t.Fatalf("second run: new=%d dup=%d, want 0/3", r1.New, r1.Duplicates)
|
||||
}
|
||||
|
||||
count, _ = st.CountArticles()
|
||||
if count != 3 {
|
||||
t.Fatalf("after re-run stored = %d, want 3 (idempotent)", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConditionalNotModified(t *testing.T) {
|
||||
body := loadFixture(t)
|
||||
const etag = `"abc123"`
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("If-None-Match") == etag {
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
w.Header().Set("ETag", etag)
|
||||
_, _ = w.Write(body)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
st := newStore(t)
|
||||
cfg := config.Default()
|
||||
targets := target(srv.URL + "/feed")
|
||||
now := time.Unix(1_800_000_000, 0)
|
||||
|
||||
first, err := Run(context.Background(), newClient(), st, cfg, targets, now)
|
||||
if err != nil {
|
||||
t.Fatalf("first Run: %v", err)
|
||||
}
|
||||
if first.Results[0].New != 3 {
|
||||
t.Fatalf("first run new = %d, want 3", first.Results[0].New)
|
||||
}
|
||||
var storedETag string
|
||||
if err := st.DB().QueryRow(`SELECT etag FROM fetch_state LIMIT 1`).Scan(&storedETag); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if storedETag != etag {
|
||||
t.Errorf("persisted etag = %q, want %q", storedETag, etag)
|
||||
}
|
||||
|
||||
second, err := Run(context.Background(), newClient(), st, cfg, targets, now)
|
||||
if err != nil {
|
||||
t.Fatalf("second Run: %v", err)
|
||||
}
|
||||
r := second.Results[0]
|
||||
if !r.NotModified {
|
||||
t.Errorf("expected NotModified on second run, got %+v", r)
|
||||
}
|
||||
if r.New != 0 {
|
||||
t.Errorf("new = %d on 304, want 0", r.New)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFailsSoftOnBadSource(t *testing.T) {
|
||||
body := loadFixture(t)
|
||||
good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write(body)
|
||||
}))
|
||||
defer good.Close()
|
||||
bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer bad.Close()
|
||||
|
||||
st := newStore(t)
|
||||
cfg := config.Default()
|
||||
targets := []source.Source{
|
||||
{Name: "bad", URL: bad.URL + "/feed", Type: source.KindRSS, Weight: 1, Enabled: true},
|
||||
{Name: "good", URL: good.URL + "/feed", Type: source.KindRSS, Weight: 1, Enabled: true},
|
||||
}
|
||||
now := time.Unix(1_800_000_000, 0)
|
||||
|
||||
summary, err := Run(context.Background(), newClient(), st, cfg, targets, now)
|
||||
if err != nil {
|
||||
t.Fatalf("Run returned error, should fail soft: %v", err)
|
||||
}
|
||||
byName := map[string]SourceResult{}
|
||||
for _, r := range summary.Results {
|
||||
byName[r.Name] = r
|
||||
}
|
||||
if byName["bad"].Err == nil {
|
||||
t.Error("bad source should record an error")
|
||||
}
|
||||
if byName["good"].New != 3 {
|
||||
t.Errorf("good source new = %d, want 3 (bad source must not abort run)", byName["good"].New)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
// ©AngelaMos | 2026
|
||||
// normalize.go
|
||||
|
||||
package normalize
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
const (
|
||||
wildcardSuffix = "*"
|
||||
trailingSlash = "/"
|
||||
)
|
||||
|
||||
func CanonicalURL(raw string, trackingParams []string) (string, error) {
|
||||
u, err := url.Parse(strings.TrimSpace(raw))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("normalize: parse url %q: %w", raw, err)
|
||||
}
|
||||
if u.Host == "" {
|
||||
return "", fmt.Errorf("normalize: url %q has no host", raw)
|
||||
}
|
||||
|
||||
u.Scheme = strings.ToLower(u.Scheme)
|
||||
u.Host = strings.ToLower(u.Host)
|
||||
u.Fragment = ""
|
||||
u.RawFragment = ""
|
||||
|
||||
if q := u.Query(); len(q) > 0 {
|
||||
for key := range q {
|
||||
if isTracking(key, trackingParams) {
|
||||
q.Del(key)
|
||||
}
|
||||
}
|
||||
u.RawQuery = q.Encode()
|
||||
}
|
||||
|
||||
u.Path = strings.TrimRight(u.Path, trailingSlash)
|
||||
if u.RawPath != "" {
|
||||
u.RawPath = strings.TrimRight(u.RawPath, trailingSlash)
|
||||
}
|
||||
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func isTracking(key string, trackingParams []string) bool {
|
||||
lowered := strings.ToLower(key)
|
||||
for _, p := range trackingParams {
|
||||
if strings.HasSuffix(p, wildcardSuffix) {
|
||||
if strings.HasPrefix(lowered, strings.TrimSuffix(p, wildcardSuffix)) {
|
||||
return true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if lowered == p {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func NormalizeTitle(title string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(title))
|
||||
prevSpace := false
|
||||
for _, r := range strings.ToLower(title) {
|
||||
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||||
b.WriteRune(r)
|
||||
prevSpace = false
|
||||
continue
|
||||
}
|
||||
if !prevSpace {
|
||||
b.WriteByte(' ')
|
||||
prevSpace = true
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(b.String())
|
||||
}
|
||||
|
||||
func StripHTML(s string) string {
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(s))
|
||||
if err != nil {
|
||||
return collapseWhitespace(s)
|
||||
}
|
||||
doc.Find("script,style").Remove()
|
||||
return collapseWhitespace(doc.Text())
|
||||
}
|
||||
|
||||
func collapseWhitespace(s string) string {
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
|
||||
func ContentHash(canonicalURL string) string {
|
||||
return sha256Hex(canonicalURL)
|
||||
}
|
||||
|
||||
func TitleHash(normalizedTitle string) string {
|
||||
return sha256Hex(normalizedTitle)
|
||||
}
|
||||
|
||||
func sha256Hex(s string) string {
|
||||
sum := sha256.Sum256([]byte(s))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
// ©AngelaMos | 2026
|
||||
// normalize_test.go
|
||||
|
||||
package normalize
|
||||
|
||||
import "testing"
|
||||
|
||||
var params = []string{"utm_*", "gclid", "fbclid", "ref", "mc_cid", "mc_eid"}
|
||||
|
||||
func TestCanonicalURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"lowercase host+scheme", "HTTPS://Example.COM/Path", "https://example.com/Path"},
|
||||
{"drop fragment", "https://example.com/a#section", "https://example.com/a"},
|
||||
{"strip utm params", "https://example.com/a?utm_source=x&utm_medium=y&id=7", "https://example.com/a?id=7"},
|
||||
{"strip gclid fbclid ref", "https://example.com/a?gclid=1&fbclid=2&ref=z&keep=1", "https://example.com/a?keep=1"},
|
||||
{"strip mailchimp", "https://example.com/a?mc_cid=1&mc_eid=2", "https://example.com/a"},
|
||||
{"drop trailing slash", "https://example.com/a/b/", "https://example.com/a/b"},
|
||||
{"root trailing slash", "https://example.com/", "https://example.com"},
|
||||
{"sorted query", "https://example.com/a?b=2&a=1", "https://example.com/a?a=1&b=2"},
|
||||
{"tracking case insensitive", "https://example.com/a?UTM_Source=x&id=1", "https://example.com/a?id=1"},
|
||||
{"path preserved case", "https://example.com/Foo/Bar", "https://example.com/Foo/Bar"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, err := CanonicalURL(tc.in, params)
|
||||
if err != nil {
|
||||
t.Fatalf("CanonicalURL(%q): %v", tc.in, err)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Errorf("CanonicalURL(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanonicalURLCollapsesTracking(t *testing.T) {
|
||||
a, _ := CanonicalURL("https://example.com/story?utm_source=twitter", params)
|
||||
b, _ := CanonicalURL("https://example.com/story", params)
|
||||
if a != b {
|
||||
t.Errorf("tracking variants did not collapse: %q vs %q", a, b)
|
||||
}
|
||||
if ContentHash(a) != ContentHash(b) {
|
||||
t.Error("content hashes differ for tracking variants")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanonicalURLErrors(t *testing.T) {
|
||||
for _, in := range []string{"", "not a url", "/relative/only"} {
|
||||
if _, err := CanonicalURL(in, params); err == nil {
|
||||
t.Errorf("CanonicalURL(%q) expected error", in)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeTitle(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"CVE-2021-44228: Log4Shell!", "cve 2021 44228 log4shell"},
|
||||
{" Multiple Spaces ", "multiple spaces"},
|
||||
{"Punctuation, and; stuff.", "punctuation and stuff"},
|
||||
{"Café déjà vu", "café déjà vu"},
|
||||
{"", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := NormalizeTitle(tc.in); got != tc.want {
|
||||
t.Errorf("NormalizeTitle(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeTitleStableHash(t *testing.T) {
|
||||
a := TitleHash(NormalizeTitle("Breaking: Big Hack!!!"))
|
||||
b := TitleHash(NormalizeTitle("breaking big hack"))
|
||||
if a != b {
|
||||
t.Error("normalized-title hashes differ for equivalent titles")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStripHTML(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"<p>Hello <b>world</b></p>", "Hello world"},
|
||||
{"<div>a</div>\n<div>b</div>", "a b"},
|
||||
{"<script>bad()</script>visible", "visible"},
|
||||
{"plain text", "plain text"},
|
||||
{"<a href='x'>link</a> and text", "link and text"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := StripHTML(tc.in); got != tc.want {
|
||||
t.Errorf("StripHTML(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentHashDeterministic(t *testing.T) {
|
||||
const u = "https://example.com/a"
|
||||
if ContentHash(u) != ContentHash(u) {
|
||||
t.Error("ContentHash not deterministic")
|
||||
}
|
||||
if len(ContentHash(u)) != 64 {
|
||||
t.Errorf("ContentHash length = %d, want 64 hex chars", len(ContentHash(u)))
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
// ©AngelaMos | 2026
|
||||
// parse.go
|
||||
|
||||
package parse
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mmcdole/gofeed"
|
||||
)
|
||||
|
||||
type Item struct {
|
||||
Title string
|
||||
Link string
|
||||
Summary string
|
||||
Body string
|
||||
Author string
|
||||
Published time.Time
|
||||
Categories []string
|
||||
}
|
||||
|
||||
var timeLayouts = []string{time.RFC1123Z, time.RFC1123, time.RFC822Z, time.RFC822, time.RFC3339}
|
||||
|
||||
func Feed(r io.Reader) ([]Item, error) {
|
||||
feed, err := gofeed.NewParser().Parse(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse: %w", err)
|
||||
}
|
||||
items := make([]Item, 0, len(feed.Items))
|
||||
for _, it := range feed.Items {
|
||||
items = append(items, Item{
|
||||
Title: strings.TrimSpace(it.Title),
|
||||
Link: strings.TrimSpace(it.Link),
|
||||
Summary: it.Description,
|
||||
Body: it.Content,
|
||||
Author: author(it),
|
||||
Published: published(it),
|
||||
Categories: it.Categories,
|
||||
})
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func author(it *gofeed.Item) string {
|
||||
if it.Author != nil && it.Author.Name != "" {
|
||||
return it.Author.Name
|
||||
}
|
||||
if len(it.Authors) > 0 {
|
||||
return it.Authors[0].Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func published(it *gofeed.Item) time.Time {
|
||||
if it.PublishedParsed != nil {
|
||||
return it.PublishedParsed.UTC()
|
||||
}
|
||||
if it.UpdatedParsed != nil {
|
||||
return it.UpdatedParsed.UTC()
|
||||
}
|
||||
raw := strings.TrimSpace(it.Published)
|
||||
if raw == "" {
|
||||
raw = strings.TrimSpace(it.Updated)
|
||||
}
|
||||
for _, layout := range timeLayouts {
|
||||
if t, err := time.Parse(layout, raw); err == nil {
|
||||
return t.UTC()
|
||||
}
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
// ©AngelaMos | 2026
|
||||
// parse_test.go
|
||||
|
||||
package parse
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const feedsDir = "../../testdata/feeds"
|
||||
|
||||
func loadFeed(t *testing.T, name string) []Item {
|
||||
t.Helper()
|
||||
f, err := os.Open(filepath.Join(feedsDir, name))
|
||||
if err != nil {
|
||||
t.Fatalf("open fixture %s: %v", name, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = f.Close() })
|
||||
items, err := Feed(f)
|
||||
if err != nil {
|
||||
t.Fatalf("Feed(%s): %v", name, err)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func TestFeedGolden(t *testing.T) {
|
||||
cases := []struct {
|
||||
file string
|
||||
wantItems int
|
||||
fullBody bool
|
||||
}{
|
||||
{"krebs.xml", 3, true},
|
||||
{"theregister.xml", 3, true},
|
||||
{"thehackernews.xml", 3, false},
|
||||
{"bleepingcomputer.xml", 3, false},
|
||||
{"securityweek.xml", 3, false},
|
||||
{"darkreading.xml", 3, false},
|
||||
{"cisa.xml", 3, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.file, func(t *testing.T) {
|
||||
items := loadFeed(t, tc.file)
|
||||
if len(items) != tc.wantItems {
|
||||
t.Fatalf("items = %d, want %d", len(items), tc.wantItems)
|
||||
}
|
||||
for i, it := range items {
|
||||
if it.Title == "" {
|
||||
t.Errorf("item %d: empty title", i)
|
||||
}
|
||||
if it.Link == "" {
|
||||
t.Errorf("item %d: empty link", i)
|
||||
}
|
||||
if it.Published.IsZero() {
|
||||
t.Errorf("item %d (%q): published time did not parse", i, it.Title)
|
||||
}
|
||||
if tc.fullBody && it.Body == "" {
|
||||
t.Errorf("item %d (%q): expected full body, got empty", i, it.Title)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedExactPublishedTime(t *testing.T) {
|
||||
cases := []struct {
|
||||
file string
|
||||
raw string
|
||||
}{
|
||||
{"thehackernews.xml", "Sat, 04 Jul 2026 18:17:53 +0530"},
|
||||
{"krebs.xml", "Thu, 02 Jul 2026 19:27:33 +0000"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.file, func(t *testing.T) {
|
||||
want, err := time.Parse(time.RFC1123Z, tc.raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parse expected time: %v", err)
|
||||
}
|
||||
items := loadFeed(t, tc.file)
|
||||
if !items[0].Published.Equal(want) {
|
||||
t.Errorf("published = %s, want %s", items[0].Published, want.UTC())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedExactTitles(t *testing.T) {
|
||||
cases := []struct {
|
||||
file string
|
||||
title string
|
||||
}{
|
||||
{"krebs.xml", "FBI Seizes NetNut Proxy Platform, Popa Botnet"},
|
||||
{"thehackernews.xml", "U.S. Government Entity Paid Kairos $1 Million in Data-Theft Extortion Case"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.file, func(t *testing.T) {
|
||||
items := loadFeed(t, tc.file)
|
||||
if items[0].Title != tc.title {
|
||||
t.Errorf("first title = %q, want %q", items[0].Title, tc.title)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
// ©AngelaMos | 2026
|
||||
// fetch_state_test.go
|
||||
|
||||
package store
|
||||
|
||||
import "testing"
|
||||
|
||||
func seedSource(t *testing.T, s *Store) int64 {
|
||||
t.Helper()
|
||||
id, err := s.UpsertSource(SourceInput{
|
||||
Name: "krebs", Title: "Krebs", URL: "https://krebsonsecurity.com/feed/",
|
||||
Type: "rss", Weight: 1.0, Tags: []string{"news"}, Enabled: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UpsertSource: %v", err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func TestFetchStateMissing(t *testing.T) {
|
||||
s := openTemp(t)
|
||||
id := seedSource(t, s)
|
||||
fs, ok, err := s.GetFetchState(id)
|
||||
if err != nil {
|
||||
t.Fatalf("GetFetchState: %v", err)
|
||||
}
|
||||
if ok {
|
||||
t.Error("expected no fetch_state for fresh source")
|
||||
}
|
||||
if fs != (FetchState{}) {
|
||||
t.Errorf("expected zero FetchState, got %+v", fs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchStateRoundTrip(t *testing.T) {
|
||||
s := openTemp(t)
|
||||
id := seedSource(t, s)
|
||||
|
||||
want := FetchState{ETag: `"v1"`, LastModified: "Wed, 01 Jul 2026 00:00:00 GMT", LastFetched: 1700, LastStatus: 200}
|
||||
if err := s.UpsertFetchState(id, want); err != nil {
|
||||
t.Fatalf("UpsertFetchState: %v", err)
|
||||
}
|
||||
|
||||
got, ok, err := s.GetFetchState(id)
|
||||
if err != nil {
|
||||
t.Fatalf("GetFetchState: %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected fetch_state to exist")
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("round-trip = %+v, want %+v", got, want)
|
||||
}
|
||||
|
||||
updated := FetchState{ETag: `"v2"`, LastModified: "Thu, 02 Jul 2026 00:00:00 GMT", LastFetched: 1800, LastStatus: 304}
|
||||
if err := s.UpsertFetchState(id, updated); err != nil {
|
||||
t.Fatalf("second UpsertFetchState: %v", err)
|
||||
}
|
||||
got, _, _ = s.GetFetchState(id)
|
||||
if got != updated {
|
||||
t.Errorf("after update = %+v, want %+v", got, updated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInsertArticleStoresTitleHash(t *testing.T) {
|
||||
s := openTemp(t)
|
||||
id := seedSource(t, s)
|
||||
if _, err := s.InsertArticle(Article{
|
||||
SourceID: id, CanonicalURL: "https://example.com/a", ContentHash: "hash1",
|
||||
TitleHash: "thash1", Title: "A",
|
||||
}); err != nil {
|
||||
t.Fatalf("InsertArticle: %v", err)
|
||||
}
|
||||
var th string
|
||||
if err := s.DB().QueryRow(`SELECT title_hash FROM articles WHERE content_hash = ?`, "hash1").Scan(&th); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if th != "thash1" {
|
||||
t.Errorf("title_hash = %q, want thash1", th)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
-- ©AngelaMos | 2026
|
||||
-- 0002_article_title_hash.sql
|
||||
|
||||
ALTER TABLE articles ADD COLUMN title_hash TEXT NOT NULL DEFAULT '';
|
||||
|
||||
CREATE INDEX idx_articles_title_hash ON articles(title_hash);
|
||||
|
|
@ -45,6 +45,7 @@ type Article struct {
|
|||
SourceID int64
|
||||
CanonicalURL string
|
||||
ContentHash string
|
||||
TitleHash string
|
||||
Title string
|
||||
Summary string
|
||||
Body string
|
||||
|
|
@ -53,6 +54,13 @@ type Article struct {
|
|||
FetchedAt int64
|
||||
}
|
||||
|
||||
type FetchState struct {
|
||||
ETag string
|
||||
LastModified string
|
||||
LastFetched int64
|
||||
LastStatus int64
|
||||
}
|
||||
|
||||
func Open(path string) (*Store, error) {
|
||||
dsn := fmt.Sprintf("file:%s?_pragma=busy_timeout(5000)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(1)", path)
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
|
|
@ -114,9 +122,9 @@ func (s *Store) GetSourceByName(name string) (SourceRow, error) {
|
|||
func (s *Store) InsertArticle(a Article) (int64, error) {
|
||||
res, err := s.db.Exec(`
|
||||
INSERT INTO articles
|
||||
(source_id, canonical_url, content_hash, title, summary, body, author, published_at, fetched_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
a.SourceID, a.CanonicalURL, a.ContentHash, a.Title, a.Summary, a.Body,
|
||||
(source_id, canonical_url, content_hash, title_hash, title, summary, body, author, published_at, fetched_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
a.SourceID, a.CanonicalURL, a.ContentHash, a.TitleHash, a.Title, a.Summary, a.Body,
|
||||
a.Author, a.PublishedAt, a.FetchedAt,
|
||||
)
|
||||
if err != nil {
|
||||
|
|
@ -141,6 +149,36 @@ func (s *Store) CountArticles() (int, error) {
|
|||
return n, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetFetchState(sourceID int64) (FetchState, bool, error) {
|
||||
var fs FetchState
|
||||
err := s.db.QueryRow(`
|
||||
SELECT etag, last_modified, last_fetched, last_status
|
||||
FROM fetch_state WHERE source_id = ?`, sourceID,
|
||||
).Scan(&fs.ETag, &fs.LastModified, &fs.LastFetched, &fs.LastStatus)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return FetchState{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return FetchState{}, false, fmt.Errorf("get fetch_state %d: %w", sourceID, err)
|
||||
}
|
||||
return fs, true, nil
|
||||
}
|
||||
|
||||
func (s *Store) UpsertFetchState(sourceID int64, fs FetchState) error {
|
||||
_, err := s.db.Exec(`
|
||||
INSERT INTO fetch_state (source_id, etag, last_modified, last_fetched, last_status)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(source_id) DO UPDATE SET
|
||||
etag = excluded.etag, last_modified = excluded.last_modified,
|
||||
last_fetched = excluded.last_fetched, last_status = excluded.last_status`,
|
||||
sourceID, fs.ETag, fs.LastModified, fs.LastFetched, fs.LastStatus,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert fetch_state %d: %w", sourceID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func boolToInt(b bool) int {
|
||||
if b {
|
||||
return 1
|
||||
|
|
|
|||
62
PROJECTS/intermediate/security-news-scraper/testdata/feeds/bleepingcomputer.xml
vendored
Normal file
62
PROJECTS/intermediate/security-news-scraper/testdata/feeds/bleepingcomputer.xml
vendored
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<rss version="2.0"
|
||||
xmlns:content="http://purl.org/rss/1.0/modules/content/"
|
||||
xmlns:wfw="http://wellformedweb.org/CommentAPI/"
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:atom="http://www.w3.org/2005/Atom"
|
||||
>
|
||||
|
||||
<channel>
|
||||
<title>BleepingComputer</title>
|
||||
|
||||
<link>https://www.bleepingcomputer.com/</link>
|
||||
<description>BleepingComputer - All Stories</description>
|
||||
<pubDate>Sun, 05 Jul 2026 16:55:00 GMT</pubDate>
|
||||
<generator>https://www.bleepingcomputer.com/</generator>
|
||||
<language>en</language>
|
||||
<atom:link href="https://www.bleepingcomputer.com/feed/" rel="self" type="application/rss+xml" />
|
||||
|
||||
<item>
|
||||
<title>Flipper Zero firmware development continues with community help</title>
|
||||
<link>https://www.bleepingcomputer.com/news/security/flipper-zero-firmware-development-continues-with-community-help/</link>
|
||||
<pubDate>Sun, 05 Jul 2026 10:14:52 -0400</pubDate>
|
||||
<dc:creator>Bill Toulas</dc:creator>
|
||||
|
||||
|
||||
<category><![CDATA[Security]]></category>
|
||||
|
||||
|
||||
<guid>https://www.bleepingcomputer.com/news/security/flipper-zero-firmware-development-continues-with-community-help/</guid>
|
||||
<description><![CDATA[Flipper Devices says development of the Flipper Zero firmware will continue, albeit with a smaller internal team and greater reliance on community contributions. [...]]]></description>
|
||||
</item>
|
||||
<item>
|
||||
<title>JadePuffer ransomware used AI agent to automate entire attack</title>
|
||||
<link>https://www.bleepingcomputer.com/news/security/jadepuffer-ransomware-used-ai-agent-to-automate-entire-attack/</link>
|
||||
<pubDate>Sat, 04 Jul 2026 10:16:38 -0400</pubDate>
|
||||
<dc:creator>Bill Toulas</dc:creator>
|
||||
|
||||
|
||||
<category><![CDATA[Security]]></category>
|
||||
|
||||
<category><![CDATA[Artificial Intelligence]]></category>
|
||||
|
||||
|
||||
<guid>https://www.bleepingcomputer.com/news/security/jadepuffer-ransomware-used-ai-agent-to-automate-entire-attack/</guid>
|
||||
<description><![CDATA[Researchers identified what they believe is the first documented case of a ransomware operation, JadePuffer, conducted entirely by a large language model (LLM) agent. [...]]]></description>
|
||||
</item>
|
||||
<item>
|
||||
<title>NetNut proxy network disrupted, 2 million infected devices cut off</title>
|
||||
<link>https://www.bleepingcomputer.com/news/security/netnut-proxy-network-disrupted-2-million-infected-devices-cut-off/</link>
|
||||
<pubDate>Fri, 03 Jul 2026 13:50:04 -0400</pubDate>
|
||||
<dc:creator>Ionut Ilascu</dc:creator>
|
||||
|
||||
|
||||
<category><![CDATA[Security]]></category>
|
||||
|
||||
|
||||
<guid>https://www.bleepingcomputer.com/news/security/netnut-proxy-network-disrupted-2-million-infected-devices-cut-off/</guid>
|
||||
<description><![CDATA[A joint operation involving Google has disrupted NetNut, a residential proxy network that gave access to millions of compromised Android devices, including smart TVs and streaming boxes. [...]]]></description>
|
||||
</item>
|
||||
|
||||
</channel>
|
||||
</rss>
|
||||
|
|
@ -0,0 +1,598 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<rss version="2.0" xml:base="https://www.cisa.gov/" xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<channel>
|
||||
<title>All CISA Advisories</title>
|
||||
<link>https://www.cisa.gov/</link>
|
||||
<description></description>
|
||||
<language>en</language>
|
||||
|
||||
<item>
|
||||
<title>CubeSpace CW0057 Reaction Wheel</title>
|
||||
<link>https://www.cisa.gov/news-events/ics-advisories/icsa-26-183-02</link>
|
||||
<description><p><a href="https://github.com/cisagov/CSAF/blob/develop/csaf_files/OT/white/2026/icsa-26-183-02.json"><strong>View CSAF</strong></a></p>
|
||||
<h2>Summary</h2>
|
||||
<p><strong>Successful exploitation of this vulnerability could allow an attacker to upload arbitrary malicious firmware to the device.</strong></p>
|
||||
<p>The following versions of CubeSpace CW0057 Reaction Wheel are affected:</p>
|
||||
<ul>
|
||||
<li>CW0057 Reaction Wheel</li>
|
||||
</ul>
|
||||
<div class="csaf-table">
|
||||
<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap>
|
||||
<thead>
|
||||
<tr>
|
||||
<th role="columnheader" data-tablesaw-priority="persist">CVSS</th>
|
||||
<th role="columnheader">Vendor</th>
|
||||
<th role="columnheader">Equipment</th>
|
||||
<th role="columnheader">Vulnerabilities</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>v3 6.1</td>
|
||||
<td>CubeSpace</td>
|
||||
<td>CubeSpace CW0057 Reaction Wheel</td>
|
||||
<td>Improper Verification of Cryptographic Signature</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<h3>Background</h3>
|
||||
<ul>
|
||||
<li><strong>Critical Infrastructure Sectors: </strong>Communications</li>
|
||||
<li><strong>Countries/Areas Deployed: </strong>Worldwide</li>
|
||||
<li><strong>Company Headquarters Location: </strong>South Africa</li>
|
||||
</ul>
|
||||
<hr>
|
||||
<h2>Vulnerabilities</h2>
|
||||
<div class="csaf-accordion">
|
||||
<p><a class="csaf-accordion-toggle-all" href="#">Expand All +</a></p>
|
||||
<div class="csaf-accordion-item">
|
||||
<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-13743</a></h3>
|
||||
<div class="csaf-accordion-content">
|
||||
<p>CubeSpace CW0057 Reaction Wheel firmware versions prior to 5.0.20 are vulnerable to an Improper Verification of Cryptographic Signature vulnerability. This could allow an attacker with physical access to the product to upload arbitrary malicious firmware to the device without authentication.</p>
|
||||
<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-13743">View CVE Details</a></p>
|
||||
<hr>
|
||||
<h4>Affected Products</h4>
|
||||
<h5>CubeSpace CW0057 Reaction Wheel</h5>
|
||||
<div class="ics-vendor-version-status">
|
||||
<div class="ics-vendor"><strong>Vendor:</strong><br>CubeSpace</div>
|
||||
<div class="ics-version"><strong>Product Version:</strong><br>CubeSpace CW0057 Reaction Wheel: &lt;firmware_5.0.20</div>
|
||||
<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div>
|
||||
</div>
|
||||
<div class="ics-remediations">
|
||||
<h6>Remediations</h6>
|
||||
<p><strong>Vendor fix</strong><br>CubeSpace has released the following firmware versions for users to enable: Firmware version 5.0.20. Firmware version 5.0.20 introduces the capability for cryptographically verified secure boot; however, this protection is not enabled by default. Users must activate signed‑boot functionality, particularly the fully immutable mode, to achieve full security.</p>
|
||||
<p><strong>Mitigation</strong><br>CubeSpace acknowledges the finding. The CW0057 reaction wheel authenticates firmware updates with a CRC-32 integrity check, which confirms image integrity but does not verify the source of an image. Exploitation requires direct physical access to the device and is not exploitable remotely. A device affected by this method remains recoverable: the bootloader operates independently of the application firmware and can reload known-good, CubeSpace-supplied images, so an affected unit cannot be permanently disabled by this method. Starting with firmware version 5.0.20, CubeSpace offers optional cryptographic secure boot of varying security levels which customers can enable. Given the physical-access prerequisite and the availability of recovery, CubeSpace assesses the practical risk as low.</p>
|
||||
</div>
|
||||
<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/347.html">CWE-347 Improper Verification of Cryptographic Signature</a></p>
|
||||
<hr>
|
||||
<h4>Metrics</h4>
|
||||
<div class="csaf-table csaf-metrics-table">
|
||||
<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap>
|
||||
<thead>
|
||||
<tr>
|
||||
<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th>
|
||||
<th role="columnheader">Base Score</th>
|
||||
<th role="columnheader">Base Severity</th>
|
||||
<th role="columnheader">Vector String</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>3.1</td>
|
||||
<td>6.1</td>
|
||||
<td>MEDIUM</td>
|
||||
<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:P/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:H">CVSS:3.1/AV:P/AC:L/PR:N/UI:N/S:U/C:N/I:H/A:H</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>4.0</td>
|
||||
<td>3.3</td>
|
||||
<td>LOW</td>
|
||||
<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:P/AC:L/AT:N/PR:N/UI:N/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N/E:P">CVSS:4.0/AV:P/AC:L/AT:N/PR:N/UI:N/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N/E:P</a></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<hr>
|
||||
<h2>Acknowledgments</h2>
|
||||
<ul>
|
||||
<li>Anthony Rose reported this vulnerability to CISA</li>
|
||||
</ul>
|
||||
<hr>
|
||||
<h2>Legal Notice and Terms of Use</h2>
|
||||
<p>This product is provided subject to this Notification (https://www.cisa.gov/notification) and this Privacy &amp; Use policy (https://www.cisa.gov/privacy-policy).</p>
|
||||
<hr>
|
||||
<h2>Recommended Practices</h2>
|
||||
<p>CISA recommends users take defensive measures to minimize the risk of exploitation of this vulnerability.</p>
|
||||
<p>Minimize network exposure for all control system devices and/or systems, ensuring they are not accessible from the internet.</p>
|
||||
<p>Locate control system networks and remote devices behind firewalls and isolating them from business networks.</p>
|
||||
<p>When remote access is required, use more secure methods, such as Virtual Private Networks (VPNs), recognizing VPNs may have vulnerabilities and should be updated to the most current version available. Also recognize VPN is only as secure as the connected devices.</p>
|
||||
<p>CISA reminds organizations to perform proper impact analysis and risk assessment prior to deploying defensive measures.</p>
|
||||
<p>CISA also provides a section for control systems security recommended practices on the ICS webpage on cisa.gov/ics. Several CISA products detailing cyber defense best practices are available for reading and download, including Improving Industrial Control Systems Cybersecurity with Defense-in-Depth Strategies.</p>
|
||||
<p>CISA encourages organizations to implement recommended cybersecurity strategies for proactive defense of ICS assets.</p>
|
||||
<p>Additional mitigation guidance and recommended practices are publicly available on the ICS webpage at cisa.gov/ics in the technical information paper, ICS-TIP-12-146-01B--Targeted Cyber Intrusion Detection and Mitigation Strategies.</p>
|
||||
<p>Organizations observing suspected malicious activity should follow established internal procedures and report findings to CISA for tracking and correlation against other incidents.</p>
|
||||
<p>CISA also recommends users take the following measures to protect themselves from social engineering attacks:</p>
|
||||
<p>Do not click web links or open attachments in unsolicited email messages.</p>
|
||||
<p>Refer to Recognizing and Avoiding Email Scams for more information on avoiding email scams.</p>
|
||||
<p>Refer to Avoiding Social Engineering and Phishing Attacks for more information on social engineering attacks.</p>
|
||||
<p>No known public exploitation specifically targeting this vulnerability has been reported to CISA at this time. This vulnerability is not exploitable remotely.</p>
|
||||
<hr>
|
||||
<h2>Revision History</h2>
|
||||
<ul>
|
||||
<li><strong>Initial Release Date: </strong>2026-07-02</li>
|
||||
</ul>
|
||||
<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap>
|
||||
<thead>
|
||||
<tr>
|
||||
<th role="columnheader" data-tablesaw-priority="persist">Date</th>
|
||||
<th role="columnheader">Revision</th>
|
||||
<th role="columnheader">Summary</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>2026-07-02</td>
|
||||
<td>1</td>
|
||||
<td>Initial Publication</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<hr>
|
||||
<h2>Legal Notice and Terms of Use</h2>
|
||||
</description>
|
||||
<pubDate>Thu, 02 Jul 26 12:00:00 +0000</pubDate>
|
||||
<dc:creator>CISA</dc:creator>
|
||||
<guid isPermaLink="false">/node/25107</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title>Gardyn IoT Hub</title>
|
||||
<link>https://www.cisa.gov/news-events/ics-advisories/icsa-26-183-03</link>
|
||||
<description><p><a href="https://github.com/cisagov/CSAF/blob/develop/csaf_files/OT/white/2026/icsa-26-183-03.json"><strong>View CSAF</strong></a></p>
|
||||
<h2>Summary</h2>
|
||||
<p><strong>Successful exploitation of these vulnerabilities could allow unauthenticated users to access and control IoT Hub managed devices.</strong></p>
|
||||
<p>The following versions of Gardyn IoT Hub are affected:</p>
|
||||
<ul>
|
||||
<li>Home Firmware</li>
|
||||
<li>Studio Firmware</li>
|
||||
<li>Cloud API &lt;2.12.2026 (CVE-2026-13768, CVE-2026-55726, CVE-2026-54477)</li>
|
||||
</ul>
|
||||
<div class="csaf-table">
|
||||
<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap>
|
||||
<thead>
|
||||
<tr>
|
||||
<th role="columnheader" data-tablesaw-priority="persist">CVSS</th>
|
||||
<th role="columnheader">Vendor</th>
|
||||
<th role="columnheader">Equipment</th>
|
||||
<th role="columnheader">Vulnerabilities</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>v3 10</td>
|
||||
<td>Gardyn</td>
|
||||
<td>Gardyn IoT Hub</td>
|
||||
<td>Use of Hard-coded Credentials, Exposure of Sensitive System Information to an Unauthorized Control Sphere, Improper Neutralization of HTTP Headers for Scripting Syntax</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<h3>Background</h3>
|
||||
<ul>
|
||||
<li><strong>Critical Infrastructure Sectors: </strong>Food and Agriculture</li>
|
||||
<li><strong>Countries/Areas Deployed: </strong>United States</li>
|
||||
<li><strong>Company Headquarters Location: </strong>United States</li>
|
||||
</ul>
|
||||
<hr>
|
||||
<h2>Vulnerabilities</h2>
|
||||
<div class="csaf-accordion">
|
||||
<p><a class="csaf-accordion-toggle-all" href="#">Expand All +</a></p>
|
||||
<div class="csaf-accordion-item">
|
||||
<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-13768</a></h3>
|
||||
<div class="csaf-accordion-content">
|
||||
<p>Gardyn devices expose a privileged iothubowner key. Access to this key will allow a malicious user to invoke an IoTHub Registry Manager function which returns connection information for all Gardyn Home Kit and Studio devices. Access to this key also allows a malicious user to execute arbitrary commands on a specific connected device and may allow the malicious user to pivot to other devices on the user's network.</p>
|
||||
<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-13768">View CVE Details</a></p>
|
||||
<hr>
|
||||
<h4>Affected Products</h4>
|
||||
<h5>Gardyn IoT Hub</h5>
|
||||
<div class="ics-vendor-version-status">
|
||||
<div class="ics-vendor"><strong>Vendor:</strong><br>Gardyn</div>
|
||||
<div class="ics-version"><strong>Product Version:</strong><br>Gardyn Home Firmware: &lt;master.627, Gardyn Studio Firmware: &lt;master.627, Gardyn Cloud API: &lt;2.12.2026</div>
|
||||
<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div>
|
||||
</div>
|
||||
<div class="ics-remediations">
|
||||
<h6>Remediations</h6>
|
||||
<p><strong>Mitigation</strong><br>Gardyn states that IoT Hub deployed infrastructure has been updated to fix the listed vulnerabilities.</p>
|
||||
<p><strong>Mitigation</strong><br>Gardyn requests that users ensure their devices have Internet connectivity in order to automatically download needed firmware updates. Unconnected devices will automatically update when configured with a working Internet connection. Gardyn also recommends that users update their mobile application to the most recent version. The current versions of the Gardyn App and the Gardyn Home firmware can be checked in the Gardyn App.</p>
|
||||
<p><strong>Mitigation</strong><br>Further information on Gardyn security can be found here: https://mygardyn.com/security/<br><a href="https://mygardyn.com/security/">https://mygardyn.com/security/</a></p>
|
||||
<p><strong>Mitigation</strong><br>Further customer support can be obtained from Gardyn at: support@mygardyn.com<br><a href="mailto:support@mygardyn.com">mailto:support@mygardyn.com</a></p>
|
||||
</div>
|
||||
<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/798.html">CWE-798 Use of Hard-coded Credentials</a></p>
|
||||
<hr>
|
||||
<h4>Metrics</h4>
|
||||
<div class="csaf-table csaf-metrics-table">
|
||||
<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap>
|
||||
<thead>
|
||||
<tr>
|
||||
<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th>
|
||||
<th role="columnheader">Base Score</th>
|
||||
<th role="columnheader">Base Severity</th>
|
||||
<th role="columnheader">Vector String</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>3.1</td>
|
||||
<td>10</td>
|
||||
<td>CRITICAL</td>
|
||||
<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:L">CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:L</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>4.0</td>
|
||||
<td>9.5</td>
|
||||
<td>CRITICAL</td>
|
||||
<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:N/VC:H/VI:H/VA:L/SC:H/SI:H/SA:L">CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:N/VC:H/VI:H/VA:L/SC:H/SI:H/SA:L</a></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="csaf-accordion-item">
|
||||
<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-55726</a></h3>
|
||||
<div class="csaf-accordion-content">
|
||||
<p>The Azure Blob Storage container used for Gardyn device logs is publicly listable without authentication. A malicious user would be able to access any device log file available in the blob storage container.</p>
|
||||
<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-55726">View CVE Details</a></p>
|
||||
<hr>
|
||||
<h4>Affected Products</h4>
|
||||
<h5>Gardyn IoT Hub</h5>
|
||||
<div class="ics-vendor-version-status">
|
||||
<div class="ics-vendor"><strong>Vendor:</strong><br>Gardyn</div>
|
||||
<div class="ics-version"><strong>Product Version:</strong><br>Gardyn Home Firmware: &lt;master.627, Gardyn Studio Firmware: &lt;master.627, Gardyn Cloud API: &lt;2.12.2026</div>
|
||||
<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div>
|
||||
</div>
|
||||
<div class="ics-remediations">
|
||||
<h6>Remediations</h6>
|
||||
<p><strong>Mitigation</strong><br>Gardyn states that IoT Hub deployed infrastructure has been updated to fix the listed vulnerabilities.</p>
|
||||
<p><strong>Mitigation</strong><br>Gardyn requests that users ensure their devices have Internet connectivity in order to automatically download needed firmware updates. Unconnected devices will automatically update when configured with a working Internet connection. Gardyn also recommends that users update their mobile application to the most recent version. The current versions of the Gardyn App and the Gardyn Home firmware can be checked in the Gardyn App.</p>
|
||||
<p><strong>Mitigation</strong><br>Further information on Gardyn security can be found here: https://mygardyn.com/security/<br><a href="https://mygardyn.com/security/">https://mygardyn.com/security/</a></p>
|
||||
<p><strong>Mitigation</strong><br>Further customer support can be obtained from Gardyn at: support@mygardyn.com<br><a href="mailto:support@mygardyn.com">mailto:support@mygardyn.com</a></p>
|
||||
</div>
|
||||
<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/497.html">CWE-497 Exposure of Sensitive System Information to an Unauthorized Control Sphere</a></p>
|
||||
<hr>
|
||||
<h4>Metrics</h4>
|
||||
<div class="csaf-table csaf-metrics-table">
|
||||
<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap>
|
||||
<thead>
|
||||
<tr>
|
||||
<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th>
|
||||
<th role="columnheader">Base Score</th>
|
||||
<th role="columnheader">Base Severity</th>
|
||||
<th role="columnheader">Vector String</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>3.1</td>
|
||||
<td>5.3</td>
|
||||
<td>MEDIUM</td>
|
||||
<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:N/A:N">CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:N/A:N</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>4.0</td>
|
||||
<td>6.9</td>
|
||||
<td>MEDIUM</td>
|
||||
<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:L/VI:N/VA:N/SC:N/SI:N/SA:N">CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:L/VI:N/VA:N/SC:N/SI:N/SA:N</a></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="csaf-accordion-item">
|
||||
<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-54477</a></h3>
|
||||
<div class="csaf-accordion-content">
|
||||
<p>The admin panel lacks standard security headers, enabling clickjacking and cross-site scripting attacks.</p>
|
||||
<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-54477">View CVE Details</a></p>
|
||||
<hr>
|
||||
<h4>Affected Products</h4>
|
||||
<h5>Gardyn IoT Hub</h5>
|
||||
<div class="ics-vendor-version-status">
|
||||
<div class="ics-vendor"><strong>Vendor:</strong><br>Gardyn</div>
|
||||
<div class="ics-version"><strong>Product Version:</strong><br>Gardyn Home Firmware: &lt;master.627, Gardyn Studio Firmware: &lt;master.627, Gardyn Cloud API: &lt;2.12.2026</div>
|
||||
<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div>
|
||||
</div>
|
||||
<div class="ics-remediations">
|
||||
<h6>Remediations</h6>
|
||||
<p><strong>Mitigation</strong><br>Gardyn states that IoT Hub deployed infrastructure has been updated to fix the listed vulnerabilities.</p>
|
||||
<p><strong>Mitigation</strong><br>Gardyn requests that users ensure their devices have Internet connectivity in order to automatically download needed firmware updates. Unconnected devices will automatically update when configured with a working Internet connection. Gardyn also recommends that users update their mobile application to the most recent version. The current versions of the Gardyn App and the Gardyn Home firmware can be checked in the Gardyn App.</p>
|
||||
<p><strong>Mitigation</strong><br>Further information on Gardyn security can be found here: https://mygardyn.com/security/<br><a href="https://mygardyn.com/security/">https://mygardyn.com/security/</a></p>
|
||||
<p><strong>Mitigation</strong><br>Further customer support can be obtained from Gardyn at: support@mygardyn.com<br><a href="mailto:support@mygardyn.com">mailto:support@mygardyn.com</a></p>
|
||||
</div>
|
||||
<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/644.html">CWE-644 Improper Neutralization of HTTP Headers for Scripting Syntax</a></p>
|
||||
<hr>
|
||||
<h4>Metrics</h4>
|
||||
<div class="csaf-table csaf-metrics-table">
|
||||
<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap>
|
||||
<thead>
|
||||
<tr>
|
||||
<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th>
|
||||
<th role="columnheader">Base Score</th>
|
||||
<th role="columnheader">Base Severity</th>
|
||||
<th role="columnheader">Vector String</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>3.1</td>
|
||||
<td>5.4</td>
|
||||
<td>MEDIUM</td>
|
||||
<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:L/I:L/A:N">CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:L/I:L/A:N</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>4.0</td>
|
||||
<td>5.1</td>
|
||||
<td>MEDIUM</td>
|
||||
<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N">CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N</a></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<hr>
|
||||
<h2>Acknowledgments</h2>
|
||||
<ul>
|
||||
<li>Michael Groberman reported these vulnerabilities to CISA</li>
|
||||
</ul>
|
||||
<hr>
|
||||
<h2>Legal Notice and Terms of Use</h2>
|
||||
<p>This product is provided subject to this Notification (https://www.cisa.gov/notification) and this Privacy &amp; Use policy (https://www.cisa.gov/privacy-policy).</p>
|
||||
<hr>
|
||||
<h2>Recommended Practices</h2>
|
||||
<p>CISA recommends users take defensive measures to minimize the risk of exploitation of these vulnerabilities.</p>
|
||||
<p>Minimize network exposure for all control system devices and/or systems, ensuring they are not accessible from the Internet.</p>
|
||||
<p>Locate control system networks and remote devices behind firewalls and isolating them from business networks.</p>
|
||||
<p>When remote access is required, use more secure methods, such as Virtual Private Networks (VPNs), recognizing VPNs may have vulnerabilities and should be updated to the most current version available. Also recognize VPN is only as secure as the connected devices.</p>
|
||||
<p>CISA reminds organizations to perform proper impact analysis and risk assessment prior to deploying defensive measures.</p>
|
||||
<p>CISA also provides a section for control systems security recommended practices on the ICS webpage on cisa.gov/ics. Several CISA products detailing cyber defense best practices are available for reading and download, including Improving Industrial Control Systems Cybersecurity with Defense-in-Depth Strategies.</p>
|
||||
<p>CISA encourages organizations to implement recommended cybersecurity strategies for proactive defense of ICS assets.</p>
|
||||
<p>Additional mitigation guidance and recommended practices are publicly available on the ICS webpage at cisa.gov/ics in the technical information paper, ICS-TIP-12-146-01B--Targeted Cyber Intrusion Detection and Mitigation Strategies.</p>
|
||||
<p>Organizations observing suspected malicious activity should follow established internal procedures and report findings to CISA for tracking and correlation against other incidents.</p>
|
||||
<p>No known public exploitation specifically targeting these vulnerabilities has been reported to CISA at this time.</p>
|
||||
<hr>
|
||||
<h2>Revision History</h2>
|
||||
<ul>
|
||||
<li><strong>Initial Release Date: </strong>2026-07-02</li>
|
||||
</ul>
|
||||
<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap>
|
||||
<thead>
|
||||
<tr>
|
||||
<th role="columnheader" data-tablesaw-priority="persist">Date</th>
|
||||
<th role="columnheader">Revision</th>
|
||||
<th role="columnheader">Summary</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>2026-07-02</td>
|
||||
<td>1</td>
|
||||
<td>Initial Publication</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<hr>
|
||||
<h2>Legal Notice and Terms of Use</h2>
|
||||
</description>
|
||||
<pubDate>Thu, 02 Jul 26 12:00:00 +0000</pubDate>
|
||||
<dc:creator>CISA</dc:creator>
|
||||
<guid isPermaLink="false">/node/25108</guid>
|
||||
</item>
|
||||
<item>
|
||||
<title>ST Engineering iDirect iQ-Series Terminals</title>
|
||||
<link>https://www.cisa.gov/news-events/ics-advisories/icsa-26-183-01</link>
|
||||
<description><p><a href="https://github.com/cisagov/CSAF/blob/develop/csaf_files/OT/white/2026/icsa-26-183-01.json"><strong>View CSAF</strong></a></p>
|
||||
<h2>Summary</h2>
|
||||
<p><strong>Successful exploitation of these vulnerabilities could allow an attacker to gain unauthorized access to device information or cause a denial-of-service condition.</strong></p>
|
||||
<p>The following versions of ST Engineering iDirect iQ-Series Terminals are affected:</p>
|
||||
<ul>
|
||||
<li>Evolution iQ‑Series terminals &lt;=4.5.2.1 (CVE-2026-38059, CVE-2026-38057)</li>
|
||||
<li>3315‑Series terminals &lt;=4.5.2.1 (CVE-2026-38059, CVE-2026-38057)</li>
|
||||
<li>9‑Series terminals &lt;=4.5.2.1 (CVE-2026-38059, CVE-2026-38057)</li>
|
||||
</ul>
|
||||
<div class="csaf-table">
|
||||
<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap>
|
||||
<thead>
|
||||
<tr>
|
||||
<th role="columnheader" data-tablesaw-priority="persist">CVSS</th>
|
||||
<th role="columnheader">Vendor</th>
|
||||
<th role="columnheader">Equipment</th>
|
||||
<th role="columnheader">Vulnerabilities</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>v3 8.1</td>
|
||||
<td>ST Engineering iDirect</td>
|
||||
<td>ST Engineering iDirect iQ-Series Terminals</td>
|
||||
<td>Missing Authentication for Critical Function, Cross-Site Request Forgery (CSRF)</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<h3>Background</h3>
|
||||
<ul>
|
||||
<li><strong>Critical Infrastructure Sectors: </strong>Communications, Defense Industrial Base, Energy, Government Services and Facilities, Transportation Systems</li>
|
||||
<li><strong>Countries/Areas Deployed: </strong>Worldwide</li>
|
||||
<li><strong>Company Headquarters Location: </strong>United States</li>
|
||||
</ul>
|
||||
<hr>
|
||||
<h2>Vulnerabilities</h2>
|
||||
<div class="csaf-accordion">
|
||||
<p><a class="csaf-accordion-toggle-all" href="#">Expand All +</a></p>
|
||||
<div class="csaf-accordion-item">
|
||||
<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-38059</a></h3>
|
||||
<div class="csaf-accordion-content">
|
||||
<p>The iDirect iQ200 exposes the /api/identity and /api/ REST API endpoints without authentication. An unauthenticated attacker with network access can retrieve sensitive device information including the serial number, Device ID (DID), Terminal Private Key identifier (TPK), MAC address, and exact firmware version. The DID and TPK are used for satellite network authentication in the iDirect platform, potentially enabling terminal impersonation and network reconnaissance.</p>
|
||||
<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-38059">View CVE Details</a></p>
|
||||
<hr>
|
||||
<h4>Affected Products</h4>
|
||||
<h5>ST Engineering iDirect iQ-Series Terminals</h5>
|
||||
<div class="ics-vendor-version-status">
|
||||
<div class="ics-vendor"><strong>Vendor:</strong><br>ST Engineering iDirect</div>
|
||||
<div class="ics-version"><strong>Product Version:</strong><br>ST Engineering iDirect Evolution iQ‑Series terminals: &lt;=4.5.2.1, ST Engineering iDirect 3315‑Series terminals: &lt;=4.5.2.1, ST Engineering iDirect 9‑Series terminals: &lt;=4.5.2.1</div>
|
||||
<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div>
|
||||
</div>
|
||||
<div class="ics-remediations">
|
||||
<h6>Remediations</h6>
|
||||
<p><strong>Mitigation</strong><br>ST Engineering iDirect has fixed the vulnerabilities and recommend users update the software to version 4.5.2.2 or newer.</p>
|
||||
<p><strong>Mitigation</strong><br>Registered users are able to download patches from the iDirect Support Portal https://support.idirect.net/s/login.<br><a href="https://support.idirect.net/s/login">https://support.idirect.net/s/login</a></p>
|
||||
<p><strong>Mitigation</strong><br>Restrict management interfaces to trusted networks (e.g., VPN, ACLs).</p>
|
||||
<p><strong>Mitigation</strong><br>Avoid exposing administrative APIs to the public internet.</p>
|
||||
<p><strong>Mitigation</strong><br>Enforce strong authentication practices.</p>
|
||||
<p><strong>Mitigation</strong><br>Monitor for anomalous API activity and unexpected device reboots.</p>
|
||||
</div>
|
||||
<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/306.html">CWE-306 Missing Authentication for Critical Function</a></p>
|
||||
<hr>
|
||||
<h4>Metrics</h4>
|
||||
<div class="csaf-table csaf-metrics-table">
|
||||
<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap>
|
||||
<thead>
|
||||
<tr>
|
||||
<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th>
|
||||
<th role="columnheader">Base Score</th>
|
||||
<th role="columnheader">Base Severity</th>
|
||||
<th role="columnheader">Vector String</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>3.1</td>
|
||||
<td>7.5</td>
|
||||
<td>HIGH</td>
|
||||
<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N">CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>4.0</td>
|
||||
<td>8.7</td>
|
||||
<td>HIGH</td>
|
||||
<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:N/VA:N/SC:N/SI:N/SA:N">CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:N/VA:N/SC:N/SI:N/SA:N</a></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="csaf-accordion-item">
|
||||
<h3><a class="csaf-accordion-toggle" href="#">CVE-2026-38057</a></h3>
|
||||
<div class="csaf-accordion-content">
|
||||
<p>The iDirect iQ200 does not validate CSRF tokens on state-changing API endpoints after authentication. The /api/reboot endpoint accepts POST requests authenticated solely by a session cookie that lacks the SameSite attribute. A remote attacker can host a malicious web page that, when visited by an authenticated administrator, automatically submits a cross-site POST request causing an immediate device reboot and satellite link loss. Repeated attacks can sustain a denial-of-service condition.</p>
|
||||
<p><a href="https://www.cve.org/CVERecord?id=CVE-2026-38057">View CVE Details</a></p>
|
||||
<hr>
|
||||
<h4>Affected Products</h4>
|
||||
<h5>ST Engineering iDirect iQ-Series Terminals</h5>
|
||||
<div class="ics-vendor-version-status">
|
||||
<div class="ics-vendor"><strong>Vendor:</strong><br>ST Engineering iDirect</div>
|
||||
<div class="ics-version"><strong>Product Version:</strong><br>ST Engineering iDirect Evolution iQ‑Series terminals: &lt;=4.5.2.1, ST Engineering iDirect 3315‑Series terminals: &lt;=4.5.2.1, ST Engineering iDirect 9‑Series terminals: &lt;=4.5.2.1</div>
|
||||
<div class="ics-status"><strong>Product Status:</strong><br>known_affected</div>
|
||||
</div>
|
||||
<div class="ics-remediations">
|
||||
<h6>Remediations</h6>
|
||||
<p><strong>Mitigation</strong><br>ST Engineering iDirect has fixed the vulnerabilities and recommend users update the software to version 4.5.2.2 or newer.</p>
|
||||
<p><strong>Mitigation</strong><br>Registered users are able to download patches from the iDirect Support Portal https://support.idirect.net/s/login.<br><a href="https://support.idirect.net/s/login">https://support.idirect.net/s/login</a></p>
|
||||
<p><strong>Mitigation</strong><br>Restrict management interfaces to trusted networks (e.g., VPN, ACLs).</p>
|
||||
<p><strong>Mitigation</strong><br>Avoid exposing administrative APIs to the public internet.</p>
|
||||
<p><strong>Mitigation</strong><br>Enforce strong authentication practices.</p>
|
||||
<p><strong>Mitigation</strong><br>Monitor for anomalous API activity and unexpected device reboots.</p>
|
||||
</div>
|
||||
<p><strong>Relevant CWE:</strong> <a href="https://cwe.mitre.org/data/definitions/352.html">CWE-352 Cross-Site Request Forgery (CSRF)</a></p>
|
||||
<hr>
|
||||
<h4>Metrics</h4>
|
||||
<div class="csaf-table csaf-metrics-table">
|
||||
<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap>
|
||||
<thead>
|
||||
<tr>
|
||||
<th role="columnheader" data-tablesaw-priority="persist">CVSS Version</th>
|
||||
<th role="columnheader">Base Score</th>
|
||||
<th role="columnheader">Base Severity</th>
|
||||
<th role="columnheader">Vector String</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>3.1</td>
|
||||
<td>8.1</td>
|
||||
<td>HIGH</td>
|
||||
<td><a href="https://www.first.org/cvss/calculator/3.1#CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:H/A:H">CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:N/I:H/A:H</a></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>4.0</td>
|
||||
<td>7</td>
|
||||
<td>HIGH</td>
|
||||
<td><a href="https://www.first.org/cvss/calculator/4.0#CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N">CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:A/VC:N/VI:H/VA:H/SC:N/SI:N/SA:N</a></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<hr>
|
||||
<h2>Acknowledgments</h2>
|
||||
<ul>
|
||||
<li>Ahmed Alqahtani of Aramco reported these vulnerabilities to CISA</li>
|
||||
</ul>
|
||||
<hr>
|
||||
<h2>Legal Notice and Terms of Use</h2>
|
||||
<p>This product is provided subject to this Notification (https://www.cisa.gov/notification) and this Privacy &amp; Use policy (https://www.cisa.gov/privacy-policy).</p>
|
||||
<hr>
|
||||
<h2>Recommended Practices</h2>
|
||||
<p>CISA recommends users take defensive measures to minimize the risk of exploitation of these vulnerabilities.</p>
|
||||
<p>Minimize network exposure for all control system devices and/or systems, ensuring they are not accessible from the internet.</p>
|
||||
<p>Locate control system networks and remote devices behind firewalls and isolating them from business networks.</p>
|
||||
<p>When remote access is required, use more secure methods, such as Virtual Private Networks (VPNs), recognizing VPNs may have vulnerabilities and should be updated to the most current version available. Also recognize VPN is only as secure as the connected devices.</p>
|
||||
<p>CISA reminds organizations to perform proper impact analysis and risk assessment prior to deploying defensive measures.</p>
|
||||
<p>CISA also provides a section for control systems security recommended practices on the ICS webpage on cisa.gov/ics. Several CISA products detailing cyber defense best practices are available for reading and download, including Improving Industrial Control Systems Cybersecurity with Defense-in-Depth Strategies.</p>
|
||||
<p>CISA encourages organizations to implement recommended cybersecurity strategies for proactive defense of ICS assets.</p>
|
||||
<p>Additional mitigation guidance and recommended practices are publicly available on the ICS webpage at cisa.gov/ics in the technical information paper, ICS-TIP-12-146-01B--Targeted Cyber Intrusion Detection and Mitigation Strategies.</p>
|
||||
<p>Organizations observing suspected malicious activity should follow established internal procedures and report findings to CISA for tracking and correlation against other incidents.</p>
|
||||
<p>CISA also recommends users take the following measures to protect themselves from social engineering attacks:</p>
|
||||
<p>Do not click web links or open attachments in unsolicited email messages.</p>
|
||||
<p>Refer to Recognizing and Avoiding Email Scams for more information on avoiding email scams.</p>
|
||||
<p>Refer to Avoiding Social Engineering and Phishing Attacks for more information on social engineering attacks.</p>
|
||||
<p>No known public exploitation specifically targeting these vulnerabilities has been reported to CISA at this time.</p>
|
||||
<hr>
|
||||
<h2>Revision History</h2>
|
||||
<ul>
|
||||
<li><strong>Initial Release Date: </strong>2026-07-02</li>
|
||||
</ul>
|
||||
<table class="tablesaw tablesaw-stack" data-tablesaw-mode="stack" data-tablesaw-minimap>
|
||||
<thead>
|
||||
<tr>
|
||||
<th role="columnheader" data-tablesaw-priority="persist">Date</th>
|
||||
<th role="columnheader">Revision</th>
|
||||
<th role="columnheader">Summary</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>2026-07-02</td>
|
||||
<td>1</td>
|
||||
<td>Initial Publication</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<hr>
|
||||
<h2>Legal Notice and Terms of Use</h2>
|
||||
</description>
|
||||
<pubDate>Thu, 02 Jul 26 12:00:00 +0000</pubDate>
|
||||
<dc:creator>CISA</dc:creator>
|
||||
<guid isPermaLink="false">/node/25106</guid>
|
||||
</item>
|
||||
|
||||
</channel>
|
||||
</rss>
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<rss version="2.0" xml:base="https://www.darkreading.com"
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:atom="http://www.w3.org/2005/Atom"
|
||||
xmlns:media="http://search.yahoo.com/mrss/"
|
||||
xmlns:content="http://purl.org/rss/1.0/modules/content/">
|
||||
<channel>
|
||||
<title>darkreading</title>
|
||||
<link>https://www.darkreading.com</link>
|
||||
<description>Public RSS feed</description>
|
||||
<language>en</language>
|
||||
<atom:link href="https://www.darkreading.com/feeds/rss.xml" rel="self" type="application/rss+xml" />
|
||||
<item>
|
||||
<title><![CDATA[Chinese LLMs Broaden the Gap Between Attackers & Defenders]]></title>
|
||||
<link><![CDATA[https://www.darkreading.com/cyber-risk/chinese-llms-broaden-gap-between-attackers-and-defenders]]></link>
|
||||
<description><![CDATA[Two new models from Chinese firms compete with top US mainstream and frontier models. Should cyber-defenders be worried?]]></description>
|
||||
<pubDate>Fri, 03 Jul 2026 13:01:00 GMT</pubDate>
|
||||
<dc:creator>Robert Lemos</dc:creator>
|
||||
<guid isPermaLink="false"><![CDATA[https://www.darkreading.com/cyber-risk/chinese-llms-broaden-gap-between-attackers-and-defenders]]></guid>
|
||||
|
||||
<media:thumbnail url="https://eu-images.contentstack.com/v3/assets/blt6d90778a997de1cd/blt0617bd7899766864/6a46aac83cc4ed702aef33b2/castle-of-the-moors-Gabriela_Beres-shutterstock.jpg?width=720&quality=80&disable=upscale" />
|
||||
<media:content url="https://eu-images.contentstack.com/v3/assets/blt6d90778a997de1cd/blt0617bd7899766864/6a46aac83cc4ed702aef33b2/castle-of-the-moors-Gabriela_Beres-shutterstock.jpg?width=720&quality=80&disable=upscale" medium="image">
|
||||
<media:title type="html">castle-of-the-moors-Gabriela_Beres-shutterstock.jpg</media:title>
|
||||
</media:content>
|
||||
</item>
|
||||
<item>
|
||||
<title><![CDATA[Aussies Face Reduced Cybercrime Risk, as Pressure Shifts to SMBs]]></title>
|
||||
<link><![CDATA[https://www.darkreading.com/cybersecurity-analytics/aussies-face-reduced-cybercrime-risk-pressure-shifts-smbs]]></link>
|
||||
<description><![CDATA[Improved institutional safeguards and stricter regulations have pushed the burdens of protection and risk reduction on to Australian businesses.]]></description>
|
||||
<pubDate>Thu, 02 Jul 2026 23:01:00 GMT</pubDate>
|
||||
<dc:creator>Nate Nelson</dc:creator>
|
||||
<guid isPermaLink="false"><![CDATA[https://www.darkreading.com/cybersecurity-analytics/aussies-face-reduced-cybercrime-risk-pressure-shifts-smbs]]></guid>
|
||||
|
||||
<media:thumbnail url="https://eu-images.contentstack.com/v3/assets/blt6d90778a997de1cd/bltef5cf25552ad1b17/6a46744dec9caa31e05a16a8/Australia-imaginima-Getty.jpg?width=720&quality=80&disable=upscale" />
|
||||
<media:content url="https://eu-images.contentstack.com/v3/assets/blt6d90778a997de1cd/bltef5cf25552ad1b17/6a46744dec9caa31e05a16a8/Australia-imaginima-Getty.jpg?width=720&quality=80&disable=upscale" medium="image">
|
||||
<media:title type="html">Australia-imaginima-Getty.jpg</media:title>
|
||||
</media:content>
|
||||
</item>
|
||||
<item>
|
||||
<title><![CDATA[Apple Reverses Age-Old Patch Policy to Keep Up With AI]]></title>
|
||||
<link><![CDATA[https://www.darkreading.com/cybersecurity-operations/apple-patch-policy-ai]]></link>
|
||||
<description><![CDATA[Expect more compressed patching cycles from Apple going forward, as attackers leverage artificial intelligence to reduce time to exploit.]]></description>
|
||||
<pubDate>Thu, 02 Jul 2026 19:31:58 GMT</pubDate>
|
||||
<dc:creator>Nate Nelson</dc:creator>
|
||||
<guid isPermaLink="false"><![CDATA[https://www.darkreading.com/cybersecurity-operations/apple-patch-policy-ai]]></guid>
|
||||
|
||||
<media:thumbnail url="https://eu-images.contentstack.com/v3/assets/blt6d90778a997de1cd/bltb900b57271d6fc24/6a46a07ecef8a87304f980df/Apple_bandage-imtmphoto-Getty.jpg?width=720&quality=80&disable=upscale" />
|
||||
<media:content url="https://eu-images.contentstack.com/v3/assets/blt6d90778a997de1cd/bltb900b57271d6fc24/6a46a07ecef8a87304f980df/Apple_bandage-imtmphoto-Getty.jpg?width=720&quality=80&disable=upscale" medium="image">
|
||||
<media:title type="html">Apple_bandage-imtmphoto-Getty.jpg</media:title>
|
||||
</media:content>
|
||||
</item> </channel>
|
||||
</rss>
|
||||
|
|
@ -0,0 +1,245 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?><rss version="2.0"
|
||||
xmlns:content="http://purl.org/rss/1.0/modules/content/"
|
||||
xmlns:wfw="http://wellformedweb.org/CommentAPI/"
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:atom="http://www.w3.org/2005/Atom"
|
||||
xmlns:sy="http://purl.org/rss/1.0/modules/syndication/"
|
||||
xmlns:slash="http://purl.org/rss/1.0/modules/slash/"
|
||||
>
|
||||
|
||||
<channel>
|
||||
<title>Krebs on Security</title>
|
||||
<atom:link href="https://krebsonsecurity.com/feed/?utm_source=chatgpt.com" rel="self" type="application/rss+xml" />
|
||||
<link>https://krebsonsecurity.com</link>
|
||||
<description>In-depth security news and investigation</description>
|
||||
<lastBuildDate>Sat, 04 Jul 2026 16:16:19 +0000</lastBuildDate>
|
||||
<language>en-US</language>
|
||||
<sy:updatePeriod>
|
||||
hourly </sy:updatePeriod>
|
||||
<sy:updateFrequency>
|
||||
1 </sy:updateFrequency>
|
||||
<generator>https://wordpress.org/?v=6.2.2</generator>
|
||||
<item>
|
||||
<title>FBI Seizes NetNut Proxy Platform, Popa Botnet</title>
|
||||
<link>https://krebsonsecurity.com/2026/07/fbi-seizes-netnut-proxy-platform-popa-botnet/</link>
|
||||
<comments>https://krebsonsecurity.com/2026/07/fbi-seizes-netnut-proxy-platform-popa-botnet/#comments</comments>
|
||||
|
||||
<dc:creator><![CDATA[BrianKrebs]]></dc:creator>
|
||||
<pubDate>Thu, 02 Jul 2026 19:27:33 +0000</pubDate>
|
||||
<category><![CDATA[A Little Sunshine]]></category>
|
||||
<category><![CDATA[Ne'er-Do-Well News]]></category>
|
||||
<category><![CDATA[Web Fraud 2.0]]></category>
|
||||
<category><![CDATA[Alarum Technologies Ltd]]></category>
|
||||
<category><![CDATA[Benjamin Brundage]]></category>
|
||||
<category><![CDATA[fbi]]></category>
|
||||
<category><![CDATA[google]]></category>
|
||||
<category><![CDATA[Google Threat Intelligence Group]]></category>
|
||||
<category><![CDATA[Internal Revenue Service Criminal Investigation]]></category>
|
||||
<category><![CDATA[IPidea]]></category>
|
||||
<category><![CDATA[LG]]></category>
|
||||
<category><![CDATA[Lumen]]></category>
|
||||
<category><![CDATA[NetNut]]></category>
|
||||
<category><![CDATA[Popa botnet]]></category>
|
||||
<category><![CDATA[Samsung]]></category>
|
||||
<category><![CDATA[Shadowserver]]></category>
|
||||
<category><![CDATA[Synthient]]></category>
|
||||
<guid isPermaLink="false">https://krebsonsecurity.com/?p=73923</guid>
|
||||
|
||||
<description><![CDATA[The Federal Bureau of Investigation (FBI) said today it worked with industry partners to seize hundreds of domains associated with NetNut, a sprawling residential proxy service operated by the publicly-traded Israeli company Alarum Technologies [NASDAQ: ALAR]. The action comes roughly two weeks after KrebsOnSecurity published findings from multiple security firms connecting NetNut to the Popa botnet, a collection of at least two million devices that have been compromised by malicious software with little or no consent from victims.]]></description>
|
||||
<content:encoded><![CDATA[<p>The <strong>Federal Bureau of Investigation</strong> (FBI) said today it worked with industry partners to seize hundreds of domains associated with <strong>NetNut</strong>, a sprawling residential proxy service operated by the publicly-traded Israeli company <strong>Alarum Technologies </strong>[NASDAQ: ALAR]. The action comes roughly two weeks after KrebsOnSecurity published findings from multiple security firms connecting NetNut to the <strong>Popa</strong> botnet, a collection of at least two million devices that have been compromised by malicious software with little or no consent from victims.</p>
|
||||
<div id="attachment_73926" style="width: 757px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73926" decoding="async" class=" wp-image-73926" src="https://krebsonsecurity.com/wp-content/uploads/2026/07/netnutseizure.png" alt="" width="747" height="415" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/07/netnutseizure.png 1407w, https://krebsonsecurity.com/wp-content/uploads/2026/07/netnutseizure-768x427.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/07/netnutseizure-782x435.png 782w" sizes="(max-width: 747px) 100vw, 747px" /><p id="caption-attachment-73926" class="wp-caption-text">The NetNut homepage today was replaced by this seizure banner from the FBI.</p></div>
|
||||
<p>On June 19, three different security firms <a href="https://krebsonsecurity.com/2026/06/popa-botnet-linked-to-publicly-traded-israeli-firm/" target="_blank" rel="noopener">issued similar findings</a>: That NetNut is a residential proxy network which populates a botnet called Popa, and distributes software for devices commonly found in homes, such as smart TVs and streaming boxes. NetNut’s software turns those systems into always-on residential proxy nodes that are rented to others, who predominantly use them to relay abusive and intrusive Internet traffic, such as mass content scraping, advertising fraud, and account takeover activity.</p>
|
||||
<p>Earlier today, NetNut’s homepage was replaced with a seizure notice from the FBI and the <strong>Internal Revenue Service Criminal Investigation</strong> division. The seizure notice thanked <strong>Google</strong>, <strong>Lumen</strong>, <strong>Shadowserver</strong> and other industry partners for their help in dismantling hundreds of domains tied to the Popa botnet, which experts say has long been synonymous with NetNut’s residential proxy infrastructure.</p>
|
||||
<p>In a blog post published today, the <strong>Google</strong> <strong>Threat Intelligence Group</strong> (GTIG) said NetNut’s proxy network is widely resold and white-labeled by a number of third-party proxy providers, and that its services are heavily sought out by cybercriminals seeking to obfuscate the source of their malicious traffic. The GTIG said that in a single week during June 2026, they observed 316 distinct clusters of threat actors using suspected NetNut exit nodes, including cybercriminal and espionage groups.</p>
|
||||
<p>“These bad actors can use NetNut to mask their origin IP address when accessing victim environments, accessing their own infrastructure, and conducting password spray attacks,” Google’s GTIG <a href="https://cloud.google.com/blog/topics/threat-intelligence/google-continued-disruption-residential-proxy-networks" target="_blank" rel="noopener">wrote</a>. “Furthermore, when a consumer device becomes an exit node, unauthorized network traffic passes through it. This means bad actors can access other private devices on the same home network, effectively exposing them to Internet threats.”</p>
|
||||
<p>Google said it disabled Google accounts and services used by NetNut for malware command and control, and that it shared technical intelligence on NetNut’s software development kits (SDKs) and backend infrastructure with platform providers, law enforcement and research firms. The company also disabled apps known to bundle NetNut’s various SDKs.</p>
|
||||
<p><strong>Omer Weiss</strong>, legal counsel for NetNut parent Alarum Technologies, said the company was aware of the FBI seizure and cooperating with investigators.</p>
|
||||
<p>“Alarum takes this matter seriously and will fully cooperate with law enforcement to ensure any misuse of its infrastructure is thoroughly investigated and those responsible are held to account,” Weiss said in a written statement.<br />
|
||||
<span id="more-73923"></span></p>
|
||||
<p><strong>Benjamin Brundage</strong> is founder of the proxy tracking service <strong>Synthient</strong>, one of the companies that <a href="https://synthient.com/blog/popa-from-sourcing-to-distribution" target="_blank" rel="noopener">published evidence last month</a> linking the Popa botnet to NetNut and Alarum Technologies. Brundage said the domain seizures appear to have disrupted both the Popa botnet and the NetNut proxy network that rides on top of it.</p>
|
||||
<p>Brundage said NetNut’s apparent demise is likely to be a great disadvantage for the cybercrime community, which was already reeling from <a href="https://cloud.google.com/blog/topics/threat-intelligence/disrupting-largest-residential-proxy-network" target="_blank" rel="noopener">legal actions by Google</a> earlier this year that seized infrastructure for NetNut’s biggest competitor — <a href="https://krebsonsecurity.com/tag/ipidea/" target="_blank" rel="noopener">IPIDEA</a>.</p>
|
||||
<p>“I think this takedown is going to have a big impact, because NetNut gained significant popularity after the IPIDEA takedown,” he said. “Also NetNut has been incredibly common among resellers, and they were on par with IPIDEA in terms of their daily traffic, quality, size, price per gigabyte, all of it.”</p>
|
||||
<div id="attachment_73938" style="width: 759px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73938" decoding="async" loading="lazy" class=" wp-image-73938" src="https://krebsonsecurity.com/wp-content/uploads/2026/07/netnut-popa-blacklotus.png" alt="" width="749" height="374" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/07/netnut-popa-blacklotus.png 1435w, https://krebsonsecurity.com/wp-content/uploads/2026/07/netnut-popa-blacklotus-768x384.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/07/netnut-popa-blacklotus-782x391.png 782w" sizes="(max-width: 749px) 100vw, 749px" /><p id="caption-attachment-73938" class="wp-caption-text">NetNut’s infrastructure, in a nutshell. Image: Black Lotus Labs, Lumen.</p></div>
|
||||
<p>The NetNut and Popa botnet takedown may have another added benefit, Brundage said: Lessening the impact of large distributed denial-of-service botnets that have been built on the backs of poorly configured residential proxy services. In January, Synthient <a href="https://krebsonsecurity.com/2026/01/the-kimwolf-botnet-is-stalking-your-local-network/" target="_blank" rel="noopener">revealed</a> how cybercriminals had built the world’s largest DDoS botnet (Kimwolf) by tunneling through IPIDEA proxy connections into the local networks of TV box owners, and infecting other Android-based devices behind the victim’s firewall.</p>
|
||||
<p>While many of the bigger proxy providers took steps to block this activity, resellers of the major proxy networks have been far slower to respond to the threat, Brundage said.</p>
|
||||
<p>“In terms of all these TV box devices getting compromised from the proxy network, it will have an impact on the DDoS botnets out there,” he said.</p>
|
||||
<p>For its part, Google reckons today’s actions have caused “significant degradation to NetNut’s proxy network and its business operations, reducing the available pool of devices for the proxy operator by millions.” But the company warns that proxy networks can rebuild themselves by effectively reselling other proxy services, as IPIDEA has done over the past few months.</p>
|
||||
<p>“Google has high confidence that many popular residential proxy brands are in fact whitelabeling the NetNut botnet,” the GTIG report concludes. “While we expect this disruption to have a larger ripple effect across the residential proxy ecosystem, observations after the disruption of IPIDEA proved that individual networks can appear resilient. What we have observed is that when faced with the degradation of their own botnet, proxy operators begin buying capacity from their competitors, effectively becoming a reseller. We recognize that creating a lasting disruption in this fluid ecosystem means we must scale our efforts to target the infrastructure of several interconnected providers.”</p>
|
||||
<p>As KrebsOnSecurity has warned repeatedly, most of the no-name TV streaming boxes for sale on the major e-commerce websites either come <a href="https://krebsonsecurity.com/2025/11/is-your-android-tv-streaming-box-part-of-a-botnet/" target="_blank" rel="noopener">pre-installed with residential proxy software</a>, or require the installation of proxy SDKs in order to use the device for its stated purpose (streaming pirated movies, sporting events and TV shows). Google’s advice here is sound: When it comes to TV boxes, stick to name brands from reputable manufacturers, and then be sparing and judicious with any apps you choose to install.</p>
|
||||
<p>The <a href="https://krebsonsecurity.com/2025/11/is-your-android-tv-streaming-box-part-of-a-botnet/" target="_blank" rel="noopener">sketchy TV boxes</a> that are being commandeered by the Popa botnet and other threats all come with or require the user to install unofficial Android operating systems that do not operate within the confines of Google’s Official Play Protect store. Google says consumers can confirm whether or not a device is built with the official Android TV OS and Play Protect certification by following <a href="https://support.google.com/googleplay/answer/7165974" target="_blank" rel="noopener">these instructions</a>.</p>
|
||||
<p>Even people without TV streaming boxes can find their smart TVs enrolled in residential proxy networks, just by installing one of thousands of apps available for download on <strong>Samsung</strong> and <strong>LG</strong> smart TVs. In <a href="https://spur.us/blog/smart-tv-apps-residential-proxy-sdks" target="_blank" rel="noopener">a report</a> released last month, the proxy tracking company <strong>Spur</strong> found 42 percent of apps available for download via the webOS operating system on LG smart TVs include SDKs that turn one’s television into an always-on residential proxy node. More than a quarter of the apps made for Samsung’s <strong>Tizen</strong> operating system had similar residential proxy components, Spur found.</p>
|
||||
<div id="attachment_73849" style="width: 758px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73849" decoding="async" loading="lazy" class=" wp-image-73849" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy.png" alt="" width="748" height="305" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy.png 1272w, https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy-768x313.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy-782x318.png 782w" sizes="(max-width: 748px) 100vw, 748px" /><p id="caption-attachment-73849" class="wp-caption-text">Image: Spur.us.</p></div>
|
||||
<p><strong>Update, 4:24 p.m. ET:</strong> Included a statement shared post-publication from an attorney representing NetNut parent Alarum Technologies.</p>
|
||||
]]></content:encoded>
|
||||
|
||||
<wfw:commentRss>https://krebsonsecurity.com/2026/07/fbi-seizes-netnut-proxy-platform-popa-botnet/feed/</wfw:commentRss>
|
||||
<slash:comments>17</slash:comments>
|
||||
|
||||
|
||||
</item>
|
||||
<item>
|
||||
<title>Scattered Spider Hackers Plead Guilty on Day 1 of Trial</title>
|
||||
<link>https://krebsonsecurity.com/2026/06/scattered-spider-hackers-plead-guilty-on-day-1-of-trial/</link>
|
||||
<comments>https://krebsonsecurity.com/2026/06/scattered-spider-hackers-plead-guilty-on-day-1-of-trial/#comments</comments>
|
||||
|
||||
<dc:creator><![CDATA[BrianKrebs]]></dc:creator>
|
||||
<pubDate>Tue, 23 Jun 2026 16:12:49 +0000</pubDate>
|
||||
<category><![CDATA[Ne'er-Do-Well News]]></category>
|
||||
<category><![CDATA[Ransomware]]></category>
|
||||
<category><![CDATA[BBC]]></category>
|
||||
<category><![CDATA[Everlynn]]></category>
|
||||
<category><![CDATA[Owen Flowers]]></category>
|
||||
<category><![CDATA[Scattered Spider]]></category>
|
||||
<category><![CDATA[Thalha Jubair]]></category>
|
||||
<category><![CDATA[Transport for London]]></category>
|
||||
<category><![CDATA[Tyler "Tylerb" Buchanan]]></category>
|
||||
<guid isPermaLink="false">https://krebsonsecurity.com/?p=73876</guid>
|
||||
|
||||
<description><![CDATA[Two men pleaded guilty in the United Kingdom this week to criminal charges stemming from an August 2024 cyberattack that crippled Transport for London, the entity responsible for the public transport network in the Greater London area. The duo were key members of a prolific cybercrime group known as Scattered Spider, and their guilty pleas came on the first day of what was expected to be a six-week trial.]]></description>
|
||||
<content:encoded><![CDATA[<p>Two men pleaded guilty in the United Kingdom this week to criminal charges stemming from an August 2024 cyberattack that crippled <strong>Transport for London</strong>, the entity responsible for the public transport network in the Greater London area. The duo were key members of a prolific cybercrime group known as <strong>Scattered Spider</strong>, and their guilty pleas came on the first day of what was expected to be a six-week trial.</p>
|
||||
<div id="attachment_73881" style="width: 760px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73881" decoding="async" loading="lazy" class=" wp-image-73881" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/flowers-jubair-nca.png" alt="" width="750" height="421" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/06/flowers-jubair-nca.png 926w, https://krebsonsecurity.com/wp-content/uploads/2026/06/flowers-jubair-nca-768x431.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/06/flowers-jubair-nca-782x439.png 782w" sizes="(max-width: 750px) 100vw, 750px" /><p id="caption-attachment-73881" class="wp-caption-text">Owen Flowers (left) 18, and Thalha Jubair, 20. Image: UK National Crime Agency (NCA).</p></div>
|
||||
<p><strong>Thalha Jubair</strong>, 20, of East London and 18-year-old <strong>Owen Flowers</strong> of Walsall admitted conspiring to commit unauthorized acts against Transport for London computer systems and causing risk of serious damage to human welfare. According to <a href="https://www.bbc.com/news/articles/czx5yp9qy0do" target="_blank" rel="noopener">a report</a> from the BBC, Flowers alone admitted to being part of a conspiracy to hack into U.S. based healthcare providers SSM Health Care Corporation and Sutter Health in September 2024.</p>
|
||||
<p>Jubair is also wanted by U.S. law enforcement agencies. In September 2025, prosecutors in New Jersey unsealed <a href="https://www.justice.gov/opa/pr/united-kingdom-national-charged-connection-multiple-cyber-attacks-including-critical" target="_blank" rel="noopener">an indictment</a> alleging Jubair and other Scattered Spider members committed computer fraud, wire fraud, and money laundering in relation to 120 computer network intrusions involving 47 U.S. entities between May 2022 and September 2025, and that the group’s victims paid at least $115 million in ransom payments.</p>
|
||||
<p>In July 2025, KrebsOnSecurity <a href="https://krebsonsecurity.com/2025/07/uk-charges-four-in-scattered-spider-ransom-group/" target="_blank" rel="noopener">reported</a> that Flowers and Jubair were arrested in the United Kingdom in connection with Scattered Spider <a href="https://www.thetimes.com/uk/technology-uk/article/ransoms-hackers-cyber-crime-t5kjldwwm" target="_blank" rel="noopener">ransom attacks</a> against the retailers <strong>Marks & Spencer</strong> and <strong>Harrods</strong>, and the British food retailer <strong>Co-op Group</strong>. Multiple sources familiar with those investigations said Flowers was the Scattered Spider member who anonymously <a href="https://krebsonsecurity.com/2024/09/the-dark-nexus-between-harm-groups-and-the-com/" target="_blank" rel="noopener">gave interviews to the media</a> in the days after the group’s September 2023 ransomware attacks disrupted operations at Las Vegas casinos operated by <strong>MGM Resorts</strong> and <strong>Caesars Entertainment</strong>.</p>
|
||||
<p>According to prosecutors, Jubair co-ran a bustling Telegram channel called <strong>Star Chat</strong>, the home of a <a href="https://krebsonsecurity.com/?s=SIM-swapping" target="_blank" rel="noopener">SIM-swapping</a> group that used voice- and SMS-based phishing attacks to steal credentials from employees at the major wireless providers in the U.S. and U.K. The group would then use that access to sell a service that could redirect a target’s phone number to a device the attackers controlled and intercept the victim’s calls and text messages (including one-time codes for multi-factor authentication).</p>
|
||||
<div id="attachment_72238" style="width: 819px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-72238" decoding="async" loading="lazy" class="size-full wp-image-72238" src="https://krebsonsecurity.com/wp-content/uploads/2025/09/rocketace-tmobile.png" alt="" width="809" height="915" srcset="https://krebsonsecurity.com/wp-content/uploads/2025/09/rocketace-tmobile.png 809w, https://krebsonsecurity.com/wp-content/uploads/2025/09/rocketace-tmobile-768x869.png 768w, https://krebsonsecurity.com/wp-content/uploads/2025/09/rocketace-tmobile-782x884.png 782w" sizes="(max-width: 809px) 100vw, 809px" /><p id="caption-attachment-72238" class="wp-caption-text">A receipt from Star Fraud Chat’s SIM-swapping service targeting a T-Mobile customer after the group gained access to internal T-Mobile employee tools. “Rocket Ace” was one of Jubair’s hacker handles, according to U.S. prosecutors.</p></div>
|
||||
<p>New Jersey prosecutors also allege Jubair also was involved in a <a href="https://krebsonsecurity.com/2022/08/how-1-time-passcodes-became-a-corporate-liability/" target="_blank" rel="noopener">mass SMS phishing campaign during the summer of 2022</a> that stole single sign-on credentials from employees at hundreds of companies. That weeks-long SMS phishing campaign led to intrusions and data thefts at more than 130 organizations, including <strong>LastPass</strong>, <strong>DoorDash</strong>, <strong>Mailchimp</strong>, <strong>Plex</strong> and <strong>Signal</strong>.<span id="more-73876"></span></p>
|
||||
<p>KrebsOnSecurity reported last year that one of Jubair’s alter egos at age 15 was “<strong>Everlynn</strong>,” a hacker who sold <a href="https://krebsonsecurity.com/?s=fake+edr" target="_blank" rel="noopener">fraudulent “emergency data requests”</a> that used compromised police and government email addresses to demand subscriber data (e.g. username, IP/email address) from major tech companies, claiming the requests concerned urgent matters of life and death and could not wait for a court order.</p>
|
||||
<p>In April 2026, 24-year-old British national and Scattered Spider member <strong>Tyler “Tylerb” Buchanan</strong> <a href="https://krebsonsecurity.com/2026/04/scattered-spider-member-tylerb-pleads-guilty/" target="_blank" rel="noopener">pleaded guilty</a> to wire fraud conspiracy and aggravated identity theft for participating in the group’s SMS phishing spree in the summer of 2022. The government said Buchanan, Jubair and others used the credentials harvested in that phishing campaign to steal at least $8 million in cryptocurrency from victims throughout the United States. Buchanan is currently scheduled to be sentenced on October 2.</p>
|
||||
<p>In August 2025, 20-year-old Scattered Spider member from Florida named <strong>Noah Michael Urban</strong> was <a href="https://krebsonsecurity.com/2025/08/sim-swapper-scattered-spider-hacker-gets-10-years/" target="_blank" rel="noopener">sentenced to 10 years in federal prison</a> and ordered to pay $13 million in restitution, after pleading guilty to charges of wire fraud and conspiracy.</p>
|
||||
<p>The U.S. Department of Justice says three alleged Scattered Spider defendants indicted along with Buchanan still face charges, including <strong>Ahmed Hossam Eldin Elbadawy</strong>, 24, a.k.a. “AD,” of College Station, Texas; <strong>Evans Onyeaka Osiebo</strong>, 21, of Dallas, Texas; and <strong>Joel Martin Evans</strong>, 26, a.k.a. “joeleoli,” of Jacksonville, North Carolina.</p>
|
||||
<p>Flowers and Jubair are slated to be sentenced in a London court on July 15, 2026.</p>
|
||||
]]></content:encoded>
|
||||
|
||||
<wfw:commentRss>https://krebsonsecurity.com/2026/06/scattered-spider-hackers-plead-guilty-on-day-1-of-trial/feed/</wfw:commentRss>
|
||||
<slash:comments>39</slash:comments>
|
||||
|
||||
|
||||
</item>
|
||||
<item>
|
||||
<title>‘Popa’ Botnet Linked to Publicly-Traded Israeli Firm</title>
|
||||
<link>https://krebsonsecurity.com/2026/06/popa-botnet-linked-to-publicly-traded-israeli-firm/</link>
|
||||
<comments>https://krebsonsecurity.com/2026/06/popa-botnet-linked-to-publicly-traded-israeli-firm/#comments</comments>
|
||||
|
||||
<dc:creator><![CDATA[BrianKrebs]]></dc:creator>
|
||||
<pubDate>Thu, 18 Jun 2026 17:37:58 +0000</pubDate>
|
||||
<category><![CDATA[A Little Sunshine]]></category>
|
||||
<category><![CDATA[Latest Warnings]]></category>
|
||||
<category><![CDATA[The Coming Storm]]></category>
|
||||
<category><![CDATA[Alarum Technologies Ltd]]></category>
|
||||
<category><![CDATA[BadBox 2.0]]></category>
|
||||
<category><![CDATA[Black Lotus Labs]]></category>
|
||||
<category><![CDATA[Brendan O'Connell]]></category>
|
||||
<category><![CDATA[Chris Formosa]]></category>
|
||||
<category><![CDATA[Confederation of Open Access Repositories]]></category>
|
||||
<category><![CDATA[CRICFy]]></category>
|
||||
<category><![CDATA[CyberFlix]]></category>
|
||||
<category><![CDATA[David Brunsdon]]></category>
|
||||
<category><![CDATA[Directory of Open Access Journals]]></category>
|
||||
<category><![CDATA[DooFlix]]></category>
|
||||
<category><![CDATA[Flixoid]]></category>
|
||||
<category><![CDATA[Include Security]]></category>
|
||||
<category><![CDATA[Jérôme Meyer]]></category>
|
||||
<category><![CDATA[LG]]></category>
|
||||
<category><![CDATA[Lumen Technologies]]></category>
|
||||
<category><![CDATA[NetNut]]></category>
|
||||
<category><![CDATA[Nick Sundvall]]></category>
|
||||
<category><![CDATA[Ninajtech]]></category>
|
||||
<category><![CDATA[Nokia Deepfield]]></category>
|
||||
<category><![CDATA[OceanStreams]]></category>
|
||||
<category><![CDATA[Popa botnet]]></category>
|
||||
<category><![CDATA[Qurium]]></category>
|
||||
<category><![CDATA[Rapid Streamz]]></category>
|
||||
<category><![CDATA[residential proxies]]></category>
|
||||
<category><![CDATA[RoboVPN]]></category>
|
||||
<category><![CDATA[RTS Tv]]></category>
|
||||
<category><![CDATA[Samsung]]></category>
|
||||
<category><![CDATA[Sean Simmons]]></category>
|
||||
<category><![CDATA[Sprozfy]]></category>
|
||||
<category><![CDATA[Spur]]></category>
|
||||
<category><![CDATA[Synthient]]></category>
|
||||
<category><![CDATA[TvMob]]></category>
|
||||
<category><![CDATA[Vo1d botnet]]></category>
|
||||
<guid isPermaLink="false">https://krebsonsecurity.com/?p=73832</guid>
|
||||
|
||||
<description><![CDATA[For the past four years, a sprawling Android-based botnet called Popa has forced millions of consumer TV boxes to relay Internet traffic linked to advertising fraud, account takeovers, and mass data-scraping efforts. This week, researchers from multiple security firms concluded that the Popa botnet is linked to NetNut, a "residential proxy" provider operated by the publicly-traded Israeli firm Alarum Technologies Ltd [NASDAQ: ALAR].]]></description>
|
||||
<content:encoded><![CDATA[<p>For the past four years, a sprawling Android-based botnet called <strong>Popa</strong> has forced millions of consumer TV boxes to relay Internet traffic linked to advertising fraud, account takeovers, and mass data-scraping efforts. This week, researchers from multiple security firms concluded that the Popa botnet is linked to <strong>NetNut</strong>, a “residential proxy” provider operated by the publicly-traded Israeli firm <strong>Alarum Technologies Ltd </strong>[NASDAQ: ALAR].</p>
|
||||
<div id="attachment_73857" style="width: 759px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73857" decoding="async" loading="lazy" class=" wp-image-73857" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/tvboxes-proxy.png" alt="Malicious streaming devices sold online that enroll the user's home Internet address in a residential proxy service. Image: Synthient. Pictured are 8 different TV boxes, including the X96 Mini Box, stick, and other no-name brands." width="749" height="311" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/06/tvboxes-proxy.png 990w, https://krebsonsecurity.com/wp-content/uploads/2026/06/tvboxes-proxy-768x319.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/06/tvboxes-proxy-782x325.png 782w" sizes="(max-width: 749px) 100vw, 749px" /><p id="caption-attachment-73857" class="wp-caption-text">Malicious streaming devices sold online that enroll the user’s home Internet address in a residential proxy service. Image: HUMAN Security.</p></div>
|
||||
<p>Popa is a massive botnet, but by all accounts it is unlike traditional botnets that enlist compromised systems in destructive activities, such as coordinating huge distributed denial-of-service attacks. Rather, Popa appears designed with a singular purpose: Implementing a persistent communications layer capable of registering a device, maintaining long-lived encrypted connections, and opening communication tunnels on demand.</p>
|
||||
<p>Experts say Popa is a plugin component associated with the <strong>Vo1d</strong> botnet, a large-scale malware campaign targeting unofficial Android-based TV boxes. These devices, which are marketed under thousands of brand names and model numbers and broadly available for purchase at top e-commerce destinations, all advertise the ability to stream hundreds of subscription video services for an up front one-time fee.</p>
|
||||
<p>But as the FBI and security industry experts have warned repeatedly, these streaming boxes typically <a href="https://krebsonsecurity.com/2025/11/is-your-android-tv-streaming-box-part-of-a-botnet/" target="_blank" rel="noopener">bundle or come pre-installed with software</a> that turns the user’s TV into a “<a href="https://synthient.com/blog/who-are-the-victims-of-residential-proxies" target="_blank" rel="noopener">residential proxy</a>” — allowing anyone to route their Internet traffic through that device for as long as it remains plugged into a wall socket and connected to a local network. More concerning, some of these proxy networks do little to stop malicious customers from communicating with and even compromising systems on the local network of the unsuspecting device owner.</p>
|
||||
<p>The first clues about Popa’s origins came in <a href="https://blog.xlab.qianxin.com/long-live-the-vo1d_botnet/" target="_blank" rel="noopener">a 2025 report</a> from the Chinese security company <strong>XLAB</strong>, which flagged at least nine domain names that were used to register and direct the activities of compromised devices. In <a href="https://www.qurium.org/forensics/finding-popa/" target="_blank" rel="noopener">a report</a> released today, the security firm <strong>Qurium</strong> described how it stumbled on some of those same domains while investigating a series of disruptive and expensive data scraping events targeting the company’s hosted organizations in May 2026, in which the scraping activity was scattered evenly across more than 1.4 million Internet addresses.</p>
|
||||
<p>Qurium said it found several dozen domains used to control Popa that were all hosted in lockstep across multiple Internet addresses over time, including <strong>gmslb[.]net</strong>, safernetwork[.]io, tera-home[.]com, and <strong>ninjatech[.]io</strong>. Digging deeper, Qurium discovered gmslb[.]net was referenced in dozens of pirated or modded video content streaming apps, such as <strong>CRICFy</strong>, <strong>DooFlix</strong>, <strong>Sprozfy</strong>, <strong>RTS Tv</strong>, <strong>Flixoid</strong>, <strong>CyberFlix</strong>, <strong>Rapid Streamz</strong>, <strong>TvMob</strong> and <strong>HD/OceanStreams</strong>.</p>
|
||||
<p>Qurium’s report notes that most of the domains long used to control the Popa botnet were <a href="https://blog.google/innovation-and-ai/technology/safety-security/google-taking-legal-action-against-the-badbox-20-botnet/" target="_blank" rel="noopener">seized or dismantled in July 2025</a>, after <strong>Google</strong>, <strong>HUMAN Security</strong> and <strong>Trend Micro</strong> teamed up to disrupt <strong>Badbox 2.0</strong>, a botnet that is closely associated with Vo1d. Qurium said that immediately after that disruption, several dozen new domains were registered to serve as controllers for the Popa botnet, but that one of those control domains was not new: <strong>ninjatech[.]io</strong>.</p>
|
||||
<p>Ninjatech is a company founded by <strong>Moishi Kramer</strong>, whose <a href="https://www.linkedin.com/in/moishikramer/" target="_blank" rel="noopener">LinkedIn profile</a> says he is vice president of research and development at NetNut. That resume credits Kramer for helping NetNut to build from the “ground up,” “designing the architecture,” and “scaling the NetNut” before the company was acquired by Alarum Technologies. A self-created listing at the job board <strong>F6S</strong> <a href="https://www.f6s.com/company/ninjatech.io" target="_blank" rel="noopener">references Kramer</a> as the sole owner of the Ninjatech domain (a screen capture of it is pictured below).</p>
|
||||
<div id="attachment_73842" style="width: 758px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73842" decoding="async" loading="lazy" class=" wp-image-73842" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/f6s-kramer-1.png" alt="" width="748" height="589" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/06/f6s-kramer-1.png 1167w, https://krebsonsecurity.com/wp-content/uploads/2026/06/f6s-kramer-1-768x605.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/06/f6s-kramer-1-782x616.png 782w" sizes="(max-width: 748px) 100vw, 748px" /><p id="caption-attachment-73842" class="wp-caption-text">Image: F6S.com.</p></div>
|
||||
<p>Responding via email, Mr. Kramer said Ninjatech ceased operations approximately five years ago, when the company sold a software development kit (SDK) called Popa that was designed to use a small portion of a device’s bandwidth and to run only after the host application obtained user consent.</p>
|
||||
<p>“That code was sold and licensed to third parties including resellers years ago,” Kramer said. “Once software is distributed that way, the original developer has no control over how others later modify, rebrand, or deploy it.”</p>
|
||||
<p>Kramer said neither he nor NetNut builds, operates or maintains the infrastructure being described as Popa, nor does he control the Ninjatech domain.</p>
|
||||
<p>“I didn’t register the June 2025 domains you mention, and I don’t know who did,” he continued. “I have no control over, or visibility into, that infrastructure. I can only tell you it isn’t operated by me or by NetNut.”</p>
|
||||
<p>But in <a href="https://synthient.com/blog/popa-from-sourcing-to-distribution" target="_blank" rel="noopener">a separate Popa research report</a> released today, the proxy-tracking company <strong>Synthient</strong> said a recent analysis of the Popa SDK revealed outbound traffic clearly associated with NetNut.</p>
|
||||
<p>“The research team assesses with high confidence that devices running Popa forward traffic from Netnut clients,” Synthient wrote. “This proves without a shadow of a doubt that Popa actively continues to be used by NetNut as part of their proxy pool.”</p>
|
||||
<div id="attachment_73854" style="width: 758px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73854" decoding="async" loading="lazy" class=" wp-image-73854" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/synthient-flixview.png" alt="" width="748" height="607" /><p id="caption-attachment-73854" class="wp-caption-text">Synthient’s platform receiving outbound traffic from Popa. Image: Synthient.com.</p></div>
|
||||
<p>Alarum Technologies, NetNut’s Tel Aviv-based parent company, said the reports by Synthient and Qurium contained “demonstrably inaccurate assertions and flawed deductions rather than verified facts.” Alarum shared a statement saying they reject the basic characterization of the SDKs and technologies discussed in the reports as a “botnet.”</p>
|
||||
<p>“The SDKs at issue are designed to facilitate bandwidth-sharing functionality and do not transform user devices into malware-controlled systems or otherwise compromise the devices on which they operate,” the statement reads. “Netnut operates a commercial proxy network and maintains policies, procedures, and technological measures designed to promote lawful and responsible use of its services.”</p>
|
||||
<p>Alarum said NetNut places “significant emphasis on appropriate notice and consent mechanisms, conducts customer due diligence, monitors for potential misuse, and takes steps intended to detect and mitigate suspicious or unauthorized activity.”</p>
|
||||
<p>“This method of operation is supported both by internal procedures and policies, including performing KYC checks and additional due diligence of NetNut’s customers, as well as employing various technological measures, designed to assist in identifying and addressing suspected misuse of the network,” their statement continued.</p>
|
||||
<p>However, in a report released on June 8, the proxy tracking service <strong>Spur</strong> asserted that NetNut does not require corporate verification or meaningful “know your customer” procedures before allowing customers to purchase proxy access.</p>
|
||||
<p>“An individual can sign up, pay, and route traffic through partner address space, including space belonging to institutions whose users never opted in,” Spur <a href="https://spur.us/blog/how-proxy-providers-co-opt-entire-networks" target="_blank" rel="noopener">wrote</a>. “The ‘verified corporations only’ claim is simply marketing for bandwidth sellers, not an access control on who actually uses the proxies.”</p>
|
||||
<p>“Nor is NetNut the only front door,” Spur continued. “A number of downstream white labelers and resellers repackage the same ISP proxy pool under their own brands. These outlets typically perform no KYC at all, less scrutiny than NetNut itself, who at the very least might assign an account manager to potential users. Anyone who knows where to look can buy access through a reseller with nothing more than a burner email address and $5 in crypto.”</p>
|
||||
<p>Synthient found that although the most recent builds of Popa (as of three months ago) have added the ability to ask the user for consent before installing proxy components, not all variants or previous versions of Popa contain this functionality.</p>
|
||||
<p>“Of the over 20 genuine Popa publishers analyzed, none of them were observed asking for user consent,” Sythient wrote.<span id="more-73832"></span></p>
|
||||
<h2>THE PREVALENCE OF POPA</h2>
|
||||
<p><strong>Chris Formosa</strong> is senior lead information security engineer for <strong>Black Lotus Labs</strong>, a division of the Internet backbone carrier <strong>Lumen Technologies</strong>.</p>
|
||||
<p>“What especially makes Popa dangerous is just how widely used NetNut is for reselling and sharing,” Formosa said, explaining that many other proxy services simply resell NetNut proxies rather than building out their own far-flung proxy networks. “So these Popa IPs appear in tons of different services all over the ecosystem, which makes it one of the most problematic and dangerous proxy botnets on the market currently.”</p>
|
||||
<p>Formosa said the Popa botnet averages between 1.5 million to 2.5 million distinct IP addresses each day, relying on between 250 and 300 Internet addresses that are used to direct its activities.</p>
|
||||
<p>“That’s why Popa is so dangerous,” Formosa said. “It may not be the largest botnet we have seen, but it is spread all over the industry, making its power very amplified.”</p>
|
||||
<p>Formosa said while that makes Popa one of the larger botnets out there today, its numbers pale in comparison to those previously boasted by <a href="https://krebsonsecurity.com/tag/ipidea/" target="_blank" rel="noopener">IPIDEA</a>, a China-based proxy provider that until recently operated a daily pool of nearly 10 million devices that they resold as proxies to anyone. In January 2026, Synthient <a href="https://krebsonsecurity.com/2026/01/the-kimwolf-botnet-is-stalking-your-local-network/" target="_blank" rel="noopener">published research</a> showing that multiple new large DDoS botnets had grown rapidly by tunneling through IPIDEA proxies into the local networks of unsuspecting TV box owners and infecting other Android-based devices behind the user’s firewall.</p>
|
||||
<p>IPIDEA is based largely on SDKs used to view pirated streaming content on a vast number of TV box devices, but the service’s numbers have dwindled since January, when Google and industry partners <a href="https://cloud.google.com/blog/topics/threat-intelligence/disrupting-largest-residential-proxy-network" target="_blank" rel="noopener">took legal action</a> to seize domain names that IPIDEA used to control devices and proxy traffic through them.</p>
|
||||
<p><strong>Jérôme Meyer</strong>, a security researcher at <strong>Nokia Deepfield</strong>, said the total population of devices participating in the Popa botnet may be far higher than Lumen’s estimates. Meyer told KrebsOnSecurity that Nokia is monitoring 26 of at least 359 known relay nodes for the botnet, and estimates that each relay node handles between 35,000 and 60,000 clients simultaneously.</p>
|
||||
<p>“On the relay node subset I am looking at (26 of them), 750,000 unique sources in 24 hours,” Meyer wrote in response to questions.</p>
|
||||
<p>Nokia Deepfield <a href="https://github.com/deepfield/public-research/blob/main/reports/2026-06-18-robovpn-neunative.md" target="_blank" rel="noopener">released its own report today</a> on <strong>RoboVPN</strong>, a VPN app tied to the Vo1d botnet’s Popa plugin that Qurium attributes to NetNut/Alarum Technologies.</p>
|
||||
<h2>THE SYMBIOSIS OF PROXIES AND DATA SCRAPING</h2>
|
||||
<p>Experts say many of the world’s largest proxy providers have updated their public-facing branding to highlight their utility for training AI platforms, implying it is a primary use case for their residential proxies. That’s because AI services tend to rely on constantly mass-scraping the Internet for new text, images and video content that can be used to train large language models (LLMs).</p>
|
||||
<div id="attachment_73850" style="width: 760px" class="wp-caption aligncenter"><a href="https://krebsonsecurity.com/wp-content/uploads/2026/06/proxy-scraping-ai.png" target="_blank" rel="noopener"><img aria-describedby="caption-attachment-73850" decoding="async" loading="lazy" class="wp-image-73850" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/proxy-scraping-ai.png" alt="" width="750" height="375" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/06/proxy-scraping-ai.png 1424w, https://krebsonsecurity.com/wp-content/uploads/2026/06/proxy-scraping-ai-768x384.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/06/proxy-scraping-ai-782x391.png 782w" sizes="(max-width: 750px) 100vw, 750px" /></a><p id="caption-attachment-73850" class="wp-caption-text">NetNut and other proxy services have recast themselves as critical infrastructure for the AI scraping economy. Image: Synthient.com.</p></div>
|
||||
<p>“AI companies depend on web-scraped content: for pre-training, for retrieval, for agent grounding, for search,” reads <a href="https://blog.includesecurity.com/2026/06/the-smart-tv-in-your-livingroom-is-a-node-in-the-aiscraping-economy/" target="_blank" rel="noopener">a report</a> this month from <strong>Include Security</strong> that examines the prevalence of proxy SDKs in smart TV apps. “But the modern web isn’t scrapeable from a datacenter. Cloudflare, DataDome, HUMAN, among others throttle or block requests from known cloud IPs. The workaround is residential proxies. A scraping job routed through a Comcast or T-Mobile subscriber’s connection arrives at the target site from an IP that belongs to a paying residential customer.”</p>
|
||||
<p>This non-stop content scraping has spawned <a href="https://copyrightalliance.org/ai-copyright-lawsuit-developments-2025/" target="_blank" rel="noopener">more than 70 copyright infringement lawsuits</a> against major tech companies that have acknowledged large-scale data scraping as a major source of the “brains” behind their commercial AI offerings. Ironically, much of that scraping is being aided by proxy services that are intimately tied to unofficial Android TV boxes and associated SDKs whose stated purpose is streaming pirated content.</p>
|
||||
<p>The scraping activity has become so aggressive that it often overwhelms the targeted websites, preventing them from being reachable by legitimate visitors. In many reported cases, nonprofit organizations, libraries and universities have complained of constantly battling to keep their services online in the face of relentless data-scraping firms hiding behind residential proxy services.</p>
|
||||
<p>A survey conducted last year by the <strong>Confederation of Open Access Repositories</strong> (COAR) <a href="https://www.cni.org/topics/ci/artificial-intelligence-bots-and-repositories-results-and-next-steps-from-coar-survey" target="_blank" rel="noopener">found</a> while some content scraping bots are rather innocuous, “others are sufficiently aggressive that they are increasingly causing service disruptions in repositories and other scholarly communications infrastructures.” More than 90 percent of survey respondents indicated their repository is encountering aggressive bots, usually more than once a week, and often leading to slow downs and service outages.</p>
|
||||
<p>“Automated web scraping is nothing new, and has been the key technology underlying search engines such as Google for over 30 years,” <a href="https://blog.doaj.org/2026/01/26/open-access-vs-open-excess-doaj-and-ai-scraper-bots/" target="_blank" rel="noopener">wrote</a> <strong>Brendan O’Connell</strong>, platform manager at the <strong>Directory of Open Access Journals</strong> (DOAJ), a free, community-curated index of peer-reviewed academic journals. “However, the current investor-fueled AI startup craze means there are now thousands of well-funded companies developing and deploying their own scraping tools to train AI models, alongside existing major players like OpenAI and Google.”</p>
|
||||
<h2>DON’T TOUCH THAT DIAL!</h2>
|
||||
<p>Across the United States, local communities are pushing back against the proliferation of new data centers aimed primarily at improving the capabilities of AI. But security experts say the general public remains largely unaware that using one of these unsanctioned Android TV boxes means their “smart TV” is almost certainly using a significant amount of bandwidth each month to help train modern AI models.</p>
|
||||
<p>Even households without these sketchy TV boxes can still have their smart TVs turned into residential proxy nodes, just by downloading one of thousands of apps made available on <strong>Samsung</strong> and <strong>LG</strong> smart TVs. Spur said it recently scraped the LG and Samsung app stores and found that each had approximately 3,000 apps available for download. Many of these apps are simple games or utilities that state in the fine print that the user’s Internet connection will be used to download data and that they can opt out at any time.</p>
|
||||
<p>Spur said it found that <em>more than 42 percent of apps available for download via the <strong>webOS</strong> operating system on <strong>LG</strong> smart TVs include SDKs that turn one’s television into an always-on residential proxy node. </em>More than a quarter of the apps made for Samsung’s <strong>Tizen</strong> operating system had similar residential proxy components, Spur found.</p>
|
||||
<div id="attachment_73849" style="width: 758px" class="wp-caption aligncenter"><a href="https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy.png" target="_blank" rel="noopener"><img aria-describedby="caption-attachment-73849" decoding="async" loading="lazy" class="wp-image-73849" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy.png" alt="" width="748" height="304" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy.png 1272w, https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy-768x313.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/06/spur-tvproxy-782x318.png 782w" sizes="(max-width: 748px) 100vw, 748px" /></a><p id="caption-attachment-73849" class="wp-caption-text">Image: Spur.us.</p></div>
|
||||
<p>Experts say it’s questionable whether TV apps with proxy SDKs can obtain meaningful consent from users for installing an always-on proxy connection, particularly when anyone in a household — including children — can effectively opt the family TV into a residential proxy network just by installing a simple game or app.</p>
|
||||
<p>“Privacy-policy disclosure is the wrong control surface for a TV,” Include Security wrote. “It is hard to scroll through a legal document navigated by arrow keys on a remote, and the in-app consent dialog doesn’t convey that a paying customer is about to route their scraping traffic through the user’s home internet.”</p>
|
||||
<p>Spur’s head of research <strong>Sean Simmons</strong> told KrebsOnSecurity that most people do not have a working mental model for what it means to sell access to their residential IP address, no matter what device they are using.</p>
|
||||
<p>“And on a TV, the gap is even wider,” Simmons said. “A one-time prompt navigated with a remote can disappear into the setup flow, while the app keeps monetizing the connection long after anyone remembers what they accepted.”</p>
|
||||
<p>Simmons said LG and Samsung should follow the lead of other TV platforms that have already drawn a line against residential proxy providers, pointing to policies by <strong>Amazon</strong> that prohibit apps facilitating proxy services for third parties. Likewise the TV streaming device maker <strong>Roku</strong> reportedly now bars developers from using proxy SDKs and has removed apps that bundled them.</p>
|
||||
<div id="attachment_73855" style="width: 759px" class="wp-caption aligncenter"><img aria-describedby="caption-attachment-73855" decoding="async" loading="lazy" class=" wp-image-73855" src="https://krebsonsecurity.com/wp-content/uploads/2026/06/tvstreamz.png" alt="" width="749" height="448" srcset="https://krebsonsecurity.com/wp-content/uploads/2026/06/tvstreamz.png 993w, https://krebsonsecurity.com/wp-content/uploads/2026/06/tvstreamz-768x459.png 768w, https://krebsonsecurity.com/wp-content/uploads/2026/06/tvstreamz-782x468.png 782w" sizes="(max-width: 749px) 100vw, 749px" /><p id="caption-attachment-73855" class="wp-caption-text">Piracy related apps pushing proxy SDKs onto unconsenting users. Image: Synthient.</p></div>
|
||||
<p>Apps that turn one’s device into a residential proxy node are not limited to smart TVs and no-name streaming boxes, of course. As noted by the security firm <strong>Infoblox</strong>, mobile app developers can embed SDKs provided by the residential proxy networks into their products to monetize their software, allowing them to receive a small amount of money on each installation.</p>
|
||||
<p>The result, Infoblox said, is that devices are frequently enrolled without the owner’s knowledge, typically through free applications such as VPNs, streaming apps, screensavers and “productivity” apps such as PDF viewers and break reminders.</p>
|
||||
<p>All too often, these proxy services are beaconing out from employee devices brought into the workplace, Infoblox found. In a blog post earlier this month, Infoblox said it discovered that fully 65% of its customer base was querying one or more residential proxy related domains.</p>
|
||||
<p>“We saw steady growth in these queries in 2025, with a 25% increase over the year to over 500 billion per month,” Infoblox <a href="https://www.infoblox.com/blog/threat-intelligence/residential-proxies-in-the-wild/" target="_blank" rel="noopener">wrote</a>. “Over 90% of our pharmaceutical and food & beverage customers have queried residential proxy indicators. Perhaps even more concerning is that over 60% of government and banking customers have as well.”</p>
|
||||
<p>Infoblox researchers <strong>Nick Sundvall</strong> and <strong>David Brunsdon</strong> warned that with residential proxies in the corporate environment, external access is granted to an organization’s IP space.</p>
|
||||
<p>“If threat actors were to abuse the residential proxy to attack a third party, the third party’s incident response would, correctly, identify your residential proxy as the source,” they wrote. “Untangling that, by proving that you were the conduit and not the threat actor, costs time, creates legal exposure, and can damage your reputation. The stunning prevalence of these services within customer environments warrants attention from both network defenders and policy makers who should consider how the risks posed by residential proxies could be impacting their security posture.”</p>
|
||||
]]></content:encoded>
|
||||
|
||||
<wfw:commentRss>https://krebsonsecurity.com/2026/06/popa-botnet-linked-to-publicly-traded-israeli-firm/feed/</wfw:commentRss>
|
||||
<slash:comments>16</slash:comments>
|
||||
|
||||
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
|
||||
<!--
|
||||
Performance optimized by W3 Total Cache. Learn more: https://www.boldgrid.com/w3-total-cache/
|
||||
|
||||
Object Caching 225/225 objects using memcached
|
||||
Page Caching using memcached
|
||||
Database Caching using memcached
|
||||
|
||||
Served from: krebsonsecurity.com @ 2026-07-05 12:52:58 by W3 Total Cache
|
||||
-->
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?><rss version="2.0"
|
||||
xmlns:content="http://purl.org/rss/1.0/modules/content/"
|
||||
xmlns:wfw="http://wellformedweb.org/CommentAPI/"
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:atom="http://www.w3.org/2005/Atom"
|
||||
xmlns:sy="http://purl.org/rss/1.0/modules/syndication/"
|
||||
xmlns:slash="http://purl.org/rss/1.0/modules/slash/"
|
||||
>
|
||||
|
||||
<channel>
|
||||
<title>SecurityWeek</title>
|
||||
<atom:link href="https://www.securityweek.com/feed/" rel="self" type="application/rss+xml" />
|
||||
<link>https://www.securityweek.com/</link>
|
||||
<description>Cybersecurity News, Insights & Analysis</description>
|
||||
<lastBuildDate>Fri, 03 Jul 2026 15:10:44 +0000</lastBuildDate>
|
||||
<language>en-US</language>
|
||||
<sy:updatePeriod>
|
||||
hourly </sy:updatePeriod>
|
||||
<sy:updateFrequency>
|
||||
1 </sy:updateFrequency>
|
||||
<generator>https://wordpress.org/?v=7.0</generator>
|
||||
|
||||
<image>
|
||||
<url>https://www.securityweek.com/wp-content/uploads/2023/01/cropped-SecurityWeek-Icon-32x32.jpeg</url>
|
||||
<title>SecurityWeek</title>
|
||||
<link>https://www.securityweek.com/</link>
|
||||
<width>32</width>
|
||||
<height>32</height>
|
||||
</image>
|
||||
<item>
|
||||
<title>In Other News: Canadian Hacker Jailed, Open Source Zero-Days, Two Sentenced for ATM Jackpotting</title>
|
||||
<link>https://www.securityweek.com/in-other-news-canadian-hacker-jailed-open-source-zero-days-two-sentenced-for-atm-jackpotting/</link>
|
||||
|
||||
<dc:creator><![CDATA[SecurityWeek News]]></dc:creator>
|
||||
<pubDate>Fri, 03 Jul 2026 15:10:42 +0000</pubDate>
|
||||
<category><![CDATA[Cybercrime]]></category>
|
||||
<category><![CDATA[Management & Strategy]]></category>
|
||||
<category><![CDATA[cybersecurity]]></category>
|
||||
<category><![CDATA[News]]></category>
|
||||
<guid isPermaLink="false">https://www.securityweek.com/?p=47625</guid>
|
||||
|
||||
<description><![CDATA[<p>Noteworthy stories that might have slipped under the radar: Anonymous-linked Canadian hacker jailed, researcher drops zero-days in open source projects, Venezuelans sentenced in the US over ATM jackpotting. </p>
|
||||
<p>The post <a href="https://www.securityweek.com/in-other-news-canadian-hacker-jailed-open-source-zero-days-two-sentenced-for-atm-jackpotting/">In Other News: Canadian Hacker Jailed, Open Source Zero-Days, Two Sentenced for ATM Jackpotting</a> appeared first on <a href="https://www.securityweek.com">SecurityWeek</a>.</p>
|
||||
]]></description>
|
||||
|
||||
|
||||
|
||||
</item>
|
||||
<item>
|
||||
<title>Agentic AI Used to Conduct Ransomware Attack via Langflow</title>
|
||||
<link>https://www.securityweek.com/agentic-ai-used-to-conduct-ransomware-attack-via-langflow/</link>
|
||||
|
||||
<dc:creator><![CDATA[Ionut Arghire]]></dc:creator>
|
||||
<pubDate>Fri, 03 Jul 2026 11:00:00 +0000</pubDate>
|
||||
<category><![CDATA[Artificial Intelligence]]></category>
|
||||
<category><![CDATA[Ransomware]]></category>
|
||||
<category><![CDATA[agentic AI]]></category>
|
||||
<category><![CDATA[AI]]></category>
|
||||
<category><![CDATA[Featured]]></category>
|
||||
<guid isPermaLink="false">https://www.securityweek.com/?p=47618</guid>
|
||||
|
||||
<description><![CDATA[<p>Attack demonstrates how LLM agents can combine known exploitation techniques with real-time reasoning to automate complex, multi-stage intrusions.</p>
|
||||
<p>The post <a href="https://www.securityweek.com/agentic-ai-used-to-conduct-ransomware-attack-via-langflow/">Agentic AI Used to Conduct Ransomware Attack via Langflow</a> appeared first on <a href="https://www.securityweek.com">SecurityWeek</a>.</p>
|
||||
]]></description>
|
||||
|
||||
|
||||
|
||||
</item>
|
||||
<item>
|
||||
<title>Medtronic Data Breach Impacts 3.8 Million People</title>
|
||||
<link>https://www.securityweek.com/medtronic-data-breach-impacts-3-8-million-people/</link>
|
||||
|
||||
<dc:creator><![CDATA[Ionut Arghire]]></dc:creator>
|
||||
<pubDate>Fri, 03 Jul 2026 10:00:00 +0000</pubDate>
|
||||
<category><![CDATA[Cybercrime]]></category>
|
||||
<category><![CDATA[Data Breaches]]></category>
|
||||
<category><![CDATA[data breach]]></category>
|
||||
<category><![CDATA[Medtronic]]></category>
|
||||
<category><![CDATA[ShinyHunters]]></category>
|
||||
<guid isPermaLink="false">https://www.securityweek.com/?p=47616</guid>
|
||||
|
||||
<description><![CDATA[<p>In April, ShinyHunters accessed the company’s corporate IT systems and stole patients’ personal and medical information.</p>
|
||||
<p>The post <a href="https://www.securityweek.com/medtronic-data-breach-impacts-3-8-million-people/">Medtronic Data Breach Impacts 3.8 Million People</a> appeared first on <a href="https://www.securityweek.com">SecurityWeek</a>.</p>
|
||||
]]></description>
|
||||
|
||||
|
||||
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?><rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:media="http://search.yahoo.com/mrss/" xmlns:slash="http://purl.org/rss/1.0/modules/slash/" xmlns:sy="http://purl.org/rss/1.0/modules/syndication/" xmlns:wfw="http://wellformedweb.org/CommentAPI/" version="2.0"><channel><title>The Hacker News</title><link>https://thehackernews.com</link><description>Most trusted, widely-read independent cybersecurity news source for everyone; supported by hackers and IT professionals — Send TIPs to admin@thehackernews.com</description><language>en-us</language><lastBuildDate>Sun, 05 Jul 2026 21:16:08 +0530</lastBuildDate><sy:updatePeriod>hourly</sy:updatePeriod><sy:updateFrequency>1</sy:updateFrequency><atom:link href="https://feeds.feedburner.com/TheHackersNews" rel="self" type="application/rss+xml"/><item><title>U.S. Government Entity Paid Kairos $1 Million in Data-Theft Extortion Case</title><description><![CDATA[A U.S. government entity paid about $1 million to keep stolen files from being leaked, according to a new case study by Rakesh Krishnan for Ransom-ISAC, built on a leaked negotiation chat and the blockchain trail the payment left.
|
||||
|
||||
The odd part: the group that took the money calls itself Kairos, but it may not be a ransomware gang at all. Krishnan found no sign that it ever locked a single]]></description><link>https://thehackernews.com/2026/07/us-government-entity-paid-kairos-group.html</link><guid isPermaLink="false">https://thehackernews.com/2026/07/us-government-entity-paid-kairos-group.html</guid><pubDate>Sat, 04 Jul 2026 18:17:53 +0530</pubDate><author>info@thehackernews.com (The Hacker News)</author><enclosure length="12216320" type="image/jpeg" url="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiEfi15-eigOUF4SV157a0LsEW4uNvhRyEIhWWYVNo8tn0tfMjuJ0Cr6zYUDeVh6VfzA-7MpWgIVX8sJgieRMD6e0KSBmB2zNmTU1ko1KPMbkBd-JHMBluVBUOjV06SebkYq4AKFssmEQ6DdGw4yy4X3qesVYJcwwgZSsQFUu4Wfy3un9alrX2Qy8IjacE/s1600/county-hacked.jpg"/></item>
|
||||
<item><title>North Korean Hackers Publish 108 Malicious Packages and Extensions in PolinRider Campaign</title><description><![CDATA[The North Korean threat actors linked to the Contagious Interview campaign have been observed publishing 108 unique packages and web browser extensions spanning npm, Packagist, Go, and Google Chrome as part of an ongoing activity referred to as PolinRider.
|
||||
|
||||
"The campaign remains active, and new malicious packages are likely to continue appearing as threat actors compromise maintainer accounts,]]></description><link>https://thehackernews.com/2026/07/north-korean-hackers-publish-108.html</link><guid isPermaLink="false">https://thehackernews.com/2026/07/north-korean-hackers-publish-108.html</guid><pubDate>Sat, 04 Jul 2026 16:47:24 +0530</pubDate><author>info@thehackernews.com (The Hacker News)</author><enclosure length="12216320" type="image/jpeg" url="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEgwdeBqtwl7nqoAp8TPhZmmr3mUTCcfxluYa7QukD2sI1AjCWoOko2YtQUrCtOLH-tJlYi2lXUA5E3RF51L26gZGyR7sT0GXu73OMB94HhINz5kajaR8-txb-tYNj2Hsm62zVwTaw7Rew6Wazf8eQgo_boWq7DjqbN1jjp5OmwSZ2yG7Y9e04hFVaroUSu6/s1600/go-code.jpg"/></item>
|
||||
<item><title>Unpatched Flaws Disclosed in Filesystem Bundled Into Millions of Embedded Devices</title><description><![CDATA[Security firm runZero has disclosed seven vulnerabilities in FatFs, a small filesystem library that lets a device read and write the FAT and exFAT formats used on USB drives and SD cards.
|
||||
|
||||
The flaws matter because FatFs is nearly everywhere. It ships inside the firmware that runs security cameras, drones, industrial controllers, hardware crypto wallets, and other devices built on]]></description><link>https://thehackernews.com/2026/07/unpatched-flaws-disclosed-in-filesystem.html</link><guid isPermaLink="false">https://thehackernews.com/2026/07/unpatched-flaws-disclosed-in-filesystem.html</guid><pubDate>Sat, 04 Jul 2026 01:49:31 +0530</pubDate><author>info@thehackernews.com (The Hacker News)</author><enclosure length="12216320" type="image/jpeg" url="https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEiJGo3ti2B3O-v3XpxdiFLMhvMVB_Ee5mmTlis-Qls6K8auWLQtPb4DSLL4snqvuZSxEvYXbGVBiUvRH1wBiw53ovRi-KTpdb0WqPwkZMkA_eHTSkTQVPkexu02-I2u7wc7PKESGKFZxlKRjR1SAkgOK50kIfpqZ9wcEjZCYBWIh6u_L82tXa8WfgUIhmA/s1600/fatfs.jpg"/></item></channel></rss>
|
||||
File diff suppressed because one or more lines are too long
Loading…
Reference in New Issue