Cybersecurity-Projects/PROJECTS/intermediate/security-news-scraper/internal/ingest/ingest_test.go

// ©AngelaMos | 2026
// ingest_test.go

package ingest

import (
	"context"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/CarterPerez-dev/nadezhda/internal/config"
	"github.com/CarterPerez-dev/nadezhda/internal/fetch"
	"github.com/CarterPerez-dev/nadezhda/internal/normalize"
	"github.com/CarterPerez-dev/nadezhda/internal/source"
	"github.com/CarterPerez-dev/nadezhda/internal/store"
)

const fixture = "../../testdata/feeds/thehackernews.xml"

func loadFixture(t *testing.T) []byte {
	t.Helper()
	b, err := os.ReadFile(fixture)
	if err != nil {
		t.Fatalf("read fixture: %v", err)
	}
	return b
}

func newStore(t *testing.T) *store.Store {
	t.Helper()
	st, err := store.Open(filepath.Join(t.TempDir(), "ingest.db"))
	if err != nil {
		t.Fatalf("open store: %v", err)
	}
	t.Cleanup(func() { _ = st.Close() })
	return st
}

func newClient() *fetch.Client {
	return fetch.New(fetch.Options{
		UserAgent:    "nadezhda-test/1.0",
		PerHostRate:  1e6,
		PerHostBurst: 1,
		Timeout:      5 * time.Second,
		MaxRetries:   2,
	})
}

func target(url string) []source.Source {
	return []source.Source{{
		Name: "test", Title: "Test Feed", URL: url,
		Type: source.KindRSS, Weight: 1.0, Tags: []string{"news"}, Enabled: true,
	}}
}

func TestRunIngestsAndDedups(t *testing.T) {
	body := loadFixture(t)
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		_, _ = w.Write(body)
	}))
	defer srv.Close()

	st := newStore(t)
	cfg := config.Default()
	targets := target(srv.URL + "/feed")
	now := time.Unix(1_800_000_000, 0)

	first, err := Run(context.Background(), newClient(), st, cfg, targets, now)
	if err != nil {
		t.Fatalf("first Run: %v", err)
	}
	r0 := first.Results[0]
	if r0.Err != nil {
		t.Fatalf("source error: %v", r0.Err)
	}
	if r0.New != 3 || r0.Parsed != 3 {
		t.Fatalf("first run: parsed=%d new=%d, want 3/3", r0.Parsed, r0.New)
	}

	count, err := st.CountArticles()
	if err != nil {
		t.Fatal(err)
	}
	if count != 3 {
		t.Fatalf("stored articles = %d, want 3", count)
	}

	var title, canonical, contentHash, titleHash string
	if err := st.DB().QueryRow(
		`SELECT title, canonical_url, content_hash, title_hash FROM articles LIMIT 1`,
	).Scan(&title, &canonical, &contentHash, &titleHash); err != nil {
		t.Fatal(err)
	}
	if contentHash != normalize.ContentHash(canonical) {
		t.Errorf("content_hash = %q, want sha256(canonical)", contentHash)
	}
	if titleHash != normalize.TitleHash(normalize.NormalizeTitle(title)) {
		t.Errorf("title_hash = %q, want sha256(normalized title)", titleHash)
	}

	second, err := Run(context.Background(), newClient(), st, cfg, targets, now)
	if err != nil {
		t.Fatalf("second Run: %v", err)
	}
	r1 := second.Results[0]
	if r1.New != 0 || r1.Duplicates != 3 {
		t.Fatalf("second run: new=%d dup=%d, want 0/3", r1.New, r1.Duplicates)
	}

	count, _ = st.CountArticles()
	if count != 3 {
		t.Fatalf("after re-run stored = %d, want 3 (idempotent)", count)
	}
}

func TestRunConditionalNotModified(t *testing.T) {
	body := loadFixture(t)
	const etag = `"abc123"`
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("If-None-Match") == etag {
			w.WriteHeader(http.StatusNotModified)
			return
		}
		w.Header().Set("ETag", etag)
		_, _ = w.Write(body)
	}))
	defer srv.Close()

	st := newStore(t)
	cfg := config.Default()
	targets := target(srv.URL + "/feed")
	now := time.Unix(1_800_000_000, 0)

	first, err := Run(context.Background(), newClient(), st, cfg, targets, now)
	if err != nil {
		t.Fatalf("first Run: %v", err)
	}
	if first.Results[0].New != 3 {
		t.Fatalf("first run new = %d, want 3", first.Results[0].New)
	}
	var storedETag string
	if err := st.DB().QueryRow(`SELECT etag FROM fetch_state LIMIT 1`).Scan(&storedETag); err != nil {
		t.Fatal(err)
	}
	if storedETag != etag {
		t.Errorf("persisted etag = %q, want %q", storedETag, etag)
	}

	second, err := Run(context.Background(), newClient(), st, cfg, targets, now)
	if err != nil {
		t.Fatalf("second Run: %v", err)
	}
	r := second.Results[0]
	if !r.NotModified {
		t.Errorf("expected NotModified on second run, got %+v", r)
	}
	if r.New != 0 {
		t.Errorf("new = %d on 304, want 0", r.New)
	}
}

func TestRunFailsSoftOnBadSource(t *testing.T) {
	body := loadFixture(t)
	good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		_, _ = w.Write(body)
	}))
	defer good.Close()
	bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusInternalServerError)
	}))
	defer bad.Close()

	st := newStore(t)
	cfg := config.Default()
	targets := []source.Source{
		{Name: "bad", URL: bad.URL + "/feed", Type: source.KindRSS, Weight: 1, Enabled: true},
		{Name: "good", URL: good.URL + "/feed", Type: source.KindRSS, Weight: 1, Enabled: true},
	}
	now := time.Unix(1_800_000_000, 0)

	summary, err := Run(context.Background(), newClient(), st, cfg, targets, now)
	if err != nil {
		t.Fatalf("Run returned error, should fail soft: %v", err)
	}
	byName := map[string]SourceResult{}
	for _, r := range summary.Results {
		byName[r.Name] = r
	}
	if byName["bad"].Err == nil {
		t.Error("bad source should record an error")
	}
	if byName["good"].New != 3 {
		t.Errorf("good source new = %d, want 3 (bad source must not abort run)", byName["good"].New)
	}
}