From 2d84ef62f773d9a7e02bb1805454fba5061228a2 Mon Sep 17 00:00:00 2001 From: CarterPerez-dev Date: Wed, 28 Jan 2026 16:30:50 -0500 Subject: [PATCH] add requirements.txt parsing, scanning, and updating support --- .../internal/cli/update.go | 26 +++- .../internal/requirements/parser.go | 91 +++++++++++++ .../internal/requirements/parser_test.go | 122 ++++++++++++++++++ .../internal/requirements/writer.go | 80 ++++++++++++ .../internal/requirements/writer_test.go | 92 +++++++++++++ .../testdata/requirements.txt | 13 ++ 6 files changed, 421 insertions(+), 3 deletions(-) create mode 100644 PROJECTS/simple-vulnerability-scanner/internal/requirements/parser.go create mode 100644 PROJECTS/simple-vulnerability-scanner/internal/requirements/parser_test.go create mode 100644 PROJECTS/simple-vulnerability-scanner/internal/requirements/writer.go create mode 100644 PROJECTS/simple-vulnerability-scanner/internal/requirements/writer_test.go create mode 100644 PROJECTS/simple-vulnerability-scanner/testdata/requirements.txt diff --git a/PROJECTS/simple-vulnerability-scanner/internal/cli/update.go b/PROJECTS/simple-vulnerability-scanner/internal/cli/update.go index bb6f887f..df7d5247 100644 --- a/PROJECTS/simple-vulnerability-scanner/internal/cli/update.go +++ b/PROJECTS/simple-vulnerability-scanner/internal/cli/update.go @@ -9,12 +9,14 @@ import ( "os" "path/filepath" "sort" + "strings" "time" "github.com/CarterPerez-dev/angela/internal/config" "github.com/CarterPerez-dev/angela/internal/osv" "github.com/CarterPerez-dev/angela/internal/pypi" "github.com/CarterPerez-dev/angela/internal/pyproject" + "github.com/CarterPerez-dev/angela/internal/requirements" "github.com/CarterPerez-dev/angela/pkg/types" "github.com/spf13/cobra" ) @@ -236,7 +238,7 @@ func runUpdate( start := time.Now() cfg := config.Load(f.file) - deps, err := pyproject.ParseFile(f.file) + deps, err := parseDeps(f.file) if err != nil { return err } @@ -274,7 +276,7 @@ func runUpdate( } if !dryRun && len(updateSpecs) > 0 { - if err := pyproject.UpdateFile(f.file, updateSpecs); err != nil { + if err := updateDepsFile(f.file, updateSpecs); err != nil { return fmt.Errorf("write updates: %w", err) } } @@ -311,7 +313,7 @@ func runScan(ctx context.Context, file string) error { start := time.Now() cfg := config.Load(file) - deps, err := pyproject.ParseFile(file) + deps, err := parseDeps(file) if err != nil { return err } @@ -491,6 +493,24 @@ func latestAny(versions []string) (pypi.Version, error) { return latest, nil } +func isRequirementsTxt(path string) bool { + return strings.HasSuffix(strings.ToLower(path), ".txt") +} + +func parseDeps(file string) ([]types.Dependency, error) { + if isRequirementsTxt(file) { + return requirements.ParseFile(file) + } + return pyproject.ParseFile(file) +} + +func updateDepsFile(file string, specs map[string]string) error { + if isRequirementsTxt(file) { + return requirements.UpdateFile(file, specs) + } + return pyproject.UpdateFile(file, specs) +} + func resolveMinSeverity(configVal string) string { if minSeverity != "" { return minSeverity diff --git a/PROJECTS/simple-vulnerability-scanner/internal/requirements/parser.go b/PROJECTS/simple-vulnerability-scanner/internal/requirements/parser.go new file mode 100644 index 00000000..81d1087e --- /dev/null +++ b/PROJECTS/simple-vulnerability-scanner/internal/requirements/parser.go @@ -0,0 +1,91 @@ +// ©AngelaMos | 2026 +// parser.go + +package requirements + +import ( + "bufio" + "fmt" + "os" + "strings" + + "github.com/CarterPerez-dev/angela/pkg/types" +) + +// ParseFile reads a requirements.txt and extracts all dependency declarations +func ParseFile(path string) ([]types.Dependency, error) { + f, err := os.Open(path) //nolint:gosec + if err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf( + "open %s: file not found", path, + ) + } + return nil, fmt.Errorf("read %s: %w", path, err) + } + defer func() { _ = f.Close() }() //nolint:errcheck + + var deps []types.Dependency + scanner := bufio.NewScanner(f) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + if line == "" || + strings.HasPrefix(line, "#") || + strings.HasPrefix(line, "-") { + continue + } + + if idx := strings.Index(line, " #"); idx >= 0 { + line = strings.TrimSpace(line[:idx]) + } + + dep := parseLine(line) + if dep.Name != "" { + deps = append(deps, dep) + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read %s: %w", path, err) + } + + if len(deps) == 0 { + return nil, fmt.Errorf( + "%s: no dependencies found", path, + ) + } + return deps, nil +} + +func parseLine(s string) types.Dependency { + var dep types.Dependency + + if idx := strings.Index(s, ";"); idx >= 0 { + dep.Markers = strings.TrimSpace(s[idx+1:]) + s = strings.TrimSpace(s[:idx]) + } + + if start := strings.Index(s, "["); start >= 0 { + end := strings.Index(s, "]") + if end > start { + for _, e := range strings.Split(s[start+1:end], ",") { + dep.Extras = append( + dep.Extras, + strings.TrimSpace(e), + ) + } + s = s[:start] + s[end+1:] + } + } + + if idx := strings.IndexAny(s, "><=!~"); idx >= 0 { + dep.Name = strings.TrimSpace(s[:idx]) + dep.Spec = strings.TrimSpace(s[idx:]) + } else { + dep.Name = strings.TrimSpace(s) + } + + return dep +} diff --git a/PROJECTS/simple-vulnerability-scanner/internal/requirements/parser_test.go b/PROJECTS/simple-vulnerability-scanner/internal/requirements/parser_test.go new file mode 100644 index 00000000..506080db --- /dev/null +++ b/PROJECTS/simple-vulnerability-scanner/internal/requirements/parser_test.go @@ -0,0 +1,122 @@ +// ©AngelaMos | 2026 +// parser_test.go + +package requirements + +import ( + "os" + "path/filepath" + "testing" +) + +func TestParseFile(t *testing.T) { + t.Parallel() + + content := `# Core deps +django>=3.2.0 +requests==2.28.1 +flask>=2.0,<3.0 +numpy # pinned +click[extra]>=8.0.0; python_version>="3.8" + +# Options to skip +-r other.txt +-e ./local-pkg +--index-url https://pypi.org/simple/ +` + + path := filepath.Join(t.TempDir(), "requirements.txt") + os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck + + deps, err := ParseFile(path) + if err != nil { + t.Fatalf("ParseFile: %v", err) + } + + if len(deps) != 5 { + t.Fatalf("len = %d, want 5", len(deps)) + } + + tests := []struct { + name string + spec string + }{ + {"django", ">=3.2.0"}, + {"requests", "==2.28.1"}, + {"flask", ">=2.0,<3.0"}, + {"numpy", ""}, + {"click", ">=8.0.0"}, + } + + for i, tt := range tests { + if deps[i].Name != tt.name { + t.Errorf("deps[%d].Name = %q, want %q", i, deps[i].Name, tt.name) + } + if deps[i].Spec != tt.spec { + t.Errorf("deps[%d].Spec = %q, want %q", i, deps[i].Spec, tt.spec) + } + } +} + +func TestParseFileExtras(t *testing.T) { + t.Parallel() + + content := "requests[security,socks]>=2.28.0\n" + path := filepath.Join(t.TempDir(), "requirements.txt") + os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck + + deps, err := ParseFile(path) + if err != nil { + t.Fatalf("ParseFile: %v", err) + } + + if len(deps) != 1 { + t.Fatalf("len = %d, want 1", len(deps)) + } + + if len(deps[0].Extras) != 2 { + t.Fatalf("extras len = %d, want 2", len(deps[0].Extras)) + } + if deps[0].Extras[0] != "security" || deps[0].Extras[1] != "socks" { + t.Errorf("extras = %v, want [security socks]", deps[0].Extras) + } +} + +func TestParseFileMarkers(t *testing.T) { + t.Parallel() + + content := "pywin32>=300; sys_platform==\"win32\"\n" + path := filepath.Join(t.TempDir(), "requirements.txt") + os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck + + deps, err := ParseFile(path) + if err != nil { + t.Fatalf("ParseFile: %v", err) + } + + if deps[0].Markers != `sys_platform=="win32"` { + t.Errorf("markers = %q, want sys_platform==\"win32\"", deps[0].Markers) + } +} + +func TestParseFileEmpty(t *testing.T) { + t.Parallel() + + content := "# just comments\n\n-r other.txt\n" + path := filepath.Join(t.TempDir(), "requirements.txt") + os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck + + _, err := ParseFile(path) + if err == nil { + t.Error("expected error for empty requirements") + } +} + +func TestParseFileNotFound(t *testing.T) { + t.Parallel() + + _, err := ParseFile("/nonexistent/requirements.txt") + if err == nil { + t.Error("expected error for missing file") + } +} diff --git a/PROJECTS/simple-vulnerability-scanner/internal/requirements/writer.go b/PROJECTS/simple-vulnerability-scanner/internal/requirements/writer.go new file mode 100644 index 00000000..6fb4cecb --- /dev/null +++ b/PROJECTS/simple-vulnerability-scanner/internal/requirements/writer.go @@ -0,0 +1,80 @@ +// ©AngelaMos | 2026 +// writer.go + +package requirements + +import ( + "fmt" + "os" + "regexp" + "strings" + + "github.com/CarterPerez-dev/angela/internal/pypi" +) + +// UpdateFile replaces version specifiers for the given packages in a +// requirements.txt file, preserving all comments and formatting +func UpdateFile(path string, updates map[string]string) error { + data, err := os.ReadFile(path) //nolint:gosec + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + + content := data + for pkg, spec := range updates { + pattern := buildPattern(pkg) + found := false + content = pattern.ReplaceAllFunc( + content, + func(match []byte) []byte { + found = true + return replaceSpec(pattern, match, spec) + }, + ) + if !found { + return fmt.Errorf( + "dependency %q not found in %s", pkg, path, + ) + } + } + + tmp := path + ".tmp" + if err := os.WriteFile(tmp, content, 0o600); err != nil { + return fmt.Errorf("write temp: %w", err) + } + if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) //nolint:errcheck + return fmt.Errorf("rename: %w", err) + } + return nil +} + +func buildPattern(name string) *regexp.Regexp { + normalized := pypi.NormalizeName(name) + parts := strings.Split(normalized, "-") + for i, p := range parts { + parts[i] = regexp.QuoteMeta(p) + } + namePattern := strings.Join(parts, `[-_.]?`) + + return regexp.MustCompile( + `(?im)^(` + namePattern + `)` + + `(\[[^\]]*\])?` + + `(\s*[><=!~][^\n;#]*)`, + ) +} + +func replaceSpec( + re *regexp.Regexp, match []byte, newSpec string, +) []byte { + groups := re.FindSubmatch(match) + if len(groups) < 4 { + return match + } + + var b []byte + b = append(b, groups[1]...) + b = append(b, groups[2]...) + b = append(b, []byte(newSpec)...) + return b +} diff --git a/PROJECTS/simple-vulnerability-scanner/internal/requirements/writer_test.go b/PROJECTS/simple-vulnerability-scanner/internal/requirements/writer_test.go new file mode 100644 index 00000000..da7c7b3d --- /dev/null +++ b/PROJECTS/simple-vulnerability-scanner/internal/requirements/writer_test.go @@ -0,0 +1,92 @@ +// ©AngelaMos | 2026 +// writer_test.go + +package requirements + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestUpdateFile(t *testing.T) { + t.Parallel() + + content := `# Core deps +django>=3.2.0 +requests==2.28.1 # HTTP client +flask>=2.0.0 +` + + path := filepath.Join(t.TempDir(), "requirements.txt") + os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck + + updates := map[string]string{ + "django": ">=6.0.1", + "requests": ">=2.32.5", + } + + if err := UpdateFile(path, updates); err != nil { + t.Fatalf("UpdateFile: %v", err) + } + + data, _ := os.ReadFile(path) //nolint:errcheck,gosec + result := string(data) + + if !strings.Contains(result, "django>=6.0.1") { + t.Error("django not updated") + } + if !strings.Contains(result, "requests>=2.32.5") { + t.Error("requests not updated") + } + if !strings.Contains(result, "# Core deps") { + t.Error("comment lost") + } + if !strings.Contains(result, "# HTTP client") { + t.Error("inline comment lost") + } + if !strings.Contains(result, "flask>=2.0.0") { + t.Error("flask was incorrectly modified") + } +} + +func TestUpdateFilePreservesFormatting(t *testing.T) { + t.Parallel() + + content := "Django>=3.2.0\nnumpy==1.24.0\n" + path := filepath.Join(t.TempDir(), "requirements.txt") + os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck + + updates := map[string]string{ + "django": ">=6.0.1", + } + + if err := UpdateFile(path, updates); err != nil { + t.Fatalf("UpdateFile: %v", err) + } + + data, _ := os.ReadFile(path) //nolint:errcheck,gosec + result := string(data) + + if !strings.Contains(result, "Django>=6.0.1") { + t.Error("original casing not preserved") + } + if !strings.Contains(result, "numpy==1.24.0") { + t.Error("numpy was incorrectly modified") + } +} + +func TestUpdateFileNotFound(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "requirements.txt") + os.WriteFile(path, []byte("flask>=2.0.0\n"), 0o600) //nolint:errcheck + + err := UpdateFile(path, map[string]string{ + "nonexistent": ">=1.0.0", + }) + if err == nil { + t.Error("expected error for missing dependency") + } +} diff --git a/PROJECTS/simple-vulnerability-scanner/testdata/requirements.txt b/PROJECTS/simple-vulnerability-scanner/testdata/requirements.txt new file mode 100644 index 00000000..813fd38f --- /dev/null +++ b/PROJECTS/simple-vulnerability-scanner/testdata/requirements.txt @@ -0,0 +1,13 @@ +# Core dependencies +django>=3.2.0 +requests>=2.28.0 +flask>=2.0.0 + +# Dev tools +black>=23.0.0 +pydantic>=1.10.0 + +# Pinned +numpy==1.24.0 +pytest>=7.0.0 +click>=8.0.0