add requirements.txt parsing, scanning, and updating support

This commit is contained in:
CarterPerez-dev 2026-01-28 16:30:50 -05:00
parent b0c58a529d
commit 2d84ef62f7
6 changed files with 421 additions and 3 deletions

View File

@ -9,12 +9,14 @@ import (
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/CarterPerez-dev/angela/internal/config"
"github.com/CarterPerez-dev/angela/internal/osv"
"github.com/CarterPerez-dev/angela/internal/pypi"
"github.com/CarterPerez-dev/angela/internal/pyproject"
"github.com/CarterPerez-dev/angela/internal/requirements"
"github.com/CarterPerez-dev/angela/pkg/types"
"github.com/spf13/cobra"
)
@ -236,7 +238,7 @@ func runUpdate(
start := time.Now()
cfg := config.Load(f.file)
deps, err := pyproject.ParseFile(f.file)
deps, err := parseDeps(f.file)
if err != nil {
return err
}
@ -274,7 +276,7 @@ func runUpdate(
}
if !dryRun && len(updateSpecs) > 0 {
if err := pyproject.UpdateFile(f.file, updateSpecs); err != nil {
if err := updateDepsFile(f.file, updateSpecs); err != nil {
return fmt.Errorf("write updates: %w", err)
}
}
@ -311,7 +313,7 @@ func runScan(ctx context.Context, file string) error {
start := time.Now()
cfg := config.Load(file)
deps, err := pyproject.ParseFile(file)
deps, err := parseDeps(file)
if err != nil {
return err
}
@ -491,6 +493,24 @@ func latestAny(versions []string) (pypi.Version, error) {
return latest, nil
}
func isRequirementsTxt(path string) bool {
return strings.HasSuffix(strings.ToLower(path), ".txt")
}
func parseDeps(file string) ([]types.Dependency, error) {
if isRequirementsTxt(file) {
return requirements.ParseFile(file)
}
return pyproject.ParseFile(file)
}
func updateDepsFile(file string, specs map[string]string) error {
if isRequirementsTxt(file) {
return requirements.UpdateFile(file, specs)
}
return pyproject.UpdateFile(file, specs)
}
func resolveMinSeverity(configVal string) string {
if minSeverity != "" {
return minSeverity

View File

@ -0,0 +1,91 @@
// ©AngelaMos | 2026
// parser.go
package requirements
import (
"bufio"
"fmt"
"os"
"strings"
"github.com/CarterPerez-dev/angela/pkg/types"
)
// ParseFile reads a requirements.txt and extracts all dependency declarations
func ParseFile(path string) ([]types.Dependency, error) {
f, err := os.Open(path) //nolint:gosec
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf(
"open %s: file not found", path,
)
}
return nil, fmt.Errorf("read %s: %w", path, err)
}
defer func() { _ = f.Close() }() //nolint:errcheck
var deps []types.Dependency
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" ||
strings.HasPrefix(line, "#") ||
strings.HasPrefix(line, "-") {
continue
}
if idx := strings.Index(line, " #"); idx >= 0 {
line = strings.TrimSpace(line[:idx])
}
dep := parseLine(line)
if dep.Name != "" {
deps = append(deps, dep)
}
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("read %s: %w", path, err)
}
if len(deps) == 0 {
return nil, fmt.Errorf(
"%s: no dependencies found", path,
)
}
return deps, nil
}
func parseLine(s string) types.Dependency {
var dep types.Dependency
if idx := strings.Index(s, ";"); idx >= 0 {
dep.Markers = strings.TrimSpace(s[idx+1:])
s = strings.TrimSpace(s[:idx])
}
if start := strings.Index(s, "["); start >= 0 {
end := strings.Index(s, "]")
if end > start {
for _, e := range strings.Split(s[start+1:end], ",") {
dep.Extras = append(
dep.Extras,
strings.TrimSpace(e),
)
}
s = s[:start] + s[end+1:]
}
}
if idx := strings.IndexAny(s, "><=!~"); idx >= 0 {
dep.Name = strings.TrimSpace(s[:idx])
dep.Spec = strings.TrimSpace(s[idx:])
} else {
dep.Name = strings.TrimSpace(s)
}
return dep
}

View File

@ -0,0 +1,122 @@
// ©AngelaMos | 2026
// parser_test.go
package requirements
import (
"os"
"path/filepath"
"testing"
)
func TestParseFile(t *testing.T) {
t.Parallel()
content := `# Core deps
django>=3.2.0
requests==2.28.1
flask>=2.0,<3.0
numpy # pinned
click[extra]>=8.0.0; python_version>="3.8"
# Options to skip
-r other.txt
-e ./local-pkg
--index-url https://pypi.org/simple/
`
path := filepath.Join(t.TempDir(), "requirements.txt")
os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck
deps, err := ParseFile(path)
if err != nil {
t.Fatalf("ParseFile: %v", err)
}
if len(deps) != 5 {
t.Fatalf("len = %d, want 5", len(deps))
}
tests := []struct {
name string
spec string
}{
{"django", ">=3.2.0"},
{"requests", "==2.28.1"},
{"flask", ">=2.0,<3.0"},
{"numpy", ""},
{"click", ">=8.0.0"},
}
for i, tt := range tests {
if deps[i].Name != tt.name {
t.Errorf("deps[%d].Name = %q, want %q", i, deps[i].Name, tt.name)
}
if deps[i].Spec != tt.spec {
t.Errorf("deps[%d].Spec = %q, want %q", i, deps[i].Spec, tt.spec)
}
}
}
func TestParseFileExtras(t *testing.T) {
t.Parallel()
content := "requests[security,socks]>=2.28.0\n"
path := filepath.Join(t.TempDir(), "requirements.txt")
os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck
deps, err := ParseFile(path)
if err != nil {
t.Fatalf("ParseFile: %v", err)
}
if len(deps) != 1 {
t.Fatalf("len = %d, want 1", len(deps))
}
if len(deps[0].Extras) != 2 {
t.Fatalf("extras len = %d, want 2", len(deps[0].Extras))
}
if deps[0].Extras[0] != "security" || deps[0].Extras[1] != "socks" {
t.Errorf("extras = %v, want [security socks]", deps[0].Extras)
}
}
func TestParseFileMarkers(t *testing.T) {
t.Parallel()
content := "pywin32>=300; sys_platform==\"win32\"\n"
path := filepath.Join(t.TempDir(), "requirements.txt")
os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck
deps, err := ParseFile(path)
if err != nil {
t.Fatalf("ParseFile: %v", err)
}
if deps[0].Markers != `sys_platform=="win32"` {
t.Errorf("markers = %q, want sys_platform==\"win32\"", deps[0].Markers)
}
}
func TestParseFileEmpty(t *testing.T) {
t.Parallel()
content := "# just comments\n\n-r other.txt\n"
path := filepath.Join(t.TempDir(), "requirements.txt")
os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck
_, err := ParseFile(path)
if err == nil {
t.Error("expected error for empty requirements")
}
}
func TestParseFileNotFound(t *testing.T) {
t.Parallel()
_, err := ParseFile("/nonexistent/requirements.txt")
if err == nil {
t.Error("expected error for missing file")
}
}

View File

@ -0,0 +1,80 @@
// ©AngelaMos | 2026
// writer.go
package requirements
import (
"fmt"
"os"
"regexp"
"strings"
"github.com/CarterPerez-dev/angela/internal/pypi"
)
// UpdateFile replaces version specifiers for the given packages in a
// requirements.txt file, preserving all comments and formatting
func UpdateFile(path string, updates map[string]string) error {
data, err := os.ReadFile(path) //nolint:gosec
if err != nil {
return fmt.Errorf("read %s: %w", path, err)
}
content := data
for pkg, spec := range updates {
pattern := buildPattern(pkg)
found := false
content = pattern.ReplaceAllFunc(
content,
func(match []byte) []byte {
found = true
return replaceSpec(pattern, match, spec)
},
)
if !found {
return fmt.Errorf(
"dependency %q not found in %s", pkg, path,
)
}
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, content, 0o600); err != nil {
return fmt.Errorf("write temp: %w", err)
}
if err := os.Rename(tmp, path); err != nil {
_ = os.Remove(tmp) //nolint:errcheck
return fmt.Errorf("rename: %w", err)
}
return nil
}
func buildPattern(name string) *regexp.Regexp {
normalized := pypi.NormalizeName(name)
parts := strings.Split(normalized, "-")
for i, p := range parts {
parts[i] = regexp.QuoteMeta(p)
}
namePattern := strings.Join(parts, `[-_.]?`)
return regexp.MustCompile(
`(?im)^(` + namePattern + `)` +
`(\[[^\]]*\])?` +
`(\s*[><=!~][^\n;#]*)`,
)
}
func replaceSpec(
re *regexp.Regexp, match []byte, newSpec string,
) []byte {
groups := re.FindSubmatch(match)
if len(groups) < 4 {
return match
}
var b []byte
b = append(b, groups[1]...)
b = append(b, groups[2]...)
b = append(b, []byte(newSpec)...)
return b
}

View File

@ -0,0 +1,92 @@
// ©AngelaMos | 2026
// writer_test.go
package requirements
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestUpdateFile(t *testing.T) {
t.Parallel()
content := `# Core deps
django>=3.2.0
requests==2.28.1 # HTTP client
flask>=2.0.0
`
path := filepath.Join(t.TempDir(), "requirements.txt")
os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck
updates := map[string]string{
"django": ">=6.0.1",
"requests": ">=2.32.5",
}
if err := UpdateFile(path, updates); err != nil {
t.Fatalf("UpdateFile: %v", err)
}
data, _ := os.ReadFile(path) //nolint:errcheck,gosec
result := string(data)
if !strings.Contains(result, "django>=6.0.1") {
t.Error("django not updated")
}
if !strings.Contains(result, "requests>=2.32.5") {
t.Error("requests not updated")
}
if !strings.Contains(result, "# Core deps") {
t.Error("comment lost")
}
if !strings.Contains(result, "# HTTP client") {
t.Error("inline comment lost")
}
if !strings.Contains(result, "flask>=2.0.0") {
t.Error("flask was incorrectly modified")
}
}
func TestUpdateFilePreservesFormatting(t *testing.T) {
t.Parallel()
content := "Django>=3.2.0\nnumpy==1.24.0\n"
path := filepath.Join(t.TempDir(), "requirements.txt")
os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck
updates := map[string]string{
"django": ">=6.0.1",
}
if err := UpdateFile(path, updates); err != nil {
t.Fatalf("UpdateFile: %v", err)
}
data, _ := os.ReadFile(path) //nolint:errcheck,gosec
result := string(data)
if !strings.Contains(result, "Django>=6.0.1") {
t.Error("original casing not preserved")
}
if !strings.Contains(result, "numpy==1.24.0") {
t.Error("numpy was incorrectly modified")
}
}
func TestUpdateFileNotFound(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "requirements.txt")
os.WriteFile(path, []byte("flask>=2.0.0\n"), 0o600) //nolint:errcheck
err := UpdateFile(path, map[string]string{
"nonexistent": ">=1.0.0",
})
if err == nil {
t.Error("expected error for missing dependency")
}
}

View File

@ -0,0 +1,13 @@
# Core dependencies
django>=3.2.0
requests>=2.28.0
flask>=2.0.0
# Dev tools
black>=23.0.0
pydantic>=1.10.0
# Pinned
numpy==1.24.0
pytest>=7.0.0
click>=8.0.0