add requirements.txt parsing, scanning, and updating support
This commit is contained in:
parent
b0c58a529d
commit
2d84ef62f7
|
|
@ -9,12 +9,14 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/CarterPerez-dev/angela/internal/config"
|
||||
"github.com/CarterPerez-dev/angela/internal/osv"
|
||||
"github.com/CarterPerez-dev/angela/internal/pypi"
|
||||
"github.com/CarterPerez-dev/angela/internal/pyproject"
|
||||
"github.com/CarterPerez-dev/angela/internal/requirements"
|
||||
"github.com/CarterPerez-dev/angela/pkg/types"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
@ -236,7 +238,7 @@ func runUpdate(
|
|||
start := time.Now()
|
||||
cfg := config.Load(f.file)
|
||||
|
||||
deps, err := pyproject.ParseFile(f.file)
|
||||
deps, err := parseDeps(f.file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -274,7 +276,7 @@ func runUpdate(
|
|||
}
|
||||
|
||||
if !dryRun && len(updateSpecs) > 0 {
|
||||
if err := pyproject.UpdateFile(f.file, updateSpecs); err != nil {
|
||||
if err := updateDepsFile(f.file, updateSpecs); err != nil {
|
||||
return fmt.Errorf("write updates: %w", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -311,7 +313,7 @@ func runScan(ctx context.Context, file string) error {
|
|||
start := time.Now()
|
||||
cfg := config.Load(file)
|
||||
|
||||
deps, err := pyproject.ParseFile(file)
|
||||
deps, err := parseDeps(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -491,6 +493,24 @@ func latestAny(versions []string) (pypi.Version, error) {
|
|||
return latest, nil
|
||||
}
|
||||
|
||||
func isRequirementsTxt(path string) bool {
|
||||
return strings.HasSuffix(strings.ToLower(path), ".txt")
|
||||
}
|
||||
|
||||
func parseDeps(file string) ([]types.Dependency, error) {
|
||||
if isRequirementsTxt(file) {
|
||||
return requirements.ParseFile(file)
|
||||
}
|
||||
return pyproject.ParseFile(file)
|
||||
}
|
||||
|
||||
func updateDepsFile(file string, specs map[string]string) error {
|
||||
if isRequirementsTxt(file) {
|
||||
return requirements.UpdateFile(file, specs)
|
||||
}
|
||||
return pyproject.UpdateFile(file, specs)
|
||||
}
|
||||
|
||||
func resolveMinSeverity(configVal string) string {
|
||||
if minSeverity != "" {
|
||||
return minSeverity
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
// ©AngelaMos | 2026
|
||||
// parser.go
|
||||
|
||||
package requirements
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/CarterPerez-dev/angela/pkg/types"
|
||||
)
|
||||
|
||||
// ParseFile reads a requirements.txt and extracts all dependency declarations
|
||||
func ParseFile(path string) ([]types.Dependency, error) {
|
||||
f, err := os.Open(path) //nolint:gosec
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf(
|
||||
"open %s: file not found", path,
|
||||
)
|
||||
}
|
||||
return nil, fmt.Errorf("read %s: %w", path, err)
|
||||
}
|
||||
defer func() { _ = f.Close() }() //nolint:errcheck
|
||||
|
||||
var deps []types.Dependency
|
||||
scanner := bufio.NewScanner(f)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
|
||||
if line == "" ||
|
||||
strings.HasPrefix(line, "#") ||
|
||||
strings.HasPrefix(line, "-") {
|
||||
continue
|
||||
}
|
||||
|
||||
if idx := strings.Index(line, " #"); idx >= 0 {
|
||||
line = strings.TrimSpace(line[:idx])
|
||||
}
|
||||
|
||||
dep := parseLine(line)
|
||||
if dep.Name != "" {
|
||||
deps = append(deps, dep)
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("read %s: %w", path, err)
|
||||
}
|
||||
|
||||
if len(deps) == 0 {
|
||||
return nil, fmt.Errorf(
|
||||
"%s: no dependencies found", path,
|
||||
)
|
||||
}
|
||||
return deps, nil
|
||||
}
|
||||
|
||||
func parseLine(s string) types.Dependency {
|
||||
var dep types.Dependency
|
||||
|
||||
if idx := strings.Index(s, ";"); idx >= 0 {
|
||||
dep.Markers = strings.TrimSpace(s[idx+1:])
|
||||
s = strings.TrimSpace(s[:idx])
|
||||
}
|
||||
|
||||
if start := strings.Index(s, "["); start >= 0 {
|
||||
end := strings.Index(s, "]")
|
||||
if end > start {
|
||||
for _, e := range strings.Split(s[start+1:end], ",") {
|
||||
dep.Extras = append(
|
||||
dep.Extras,
|
||||
strings.TrimSpace(e),
|
||||
)
|
||||
}
|
||||
s = s[:start] + s[end+1:]
|
||||
}
|
||||
}
|
||||
|
||||
if idx := strings.IndexAny(s, "><=!~"); idx >= 0 {
|
||||
dep.Name = strings.TrimSpace(s[:idx])
|
||||
dep.Spec = strings.TrimSpace(s[idx:])
|
||||
} else {
|
||||
dep.Name = strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
return dep
|
||||
}
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
// ©AngelaMos | 2026
|
||||
// parser_test.go
|
||||
|
||||
package requirements
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
content := `# Core deps
|
||||
django>=3.2.0
|
||||
requests==2.28.1
|
||||
flask>=2.0,<3.0
|
||||
numpy # pinned
|
||||
click[extra]>=8.0.0; python_version>="3.8"
|
||||
|
||||
# Options to skip
|
||||
-r other.txt
|
||||
-e ./local-pkg
|
||||
--index-url https://pypi.org/simple/
|
||||
`
|
||||
|
||||
path := filepath.Join(t.TempDir(), "requirements.txt")
|
||||
os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck
|
||||
|
||||
deps, err := ParseFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFile: %v", err)
|
||||
}
|
||||
|
||||
if len(deps) != 5 {
|
||||
t.Fatalf("len = %d, want 5", len(deps))
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
spec string
|
||||
}{
|
||||
{"django", ">=3.2.0"},
|
||||
{"requests", "==2.28.1"},
|
||||
{"flask", ">=2.0,<3.0"},
|
||||
{"numpy", ""},
|
||||
{"click", ">=8.0.0"},
|
||||
}
|
||||
|
||||
for i, tt := range tests {
|
||||
if deps[i].Name != tt.name {
|
||||
t.Errorf("deps[%d].Name = %q, want %q", i, deps[i].Name, tt.name)
|
||||
}
|
||||
if deps[i].Spec != tt.spec {
|
||||
t.Errorf("deps[%d].Spec = %q, want %q", i, deps[i].Spec, tt.spec)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileExtras(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
content := "requests[security,socks]>=2.28.0\n"
|
||||
path := filepath.Join(t.TempDir(), "requirements.txt")
|
||||
os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck
|
||||
|
||||
deps, err := ParseFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFile: %v", err)
|
||||
}
|
||||
|
||||
if len(deps) != 1 {
|
||||
t.Fatalf("len = %d, want 1", len(deps))
|
||||
}
|
||||
|
||||
if len(deps[0].Extras) != 2 {
|
||||
t.Fatalf("extras len = %d, want 2", len(deps[0].Extras))
|
||||
}
|
||||
if deps[0].Extras[0] != "security" || deps[0].Extras[1] != "socks" {
|
||||
t.Errorf("extras = %v, want [security socks]", deps[0].Extras)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileMarkers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
content := "pywin32>=300; sys_platform==\"win32\"\n"
|
||||
path := filepath.Join(t.TempDir(), "requirements.txt")
|
||||
os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck
|
||||
|
||||
deps, err := ParseFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseFile: %v", err)
|
||||
}
|
||||
|
||||
if deps[0].Markers != `sys_platform=="win32"` {
|
||||
t.Errorf("markers = %q, want sys_platform==\"win32\"", deps[0].Markers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
content := "# just comments\n\n-r other.txt\n"
|
||||
path := filepath.Join(t.TempDir(), "requirements.txt")
|
||||
os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck
|
||||
|
||||
_, err := ParseFile(path)
|
||||
if err == nil {
|
||||
t.Error("expected error for empty requirements")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFileNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
_, err := ParseFile("/nonexistent/requirements.txt")
|
||||
if err == nil {
|
||||
t.Error("expected error for missing file")
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
// ©AngelaMos | 2026
|
||||
// writer.go
|
||||
|
||||
package requirements
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/CarterPerez-dev/angela/internal/pypi"
|
||||
)
|
||||
|
||||
// UpdateFile replaces version specifiers for the given packages in a
|
||||
// requirements.txt file, preserving all comments and formatting
|
||||
func UpdateFile(path string, updates map[string]string) error {
|
||||
data, err := os.ReadFile(path) //nolint:gosec
|
||||
if err != nil {
|
||||
return fmt.Errorf("read %s: %w", path, err)
|
||||
}
|
||||
|
||||
content := data
|
||||
for pkg, spec := range updates {
|
||||
pattern := buildPattern(pkg)
|
||||
found := false
|
||||
content = pattern.ReplaceAllFunc(
|
||||
content,
|
||||
func(match []byte) []byte {
|
||||
found = true
|
||||
return replaceSpec(pattern, match, spec)
|
||||
},
|
||||
)
|
||||
if !found {
|
||||
return fmt.Errorf(
|
||||
"dependency %q not found in %s", pkg, path,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, content, 0o600); err != nil {
|
||||
return fmt.Errorf("write temp: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, path); err != nil {
|
||||
_ = os.Remove(tmp) //nolint:errcheck
|
||||
return fmt.Errorf("rename: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildPattern(name string) *regexp.Regexp {
|
||||
normalized := pypi.NormalizeName(name)
|
||||
parts := strings.Split(normalized, "-")
|
||||
for i, p := range parts {
|
||||
parts[i] = regexp.QuoteMeta(p)
|
||||
}
|
||||
namePattern := strings.Join(parts, `[-_.]?`)
|
||||
|
||||
return regexp.MustCompile(
|
||||
`(?im)^(` + namePattern + `)` +
|
||||
`(\[[^\]]*\])?` +
|
||||
`(\s*[><=!~][^\n;#]*)`,
|
||||
)
|
||||
}
|
||||
|
||||
func replaceSpec(
|
||||
re *regexp.Regexp, match []byte, newSpec string,
|
||||
) []byte {
|
||||
groups := re.FindSubmatch(match)
|
||||
if len(groups) < 4 {
|
||||
return match
|
||||
}
|
||||
|
||||
var b []byte
|
||||
b = append(b, groups[1]...)
|
||||
b = append(b, groups[2]...)
|
||||
b = append(b, []byte(newSpec)...)
|
||||
return b
|
||||
}
|
||||
|
|
@ -0,0 +1,92 @@
|
|||
// ©AngelaMos | 2026
|
||||
// writer_test.go
|
||||
|
||||
package requirements
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUpdateFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
content := `# Core deps
|
||||
django>=3.2.0
|
||||
requests==2.28.1 # HTTP client
|
||||
flask>=2.0.0
|
||||
`
|
||||
|
||||
path := filepath.Join(t.TempDir(), "requirements.txt")
|
||||
os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck
|
||||
|
||||
updates := map[string]string{
|
||||
"django": ">=6.0.1",
|
||||
"requests": ">=2.32.5",
|
||||
}
|
||||
|
||||
if err := UpdateFile(path, updates); err != nil {
|
||||
t.Fatalf("UpdateFile: %v", err)
|
||||
}
|
||||
|
||||
data, _ := os.ReadFile(path) //nolint:errcheck,gosec
|
||||
result := string(data)
|
||||
|
||||
if !strings.Contains(result, "django>=6.0.1") {
|
||||
t.Error("django not updated")
|
||||
}
|
||||
if !strings.Contains(result, "requests>=2.32.5") {
|
||||
t.Error("requests not updated")
|
||||
}
|
||||
if !strings.Contains(result, "# Core deps") {
|
||||
t.Error("comment lost")
|
||||
}
|
||||
if !strings.Contains(result, "# HTTP client") {
|
||||
t.Error("inline comment lost")
|
||||
}
|
||||
if !strings.Contains(result, "flask>=2.0.0") {
|
||||
t.Error("flask was incorrectly modified")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateFilePreservesFormatting(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
content := "Django>=3.2.0\nnumpy==1.24.0\n"
|
||||
path := filepath.Join(t.TempDir(), "requirements.txt")
|
||||
os.WriteFile(path, []byte(content), 0o600) //nolint:errcheck
|
||||
|
||||
updates := map[string]string{
|
||||
"django": ">=6.0.1",
|
||||
}
|
||||
|
||||
if err := UpdateFile(path, updates); err != nil {
|
||||
t.Fatalf("UpdateFile: %v", err)
|
||||
}
|
||||
|
||||
data, _ := os.ReadFile(path) //nolint:errcheck,gosec
|
||||
result := string(data)
|
||||
|
||||
if !strings.Contains(result, "Django>=6.0.1") {
|
||||
t.Error("original casing not preserved")
|
||||
}
|
||||
if !strings.Contains(result, "numpy==1.24.0") {
|
||||
t.Error("numpy was incorrectly modified")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateFileNotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "requirements.txt")
|
||||
os.WriteFile(path, []byte("flask>=2.0.0\n"), 0o600) //nolint:errcheck
|
||||
|
||||
err := UpdateFile(path, map[string]string{
|
||||
"nonexistent": ">=1.0.0",
|
||||
})
|
||||
if err == nil {
|
||||
t.Error("expected error for missing dependency")
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
# Core dependencies
|
||||
django>=3.2.0
|
||||
requests>=2.28.0
|
||||
flask>=2.0.0
|
||||
|
||||
# Dev tools
|
||||
black>=23.0.0
|
||||
pydantic>=1.10.0
|
||||
|
||||
# Pinned
|
||||
numpy==1.24.0
|
||||
pytest>=7.0.0
|
||||
click>=8.0.0
|
||||
Loading…
Reference in New Issue