343 lines
12 KiB
Python
343 lines
12 KiB
Python
"""Stale ``min-release-age-exclude[]`` entries in ``.npmrc`` files.
|
|
|
|
The ``min-release-age`` gate blocks npm from *resolving* package versions
|
|
younger than N days (supply-chain quarantine). An exclude exists to let
|
|
resolution pick a specific younger release — usually a vuln fix whose
|
|
comment literally says "remove when > 2wks old". Once every locked
|
|
version of the excluded package is itself older than N days, resolution
|
|
succeeds without the exclude: it is dead weight and a standing hole in
|
|
the age gate for that package. This lint automates exactly the removal
|
|
instruction those comments carry.
|
|
|
|
Semantics per exclude pattern (exact name, ``@scope/*``, or trailing
|
|
glob), matched against the sibling lockfile's locked packages:
|
|
|
|
- pattern matches no locked package -> stale (nothing to exempt)
|
|
- every matched locked version was published more than
|
|
``min-release-age`` (+1 day boundary grace) days ago -> stale
|
|
- any registry fetch fails -> pattern skipped (fail open: never remove
|
|
an exclude on partial data; worst case it survives one merge cycle)
|
|
|
|
Opt-out: a ``# lint: keep`` comment anywhere in the entry's comment
|
|
block exempts the whole block (for intentional standing excludes like
|
|
fast-moving upstreams).
|
|
|
|
The fixer deletes stale exclude lines; an entry group's comment block
|
|
goes with it only when every exclude in the group was removed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import urllib.request
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timedelta, timezone
|
|
from fnmatch import fnmatch
|
|
from pathlib import Path
|
|
from typing import Callable
|
|
from urllib.parse import quote
|
|
|
|
from lints import REPO_ROOT, Finding, Lint
|
|
|
|
REGISTRY = "https://registry.npmjs.org"
|
|
KEEP_MARKER = re.compile(r"#\s*lint:\s*keep\b", re.IGNORECASE)
|
|
# One day of grace above min-release-age so a package hovering at the
|
|
# boundary doesn't get its exclude removed and re-added across runs.
|
|
GRACE = timedelta(days=1)
|
|
|
|
_EXCLUDE_RE = re.compile(r"^min-release-age-exclude\[\]=(?P<pattern>\S+)\s*$")
|
|
_MIN_AGE_RE = re.compile(r"^min-release-age=(?P<days>\d+)\s*$")
|
|
|
|
|
|
@dataclass
|
|
class ExcludeGroup:
|
|
"""A contiguous comment block plus the exclude lines under it."""
|
|
|
|
comment_idxs: list[int] = field(default_factory=list)
|
|
# (line_idx, pattern) pairs
|
|
entries: list[tuple[int, str]] = field(default_factory=list)
|
|
keep: bool = False
|
|
|
|
|
|
def parse_npmrc(text: str) -> tuple[int | None, list[ExcludeGroup]]:
|
|
"""Extract ``min-release-age`` and exclude groups from .npmrc text.
|
|
|
|
A group is the contiguous run of ``#`` comment lines directly above
|
|
an exclude (no blank line or other directive in between) plus every
|
|
exclude line contiguous below it.
|
|
"""
|
|
lines = text.splitlines()
|
|
min_age: int | None = None
|
|
groups: list[ExcludeGroup] = []
|
|
pending_comments: list[int] = []
|
|
current: ExcludeGroup | None = None
|
|
|
|
for i, line in enumerate(lines):
|
|
stripped = line.strip()
|
|
if m := _MIN_AGE_RE.match(stripped):
|
|
min_age = int(m.group("days"))
|
|
pending_comments = []
|
|
current = None
|
|
elif m := _EXCLUDE_RE.match(stripped):
|
|
if current is None:
|
|
current = ExcludeGroup(comment_idxs=pending_comments)
|
|
current.keep = any(
|
|
KEEP_MARKER.search(lines[c]) for c in pending_comments
|
|
)
|
|
groups.append(current)
|
|
pending_comments = []
|
|
current.entries.append((i, m.group("pattern")))
|
|
elif stripped.startswith("#"):
|
|
if current is not None:
|
|
# Comment directly after excludes starts a new block.
|
|
current = None
|
|
pending_comments.append(i)
|
|
else:
|
|
# Blank line or other directive breaks both runs.
|
|
pending_comments = []
|
|
current = None
|
|
|
|
return min_age, groups
|
|
|
|
|
|
def locked_versions(lockfile: Path) -> dict[str, set[str]]:
|
|
"""``name -> {versions}`` from a v2/v3 package-lock.json.
|
|
|
|
Keys in ``packages`` are install paths (``node_modules/<name>``,
|
|
possibly nested); the package name is everything after the last
|
|
``node_modules/`` segment.
|
|
"""
|
|
data = json.loads(lockfile.read_text(encoding="utf-8"))
|
|
out: dict[str, set[str]] = {}
|
|
for path, meta in data.get("packages", {}).items():
|
|
if not path or "node_modules/" not in path:
|
|
continue
|
|
name = path.rsplit("node_modules/", 1)[1]
|
|
version = meta.get("version")
|
|
if name and version:
|
|
out.setdefault(name, set()).add(version)
|
|
return out
|
|
|
|
|
|
def _fetch_times(name: str) -> dict[str, str]:
|
|
"""``version -> ISO publish date`` from the npm registry."""
|
|
url = f"{REGISTRY}/{quote(name, safe='@/')}"
|
|
req = urllib.request.Request(url, headers={"Accept": "application/json"})
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
return json.load(resp).get("time", {})
|
|
|
|
|
|
def _parse_iso(stamp: str) -> datetime | None:
|
|
try:
|
|
return datetime.fromisoformat(stamp.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def evaluate_file(
|
|
npmrc: Path,
|
|
lockfile: Path,
|
|
fetch_times: Callable[[str], dict[str, str]],
|
|
now: datetime | None = None,
|
|
) -> list[tuple[int, str, str]]:
|
|
"""Return stale entries as ``(line_idx, pattern, reason)``."""
|
|
now = now or datetime.now(timezone.utc)
|
|
min_age, groups = parse_npmrc(npmrc.read_text(encoding="utf-8"))
|
|
if min_age is None or not groups:
|
|
return []
|
|
if not lockfile.exists():
|
|
return []
|
|
locked = locked_versions(lockfile)
|
|
threshold = now - timedelta(days=min_age) - GRACE
|
|
|
|
times_cache: dict[str, dict[str, str] | None] = {}
|
|
|
|
def times_for(name: str) -> dict[str, str] | None:
|
|
if name not in times_cache:
|
|
try:
|
|
times_cache[name] = fetch_times(name)
|
|
except Exception: # noqa: BLE001 — fail open on any fetch error
|
|
times_cache[name] = None
|
|
return times_cache[name]
|
|
|
|
stale: list[tuple[int, str, str]] = []
|
|
for group in groups:
|
|
if group.keep:
|
|
continue
|
|
for line_idx, pattern in group.entries:
|
|
matched = {
|
|
name: versions
|
|
for name, versions in locked.items()
|
|
if fnmatch(name, pattern)
|
|
}
|
|
if not matched:
|
|
stale.append(
|
|
(
|
|
line_idx,
|
|
pattern,
|
|
"matches no package in the lockfile — nothing to exempt",
|
|
)
|
|
)
|
|
continue
|
|
|
|
all_old = True
|
|
fetch_failed = False
|
|
for name, versions in matched.items():
|
|
times = times_for(name)
|
|
if times is None:
|
|
fetch_failed = True
|
|
break
|
|
for version in versions:
|
|
published = _parse_iso(times.get(version, ""))
|
|
if published is None or published > threshold:
|
|
# Unknown date counts as young — fail open.
|
|
all_old = False
|
|
break
|
|
if not all_old:
|
|
break
|
|
|
|
if fetch_failed or not all_old:
|
|
continue
|
|
stale.append(
|
|
(
|
|
line_idx,
|
|
pattern,
|
|
f"every locked version is older than min-release-age "
|
|
f"({min_age}d) — the age gate no longer needs this exclude",
|
|
)
|
|
)
|
|
return stale
|
|
|
|
|
|
def remove_stale_lines(text: str, stale_idxs: set[int]) -> str:
|
|
"""Delete stale exclude lines; drop a group's comment block only when
|
|
the group has no surviving excludes."""
|
|
_, groups = parse_npmrc(text)
|
|
to_delete = set(stale_idxs)
|
|
for group in groups:
|
|
entry_idxs = {i for i, _ in group.entries}
|
|
if entry_idxs and entry_idxs <= to_delete:
|
|
to_delete.update(group.comment_idxs)
|
|
|
|
lines = text.splitlines()
|
|
kept = [line for i, line in enumerate(lines) if i not in to_delete]
|
|
out: list[str] = []
|
|
# Collapse the double blank lines that full-group deletion leaves.
|
|
for line in kept:
|
|
if line.strip() == "" and out and out[-1].strip() == "":
|
|
continue
|
|
out.append(line)
|
|
result = "\n".join(out)
|
|
if text.endswith("\n") and not result.endswith("\n"):
|
|
result += "\n"
|
|
return result
|
|
|
|
|
|
def _tracked_npmrc_files() -> list[Path]:
|
|
out = subprocess.run(
|
|
["git", "ls-files", "*.npmrc", "**/.npmrc", ".npmrc"],
|
|
cwd=REPO_ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
encoding="utf-8",
|
|
check=False,
|
|
)
|
|
return [REPO_ROOT / p for p in dict.fromkeys(out.stdout.split())]
|
|
|
|
|
|
def _evaluate_repo() -> list[tuple[Path, list[tuple[int, str, str]]]]:
|
|
results = []
|
|
for npmrc in _tracked_npmrc_files():
|
|
lockfile = npmrc.parent / "package-lock.json"
|
|
stale = evaluate_file(npmrc, lockfile, _fetch_times)
|
|
if stale:
|
|
results.append((npmrc, stale))
|
|
return results
|
|
|
|
|
|
def check() -> list[Finding]:
|
|
findings = []
|
|
for npmrc, stale in _evaluate_repo():
|
|
rel = npmrc.relative_to(REPO_ROOT).as_posix()
|
|
for line_idx, pattern, reason in stale:
|
|
findings.append(
|
|
Finding(
|
|
lint_id="npmrc-stale-exclude",
|
|
path=rel,
|
|
line=line_idx + 1,
|
|
message=(
|
|
f"min-release-age-exclude[]={pattern} is stale: {reason}. "
|
|
"Add `# lint: keep` above it if it is intentional."
|
|
),
|
|
fixable=True,
|
|
)
|
|
)
|
|
return findings
|
|
|
|
|
|
def _verify_only_exclude_deletions(old: str, new: str) -> None:
|
|
"""Abort unless ``new`` is ``old`` minus deletions of exclude lines,
|
|
comment lines, and blank lines — the only edits this fixer makes.
|
|
|
|
Structural guarantee, not a diff heuristic: every surviving line must
|
|
appear in the original in order (pure-deletion check), and every
|
|
dropped line must be an exclude entry, a comment, or blank. Any other
|
|
delta means a bug in remove_stale_lines, and the file must not be
|
|
written.
|
|
"""
|
|
old_lines = old.splitlines()
|
|
new_lines = new.splitlines()
|
|
|
|
i = 0
|
|
dropped: list[str] = []
|
|
for line in old_lines:
|
|
if i < len(new_lines) and new_lines[i] == line:
|
|
i += 1
|
|
else:
|
|
dropped.append(line)
|
|
if i != len(new_lines):
|
|
raise RuntimeError(
|
|
"npmrc-stale-exclude: fixer produced lines not present in the "
|
|
"original file — refusing to write"
|
|
)
|
|
|
|
for line in dropped:
|
|
stripped = line.strip()
|
|
if stripped == "" or stripped.startswith("#"):
|
|
continue
|
|
if _EXCLUDE_RE.match(stripped):
|
|
continue
|
|
raise RuntimeError(
|
|
"npmrc-stale-exclude: fixer would delete a non-exclude "
|
|
f"directive {stripped!r} — refusing to write"
|
|
)
|
|
|
|
|
|
def fix() -> list[str]:
|
|
changed = []
|
|
for npmrc, stale in _evaluate_repo():
|
|
text = npmrc.read_text(encoding="utf-8")
|
|
new_text = remove_stale_lines(text, {idx for idx, _, _ in stale})
|
|
if new_text != text:
|
|
_verify_only_exclude_deletions(text, new_text)
|
|
npmrc.write_text(new_text, encoding="utf-8")
|
|
changed.append(npmrc.relative_to(REPO_ROOT).as_posix())
|
|
return changed
|
|
|
|
|
|
LINT = Lint(
|
|
id="npmrc-stale-exclude",
|
|
description=(
|
|
"min-release-age-exclude[] entries whose locked versions have all "
|
|
"outgrown min-release-age are dead weight in the age gate."
|
|
),
|
|
severity="blocking",
|
|
autofix=True,
|
|
check=check,
|
|
fix=fix,
|
|
network=True,
|
|
fix_touches=("*.npmrc",),
|
|
)
|