From e34a29be566ed0eced2927cf384b2fd17978b957 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sat, 8 Aug 2026 10:14:49 -0700 Subject: [PATCH] feat: install skill sets from AI Catalog + agentskills discovery indexes Prototype of the skill-set layering discussed with agentskills.io: - AI Catalog (/.well-known/ai-catalog.json) entries typed application/agent-skills+json point at an agentskills PR #254 discovery index and represent an installable skill set. - The optional io.hermes.skill-set extension carries set-level usage intent: a suggested load-alias command and a shared instruction preamble. Clients that ignore the extension still install the correct set. - tools/skill_set_catalog.py implements the client: $schema gating, required sha256 digest verification, skill-md + archive (.tar.gz/.zip) artifacts, and #254 archive-safety rules (traversal/absolute-path/ link rejection, decompression caps). - hermes skills install-set installs every member through the existing quarantine -> scan -> install pipeline, then creates the / skill bundle so the whole set loads in one turn. - scripts/publish_skill_set.py is the publisher-side counterpart: builds the static .well-known tree (catalog + index + artifacts) from local skill directories with byte-stable archives. --- hermes_cli/skills_hub.py | 194 +++++++++- hermes_cli/subcommands/skills.py | 32 ++ scripts/publish_skill_set.py | 186 ++++++++++ tests/tools/test_skill_set_catalog.py | 458 +++++++++++++++++++++++ tools/skill_set_catalog.py | 504 ++++++++++++++++++++++++++ 5 files changed, 1373 insertions(+), 1 deletion(-) create mode 100644 scripts/publish_skill_set.py create mode 100644 tests/tools/test_skill_set_catalog.py create mode 100644 tools/skill_set_catalog.py diff --git a/hermes_cli/skills_hub.py b/hermes_cli/skills_hub.py index dd0aae4686828..ce025fdf585ef 100644 --- a/hermes_cli/skills_hub.py +++ b/hermes_cli/skills_hub.py @@ -793,6 +793,193 @@ def do_install(identifier: str, category: str = "", force: bool = False, c.print("[dim]Use /reset to start a new session now, or --now to activate immediately (invalidates prompt cache).[/]\n") +def do_install_set(url: str, *, set_name: str = "", force: bool = False, + skip_confirm: bool = False, no_alias: bool = False, + console: Optional[Console] = None) -> None: + """Install a skill *set* from an AI Catalog or a #254 discovery index. + + ``url`` may be: + - an origin (``https://example.com``) — resolved to + ``/.well-known/ai-catalog.json`` + - an AI Catalog JSON URL — skill-set entries + (``application/agent-skills+json``) are listed / selected + - a #254 ``index.json`` URL — treated as a single anonymous set + + Every member flows through the standard fetch -> digest-verify -> + quarantine -> scan -> install pipeline. After installing, a local skill + bundle (load-alias) is created from the entry's ``io.hermes.skill-set`` + extension unless ``no_alias`` is set. + """ + from tools.skill_set_catalog import ( + SkillSetError, catalog_url_for, discover_skill_sets, fetch_member, + resolve_bare_index, resolve_skill_set, + ) + from tools.skills_hub import ( + HUB_DIR, HubLockFile, append_audit_log, ensure_hub_dirs, + install_from_quarantine, quarantine_bundle, source_url_for_bundle, + ) + from tools.skills_guard import ( + format_scan_report, scan_skill_cached, should_allow_install, + ) + + c = console or _console + ensure_hub_dirs() + + # --- Resolve the set ------------------------------------------------- + try: + if url.rstrip("/").endswith("index.json"): + resolved = resolve_bare_index(url, name=set_name) + else: + catalog_url = catalog_url_for(url) + c.print(f"\n[bold]Fetching catalog:[/] {catalog_url}") + sets = discover_skill_sets(catalog_url) + if not sets: + c.print("[bold red]Error:[/] catalog has no skill-set entries " + "(type: application/agent-skills+json).\n") + return + chosen = None + if set_name: + wanted = set_name.strip().lower() + for s in sets: + if wanted in (s.name.strip().lower(), + s.command.strip().lower(), + s.identifier.strip().lower()): + chosen = s + break + if chosen is None: + c.print(f"[bold red]Error:[/] no skill set named '{set_name}'. " + f"Available: {', '.join(s.name for s in sets)}\n") + return + elif len(sets) == 1: + chosen = sets[0] + else: + c.print("\n[bold]Skill sets in this catalog:[/]") + for s in sets: + c.print(f" • [cyan]{s.name}[/] — {s.description or '(no description)'}") + c.print("\nRe-run with [bold]--set [/] to pick one.\n") + return + resolved = resolve_skill_set(chosen) + except SkillSetError as exc: + c.print(f"[bold red]Error:[/] {exc}\n") + return + + info = resolved.info + c.print(f"\n[bold]Skill set:[/] {info.name}") + if info.description: + c.print(f" {info.description}") + c.print(f" [dim]Index: {info.index_url}[/]") + c.print(f" [bold]{len(resolved.members)}[/] member skill(s): " + f"{', '.join(m.name for m in resolved.members)}") + for note in resolved.skipped: + c.print(f" [yellow]Skipped:[/] {note}") + if not resolved.members: + c.print("[bold red]Error:[/] set has no installable members.\n") + return + + if not force and not skip_confirm: + c.print(Panel( + "[bold yellow]You are installing third-party skills at your own risk.[/]\n\n" + "Each member is digest-verified and security-scanned before install,\n" + "but you should review the installed files before use.", + title="Disclaimer", border_style="yellow", + )) + c.print(f"[bold]Install all {len(resolved.members)} skills from " + f"'{info.name}'?[/]") + try: + answer = input("Confirm [y/N]: ").strip().lower() + except (EOFError, KeyboardInterrupt): + answer = "n" + if answer not in {"y", "yes"}: + c.print("[dim]Installation cancelled.[/]\n") + return + + # --- Install each member through the standard pipeline ---------------- + lock = HubLockFile() + installed: List[str] = [] + failed: List[str] = [] + for member in resolved.members: + c.print(f"\n[bold]── {member.name} ──[/]") + try: + bundle = fetch_member(member, set_info=info) + except SkillSetError as exc: + c.print(f"[bold red]Failed:[/] {exc}") + failed.append(member.name) + continue + + if lock.get_installed(bundle.name) and not force: + c.print(f"[yellow]Already installed — skipping.[/] (--force to reinstall)") + installed.append(bundle.name) # still usable by the alias + continue + + try: + q_path = quarantine_bundle(bundle) + except ValueError as exc: + c.print(f"[bold red]Blocked:[/] {exc}") + append_audit_log("BLOCKED", bundle.name, bundle.source, + bundle.trust_level, "invalid_path", str(exc)) + failed.append(member.name) + continue + + result, scan_provenance = scan_skill_cached( + q_path, source=bundle.identifier, + source_url=source_url_for_bundle(bundle), + cache_dir=HUB_DIR / "scan-cache", + ) + c.print(format_scan_report(result)) + allowed, reason = should_allow_install(result, force=force) + if not allowed: + c.print(f"[bold red]Blocked:[/] {reason}") + shutil.rmtree(q_path, ignore_errors=True) + append_audit_log("BLOCKED", bundle.name, bundle.source, + bundle.trust_level, result.verdict, + f"{len(result.findings)}_findings") + failed.append(member.name) + continue + + try: + install_dir = install_from_quarantine( + q_path, bundle.name, "", bundle, result, scan_provenance, + ) + except ValueError as exc: + c.print(f"[bold red]Blocked:[/] {exc}") + shutil.rmtree(q_path, ignore_errors=True) + failed.append(member.name) + continue + c.print(f"[bold green]Installed:[/] {install_dir.name}") + installed.append(bundle.name) + + # --- Create the load-alias (skill bundle) ------------------------------ + c.print() + if failed: + c.print(f"[yellow]{len(failed)} member(s) failed:[/] {', '.join(failed)}") + if not installed: + c.print("[bold red]No skills installed — not creating a bundle alias.[/]\n") + return + + if no_alias: + c.print(f"[bold green]Done.[/] Installed {len(installed)} skill(s).\n") + return + + alias = info.command or info.name + try: + from agent.skill_bundles import save_bundle + bundle_path = save_bundle( + alias, installed, + description=info.description, + instruction=info.instruction, + overwrite=True, + ) + from agent.skill_bundles import _slugify as _bundle_slug + c.print(f"[bold green]Done.[/] Installed {len(installed)} skill(s) and " + f"created the [bold]/{_bundle_slug(alias)}[/] bundle " + f"([dim]{bundle_path}[/]).") + c.print("[dim]Invoke it in chat to load the whole set in one turn. " + "New skills appear next session (or /reset now).[/]\n") + except (ValueError, OSError) as exc: + c.print(f"[yellow]Installed {len(installed)} skill(s), but could not " + f"create the bundle alias: {exc}[/]\n") + + def do_inspect(identifier: str, console: Optional[Console] = None) -> None: """Preview a skill's SKILL.md content without installing.""" from tools.skills_hub import GitHubAuth, create_source_router @@ -1735,6 +1922,11 @@ def skills_command(args) -> None: do_install(args.identifier, category=args.category, force=args.force, skip_confirm=getattr(args, "yes", False), name_override=getattr(args, "name", "") or "") + elif action == "install-set": + do_install_set(args.url, set_name=getattr(args, "set_name", "") or "", + force=getattr(args, "force", False), + skip_confirm=getattr(args, "yes", False), + no_alias=getattr(args, "no_alias", False)) elif action == "inspect": do_inspect(args.identifier) elif action == "list": @@ -1788,7 +1980,7 @@ def skills_command(args) -> None: return do_tap(tap_action, repo=repo) else: - _console.print("Usage: hermes skills [browse|search|install|inspect|list|list-modified|diff|check|update|audit|uninstall|reset|opt-out|opt-in|publish|snapshot|tap]\n") + _console.print("Usage: hermes skills [browse|search|install|install-set|inspect|list|list-modified|diff|check|update|audit|uninstall|reset|opt-out|opt-in|publish|snapshot|tap]\n") _console.print("Run 'hermes skills --help' for details.\n") diff --git a/hermes_cli/subcommands/skills.py b/hermes_cli/subcommands/skills.py index 4eb68a01b1c7a..291a68ff3300a 100644 --- a/hermes_cli/subcommands/skills.py +++ b/hermes_cli/subcommands/skills.py @@ -108,6 +108,38 @@ def build_skills_parser(subparsers, *, cmd_skills: Callable) -> None: help="Skip confirmation prompt (needed in TUI mode)", ) + skills_install_set = skills_subparsers.add_parser( + "install-set", + help="Install a skill set from an AI Catalog or agent-skills discovery index", + description=( + "Install a group of skills published as a set. Accepts an origin " + "(https://example.com — resolved to /.well-known/ai-catalog.json), " + "an AI Catalog JSON URL, or a direct agent-skills index.json URL. " + "Members are digest-verified and security-scanned individually; " + "a / bundle alias is created so the whole set loads in one turn." + ), + ) + skills_install_set.add_argument( + "url", + help="Origin, AI Catalog URL, or agent-skills index.json URL", + ) + skills_install_set.add_argument( + "--set", dest="set_name", default="", + help="Which skill set to install when the catalog has several", + ) + skills_install_set.add_argument( + "--no-alias", action="store_true", + help="Install the skills but skip creating the / bundle alias", + ) + skills_install_set.add_argument( + "--force", action="store_true", + help="Reinstall members that are already installed", + ) + skills_install_set.add_argument( + "--yes", "-y", action="store_true", + help="Skip confirmation prompt", + ) + skills_inspect = skills_subparsers.add_parser( "inspect", help="Preview a skill without installing" ) diff --git a/scripts/publish_skill_set.py b/scripts/publish_skill_set.py new file mode 100644 index 0000000000000..43556ebe61ba4 --- /dev/null +++ b/scripts/publish_skill_set.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Build a publishable skill-set catalog from local skill directories. + +Publisher-side companion to ``hermes skills install-set``. Given one or more +skill directories (each containing a ``SKILL.md``), this produces a static +site tree implementing the agentskills #254 discovery index plus an AI +Catalog (https://ai-catalog.io) wrapper entry: + + out/ + ├── .well-known/ + │ ├── ai-catalog.json # AI Catalog with one skill-set entry + │ └── agent-skills/ + │ ├── index.json # #254 discovery index + │ ├── /SKILL.md # type: skill-md + │ └── .tar.gz # type: archive + └── (serve this directory over HTTPS) + +Skills with only a SKILL.md are published as ``type: "skill-md"``; skills +with supporting files (scripts/, references/, ...) are packed into a +``.tar.gz`` with SKILL.md at the archive root, per the spec. Every artifact +gets a required ``sha256:`` digest. + +Usage: + python scripts/publish_skill_set.py \ + --name "Backend Dev" \ + --command backend-dev \ + --description "Everything for backend feature work." \ + --instruction "Prefer TDD. Run the linter before opening a PR." \ + --host-name "Example Corp" \ + --out ./public \ + path/to/skill-a path/to/skill-b ... + +Then serve ``./public`` at your origin and install with: + hermes skills install-set https://your-origin.example +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import io +import json +import sys +import tarfile +from pathlib import Path + +INDEX_SCHEMA = "https://schemas.agentskills.io/discovery/0.2.0/schema.json" +SKILL_SET_ENTRY_TYPE = "application/agent-skills+json" +HERMES_SET_EXTENSION = "io.hermes.skill-set" + + +def _digest(content: bytes) -> str: + return f"sha256:{hashlib.sha256(content).hexdigest()}" + + +def _frontmatter_field(skill_md: str, field: str) -> str: + """Cheap YAML frontmatter single-field read (name/description).""" + lines = skill_md.splitlines() + if not lines or lines[0].strip() != "---": + return "" + for line in lines[1:]: + if line.strip() == "---": + break + if line.startswith(f"{field}:"): + return line.split(":", 1)[1].strip().strip("\"'") + return "" + + +def _pack_tar_gz(skill_dir: Path) -> bytes: + """Deterministic tar.gz of a skill directory, SKILL.md at archive root.""" + buf = io.BytesIO() + # mtime=0 + sorted names -> byte-stable archives, so digests only change + # when content changes (plays nice with #254's digest-based caching). + with gzip.GzipFile(fileobj=buf, mode="wb", mtime=0) as gz: + with tarfile.open(fileobj=gz, mode="w") as tf: + for path in sorted(skill_dir.rglob("*")): + if not path.is_file() or path.is_symlink(): + continue + rel = path.relative_to(skill_dir).as_posix() + info = tarfile.TarInfo(name=rel) + data = path.read_bytes() + info.size = len(data) + info.mtime = 0 + info.mode = 0o644 + tf.addfile(info, io.BytesIO(data)) + return buf.getvalue() + + +def main() -> int: + ap = argparse.ArgumentParser(description=(__doc__ or "").split("\n", 1)[0]) + ap.add_argument("skill_dirs", nargs="+", type=Path, + help="Skill directories (each must contain SKILL.md)") + ap.add_argument("--name", required=True, help="Skill set display name") + ap.add_argument("--description", default="", help="Skill set description") + ap.add_argument("--command", default="", + help="Suggested load-alias (io.hermes.skill-set extension)") + ap.add_argument("--instruction", default="", + help="Shared instruction preamble (io.hermes.skill-set extension)") + ap.add_argument("--host-name", default="", help="AI Catalog host displayName") + ap.add_argument("--host-id", default="", help="AI Catalog host identifier (e.g. did:web:...)") + ap.add_argument("--out", type=Path, default=Path("public"), help="Output directory") + args = ap.parse_args() + + well_known = args.out / ".well-known" + skills_dir = well_known / "agent-skills" + skills_dir.mkdir(parents=True, exist_ok=True) + + entries = [] + for skill_dir in args.skill_dirs: + skill_md_path = skill_dir / "SKILL.md" + if not skill_md_path.is_file(): + print(f"error: {skill_dir} has no SKILL.md", file=sys.stderr) + return 1 + skill_md = skill_md_path.read_text(encoding="utf-8") + name = _frontmatter_field(skill_md, "name") or skill_dir.name + description = _frontmatter_field(skill_md, "description") + + extra_files = [p for p in skill_dir.rglob("*") + if p.is_file() and p != skill_md_path] + if extra_files: + artifact = _pack_tar_gz(skill_dir) + (skills_dir / f"{name}.tar.gz").write_bytes(artifact) + entries.append({ + "name": name, + "type": "archive", + "description": description, + "url": f"/.well-known/agent-skills/{name}.tar.gz", + "digest": _digest(artifact), + }) + print(f" archive {name} ({len(extra_files) + 1} files, " + f"{len(artifact):,} bytes)") + else: + content = skill_md.encode("utf-8") + dest = skills_dir / name + dest.mkdir(parents=True, exist_ok=True) + (dest / "SKILL.md").write_bytes(content) + entries.append({ + "name": name, + "type": "skill-md", + "description": description, + "url": f"/.well-known/agent-skills/{name}/SKILL.md", + "digest": _digest(content), + }) + print(f" skill-md {name}") + + index = {"$schema": INDEX_SCHEMA, "skills": entries} + (skills_dir / "index.json").write_text( + json.dumps(index, indent=2) + "\n", encoding="utf-8") + + set_entry = { + "identifier": f"urn:air:{args.host_id or 'example'}:skill-set:" + f"{args.command or args.name.lower().replace(' ', '-')}", + "displayName": args.name, + "description": args.description, + "type": SKILL_SET_ENTRY_TYPE, + "url": "/.well-known/agent-skills/index.json", + } + ext: dict = {} + if args.command: + ext["command"] = args.command + if args.instruction: + ext["instruction"] = args.instruction + if ext: + set_entry["extensions"] = {HERMES_SET_EXTENSION: ext} + + catalog = { + "specVersion": "1.0", + "host": { + "displayName": args.host_name or args.name, + **({"identifier": args.host_id} if args.host_id else {}), + }, + "entries": [set_entry], + } + (well_known / "ai-catalog.json").write_text( + json.dumps(catalog, indent=2) + "\n", encoding="utf-8") + + print(f"\nWrote {args.out}/.well-known/ai-catalog.json " + f"and agent-skills/index.json ({len(entries)} skills).") + print("Serve the output directory at your origin, then:") + print(" hermes skills install-set https://") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/tools/test_skill_set_catalog.py b/tests/tools/test_skill_set_catalog.py new file mode 100644 index 0000000000000..16526d6aa7d93 --- /dev/null +++ b/tests/tools/test_skill_set_catalog.py @@ -0,0 +1,458 @@ +"""Tests for tools/skill_set_catalog.py — the #254 + AI Catalog skill-set client. + +Unit tests cover schema gating, digest verification, archive safety, and +catalog/index parsing with mocked HTTP. The E2E test at the bottom runs a +real local HTTP server built by scripts/publish_skill_set.py and exercises +discover -> resolve -> fetch against actual bytes on the wire. +""" + +import gzip +import hashlib +import io +import json +import tarfile +import zipfile +from unittest.mock import patch + +import pytest + +from tools.skill_set_catalog import ( + ArchiveSafetyError, + DigestError, + SchemaError, + SkillSetError, + SkillSetInfo, + catalog_url_for, + compute_digest, + discover_skill_sets, + fetch_member, + resolve_bare_index, + resolve_skill_set, + verify_digest, + SkillSetMember, + KNOWN_INDEX_SCHEMAS, + HERMES_SET_EXTENSION, + SKILL_SET_ENTRY_TYPE, +) + +SCHEMA = next(iter(KNOWN_INDEX_SCHEMAS)) +BASE = "https://skills.example.com" +INDEX_URL = f"{BASE}/.well-known/agent-skills/index.json" +CATALOG_URL = f"{BASE}/.well-known/ai-catalog.json" + +SKILL_MD = ( + "---\nname: code-review\ndescription: Review code.\n---\n\n# Code Review Skill\n" +) + + +def _tar_gz(files: dict) -> bytes: + buf = io.BytesIO() + with gzip.GzipFile(fileobj=buf, mode="wb", mtime=0) as gz: + with tarfile.open(fileobj=gz, mode="w") as tf: + for name, data in files.items(): + raw = data.encode() if isinstance(data, str) else data + info = tarfile.TarInfo(name=name) + info.size = len(raw) + tf.addfile(info, io.BytesIO(raw)) + return buf.getvalue() + + +def _zip(files: dict) -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + for name, data in files.items(): + zf.writestr(name, data) + return buf.getvalue() + + +def _serve(pages: dict): + """Patch the module HTTP layer with a URL -> bytes dict.""" + def fake_get(url, *, timeout=30): + val = pages.get(url) + if val is None: + return None + return val.encode() if isinstance(val, str) else val + return patch("tools.skill_set_catalog._http_get_bytes", side_effect=fake_get) + + +# --------------------------------------------------------------------------- +# Digest verification +# --------------------------------------------------------------------------- + +class TestDigest: + def test_roundtrip(self): + content = b"hello skills" + verify_digest(content, compute_digest(content)) # no raise + + def test_mismatch_rejected(self): + with pytest.raises(DigestError, match="mismatch"): + verify_digest(b"tampered", compute_digest(b"original")) + + def test_missing_digest_rejected(self): + with pytest.raises(DigestError, match="missing digest"): + verify_digest(b"x", "") + + @pytest.mark.parametrize("bad", [ + "sha256:short", + "sha256:" + "G" * 64, # non-hex + "sha256:" + "A" * 64, # uppercase — spec says lowercase + "md5:" + "a" * 32, + "a" * 64, # bare hex without prefix + ]) + def test_malformed_digest_rejected(self, bad): + with pytest.raises(DigestError, match="malformed"): + verify_digest(b"x", bad) + + +# --------------------------------------------------------------------------- +# AI Catalog discovery +# --------------------------------------------------------------------------- + +def _catalog(entries) -> str: + return json.dumps({ + "specVersion": "1.0", + "host": {"displayName": "Example"}, + "entries": entries, + }) + + +class TestCatalogDiscovery: + def test_catalog_url_for_origin(self): + assert catalog_url_for("https://x.example") == \ + "https://x.example/.well-known/ai-catalog.json" + assert catalog_url_for("https://x.example/custom/cat.json") == \ + "https://x.example/custom/cat.json" + + def test_finds_skill_set_entries_with_extension(self): + pages = {CATALOG_URL: _catalog([ + { + "identifier": "urn:air:example:skill-set:backend", + "displayName": "Backend Dev", + "description": "Backend feature work.", + "type": SKILL_SET_ENTRY_TYPE, + "url": "/.well-known/agent-skills/index.json", + "extensions": {HERMES_SET_EXTENSION: { + "command": "backend-dev", + "instruction": "Prefer TDD.", + }}, + }, + {"identifier": "urn:air:example:mcp:weather", + "type": "application/mcp-server-card+json", + "url": "https://api.example.com/mcp"}, + ])} + with _serve(pages): + sets = discover_skill_sets(CATALOG_URL) + assert len(sets) == 1 + s = sets[0] + assert s.name == "Backend Dev" + assert s.index_url == INDEX_URL + assert s.command == "backend-dev" + assert s.instruction == "Prefer TDD." + + def test_entry_without_extension_still_discovered(self): + pages = {CATALOG_URL: _catalog([ + {"displayName": "Plain Set", "type": SKILL_SET_ENTRY_TYPE, + "url": "/.well-known/agent-skills/index.json"}, + ])} + with _serve(pages): + sets = discover_skill_sets(CATALOG_URL) + assert len(sets) == 1 + assert sets[0].command == "" + assert sets[0].instruction == "" + + def test_follows_sub_catalog_one_level(self): + sub_url = f"{BASE}/catalogs/eng.json" + pages = { + CATALOG_URL: _catalog([ + {"displayName": "Engineering", "type": "application/ai-catalog+json", + "url": "/catalogs/eng.json"}, + ]), + sub_url: _catalog([ + {"displayName": "Backend Dev", "type": SKILL_SET_ENTRY_TYPE, + "url": "/.well-known/agent-skills/index.json"}, + ]), + } + with _serve(pages): + sets = discover_skill_sets(CATALOG_URL) + assert [s.name for s in sets] == ["Backend Dev"] + assert sets[0].index_url == INDEX_URL + + def test_not_a_catalog_raises(self): + with _serve({CATALOG_URL: json.dumps({"skills": []})}): + with pytest.raises(SkillSetError, match="not an AI Catalog"): + discover_skill_sets(CATALOG_URL) + + def test_unreachable_catalog_raises(self): + with _serve({}): + with pytest.raises(SkillSetError, match="Could not fetch"): + discover_skill_sets(CATALOG_URL) + + +# --------------------------------------------------------------------------- +# #254 index resolution +# --------------------------------------------------------------------------- + +def _index(skills, schema: "str | None" = SCHEMA) -> str: + payload = {"skills": skills} + if schema is not None: + payload["$schema"] = schema + return json.dumps(payload) + + +def _info() -> SkillSetInfo: + return SkillSetInfo(name="Test Set", description="", index_url=INDEX_URL) + + +class TestIndexResolution: + def test_members_parsed_and_urls_resolved(self): + pages = {INDEX_URL: _index([ + {"name": "code-review", "type": "skill-md", + "description": "Review code.", + "url": "code-review/SKILL.md", "digest": compute_digest(b"x")}, + {"name": "wrangler", "type": "archive", + "description": "Deploy workers.", + "url": "/.well-known/agent-skills/wrangler.tar.gz", + "digest": compute_digest(b"y")}, + ])} + with _serve(pages): + resolved = resolve_skill_set(_info()) + assert [m.name for m in resolved.members] == ["code-review", "wrangler"] + # Relative resolved against index directory; path-absolute against origin. + assert resolved.members[0].url == \ + f"{BASE}/.well-known/agent-skills/code-review/SKILL.md" + assert resolved.members[1].url == \ + f"{BASE}/.well-known/agent-skills/wrangler.tar.gz" + + def test_unknown_schema_refused(self): + pages = {INDEX_URL: _index([], schema="https://example.com/other/1.0.json")} + with _serve(pages): + with pytest.raises(SchemaError, match="Unrecognized index"): + resolve_skill_set(_info()) + + def test_absent_schema_refused(self): + pages = {INDEX_URL: _index([], schema=None)} + with _serve(pages): + with pytest.raises(SchemaError): + resolve_skill_set(_info()) + + def test_unrecognized_type_skipped_with_warning(self): + pages = {INDEX_URL: _index([ + {"name": "good", "type": "skill-md", "url": "good/SKILL.md", + "digest": compute_digest(b"x")}, + {"name": "weird", "type": "oci-image", "url": "weird.oci", + "digest": compute_digest(b"y")}, + ])} + with _serve(pages): + resolved = resolve_skill_set(_info()) + assert [m.name for m in resolved.members] == ["good"] + assert any("weird" in s for s in resolved.skipped) + + def test_bare_index_fallback_name(self): + pages = {INDEX_URL: _index([])} + with _serve(pages): + resolved = resolve_bare_index(INDEX_URL) + assert resolved.info.name == "skills.example.com" + + +# --------------------------------------------------------------------------- +# Member fetching — digest + archive safety +# --------------------------------------------------------------------------- + +def _member(name="code-review", mtype="skill-md", url=None, digest=""): + return SkillSetMember( + name=name, description="", type=mtype, + url=url or f"{BASE}/.well-known/agent-skills/{name}/SKILL.md", + digest=digest, + ) + + +class TestFetchMember: + def test_skill_md_happy_path(self): + content = SKILL_MD.encode() + m = _member(digest=compute_digest(content)) + with _serve({m.url: content}): + bundle = fetch_member(m, set_info=_info()) + assert bundle.name == "code-review" + assert bundle.files == {"SKILL.md": SKILL_MD} + assert bundle.source == "skill-set" + assert bundle.metadata["digest"] == m.digest + + def test_tampered_content_rejected(self): + m = _member(digest=compute_digest(SKILL_MD.encode())) + with _serve({m.url: b"---\nname: evil\n---\nrm -rf /"}): + with pytest.raises(DigestError, match="mismatch"): + fetch_member(m, set_info=_info()) + + def test_archive_happy_path(self): + artifact = _tar_gz({ + "SKILL.md": SKILL_MD, + "scripts/deploy.sh": "#!/bin/sh\necho hi\n", + "references/API.md": "# API\n", + }) + url = f"{BASE}/.well-known/agent-skills/wrangler.tar.gz" + m = _member("wrangler", "archive", url, compute_digest(artifact)) + with _serve({url: artifact}): + bundle = fetch_member(m, set_info=_info()) + assert set(bundle.files) == {"SKILL.md", "scripts/deploy.sh", + "references/API.md"} + + def test_zip_happy_path(self): + artifact = _zip({"SKILL.md": SKILL_MD, "references/NOTES.md": "notes"}) + url = f"{BASE}/.well-known/agent-skills/z.zip" + m = _member("zskill", "archive", url, compute_digest(artifact)) + with _serve({url: artifact}): + bundle = fetch_member(m, set_info=_info()) + assert set(bundle.files) == {"SKILL.md", "references/NOTES.md"} + + def test_archive_without_root_skill_md_rejected(self): + artifact = _tar_gz({"nested/SKILL.md": SKILL_MD}) + url = f"{BASE}/.well-known/agent-skills/bad.tar.gz" + m = _member("bad", "archive", url, compute_digest(artifact)) + with _serve({url: artifact}): + with pytest.raises(ArchiveSafetyError, match="no SKILL.md at its root"): + fetch_member(m, set_info=_info()) + + def test_path_traversal_rejected(self): + artifact = _tar_gz({"SKILL.md": SKILL_MD, "../../evil.sh": "boom"}) + url = f"{BASE}/.well-known/agent-skills/trav.tar.gz" + m = _member("trav", "archive", url, compute_digest(artifact)) + with _serve({url: artifact}): + with pytest.raises(ArchiveSafetyError): + fetch_member(m, set_info=_info()) + + def test_absolute_path_rejected(self): + artifact = _tar_gz({"SKILL.md": SKILL_MD, "/etc/cron.d/evil": "boom"}) + url = f"{BASE}/.well-known/agent-skills/abs.tar.gz" + m = _member("abs", "archive", url, compute_digest(artifact)) + with _serve({url: artifact}): + with pytest.raises(ArchiveSafetyError): + fetch_member(m, set_info=_info()) + + def test_symlink_member_rejected(self): + buf = io.BytesIO() + with gzip.GzipFile(fileobj=buf, mode="wb", mtime=0) as gz: + with tarfile.open(fileobj=gz, mode="w") as tf: + data = SKILL_MD.encode() + info = tarfile.TarInfo(name="SKILL.md") + info.size = len(data) + tf.addfile(info, io.BytesIO(data)) + link = tarfile.TarInfo(name="creds") + link.type = tarfile.SYMTYPE + link.linkname = "/home/user/.ssh/id_rsa" + tf.addfile(link) + artifact = buf.getvalue() + url = f"{BASE}/.well-known/agent-skills/lnk.tar.gz" + m = _member("lnk", "archive", url, compute_digest(artifact)) + with _serve({url: artifact}): + with pytest.raises(ArchiveSafetyError, match="link member"): + fetch_member(m, set_info=_info()) + + def test_decompression_bomb_rejected(self): + # 20MB of zeros compresses tiny but exceeds the per-member cap. + artifact = _tar_gz({"SKILL.md": SKILL_MD, "big.bin": b"\0" * (6 * 1024 * 1024)}) + url = f"{BASE}/.well-known/agent-skills/bomb.tar.gz" + m = _member("bomb", "archive", url, compute_digest(artifact)) + with _serve({url: artifact}): + with pytest.raises(ArchiveSafetyError): + fetch_member(m, set_info=_info()) + + def test_unsupported_archive_extension_rejected(self): + url = f"{BASE}/.well-known/agent-skills/skill.rar" + m = _member("rarred", "archive", url, compute_digest(b"data")) + with _serve({url: b"data"}): + with pytest.raises(ArchiveSafetyError, match="unsupported archive format"): + fetch_member(m, set_info=_info()) + + def test_unsafe_skill_name_rejected(self): + m = _member("../escape", digest=compute_digest(b"x")) + with _serve({m.url: b"x"}): + with pytest.raises(ValueError): + fetch_member(m, set_info=_info()) + + +# --------------------------------------------------------------------------- +# E2E: publisher script -> real HTTP server -> full client flow +# --------------------------------------------------------------------------- + +class TestEndToEnd: + @pytest.fixture() + def published_site(self, tmp_path): + """Build a real publisher tree with scripts/publish_skill_set.py.""" + import subprocess + import sys as _sys + from pathlib import Path as _P + + repo_root = _P(__file__).resolve().parents[2] + script = repo_root / "scripts" / "publish_skill_set.py" + + # One single-file skill, one multi-file skill. + s1 = tmp_path / "src" / "code-review" + s1.mkdir(parents=True) + (s1 / "SKILL.md").write_text(SKILL_MD) + s2 = tmp_path / "src" / "deploy-tool" + (s2 / "scripts").mkdir(parents=True) + (s2 / "SKILL.md").write_text( + "---\nname: deploy-tool\ndescription: Deploy things.\n---\n\n# Deploy\n") + (s2 / "scripts" / "run.sh").write_text("#!/bin/sh\necho deploy\n") + + out = tmp_path / "public" + subprocess.run( + [_sys.executable, str(script), + "--name", "Backend Dev", "--command", "backend-dev", + "--description", "Backend feature work.", + "--instruction", "Prefer TDD.", + "--out", str(out), str(s1), str(s2)], + check=True, capture_output=True, text=True, + ) + return out + + def test_full_flow_over_real_http(self, published_site): + import http.server + import threading + + handler = type("H", (http.server.SimpleHTTPRequestHandler,), { + "directory": str(published_site), + "log_message": lambda self, *a: None, + }) + httpd = http.server.ThreadingHTTPServer( + ("127.0.0.1", 0), + lambda *a, **kw: handler(*a, directory=str(published_site), **kw), + ) + port = httpd.server_address[1] + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + try: + origin = f"http://127.0.0.1:{port}" + + # Bypass the SSRF guard for the loopback test server only. + def local_get(url, *, timeout=30): + import httpx + resp = httpx.get(url, timeout=timeout) + return resp.content if resp.status_code == 200 else None + + with patch("tools.skill_set_catalog._http_get_bytes", + side_effect=local_get): + sets = discover_skill_sets(catalog_url_for(origin)) + assert len(sets) == 1 + info = sets[0] + assert info.name == "Backend Dev" + assert info.command == "backend-dev" + assert info.instruction == "Prefer TDD." + + resolved = resolve_skill_set(info) + assert {m.name for m in resolved.members} == \ + {"code-review", "deploy-tool"} + types = {m.name: m.type for m in resolved.members} + assert types["code-review"] == "skill-md" + assert types["deploy-tool"] == "archive" + + bundles = {m.name: fetch_member(m, set_info=info) + for m in resolved.members} + assert bundles["code-review"].files["SKILL.md"] == SKILL_MD + assert "scripts/run.sh" in bundles["deploy-tool"].files + # Digest verification ran on real bytes for every member. + for b in bundles.values(): + assert b.metadata["digest"].startswith("sha256:") + finally: + httpd.shutdown() + httpd.server_close() diff --git a/tools/skill_set_catalog.py b/tools/skill_set_catalog.py new file mode 100644 index 0000000000000..74ebad89b6f89 --- /dev/null +++ b/tools/skill_set_catalog.py @@ -0,0 +1,504 @@ +"""Skill-set catalog client — prototype for the agentskills #254 + AI Catalog layering. + +Implements the design discussed with agentskills.io: + +1. **AI Catalog** (https://ai-catalog.io) — a typed discovery index served at + ``/.well-known/ai-catalog.json``. Entries with + ``type: "application/agent-skills+json"`` point at an agent-skills + discovery index and represent an installable *skill set*. + +2. **agentskills PR #254 discovery index** — the index the entry points at: + ``{"$schema": ..., "skills": [{name, description, type, url, digest}]}`` + where ``type`` is ``"skill-md"`` (single file) or ``"archive"`` + (``.tar.gz`` / ``.zip`` with SKILL.md at the archive root). + +3. **``io.hermes.skill-set`` extension** — optional namespaced metadata on + the AI Catalog entry carrying set-level usage intent:: + + "extensions": { + "io.hermes.skill-set": { + "command": "backend-dev", + "instruction": "Prefer TDD. Run the linter before opening a PR." + } + } + + ``command`` is the suggested local load-alias (Hermes creates a skill + bundle so ``/backend-dev`` loads every member in one turn); + ``instruction`` is a shared preamble injected above the member skills. + Clients that don't recognize the extension still install the correct + set — they only miss the alias/preamble sugar. + +Security posture matches the rest of the skills hub: all HTTP goes through +the SSRF-guarded fetcher, member artifacts are digest-verified (SHA-256, +required by #254), archive extraction rejects path traversal / absolute +paths / symlinks / hardlinks and caps decompressed size, and every fetched +skill flows through the existing quarantine -> scan -> install pipeline. + +This module only *fetches and validates*; installation is orchestrated by +``hermes_cli.skills_hub.do_install_set``. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import logging +import posixpath +import re +import tarfile +import zipfile +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Union +from urllib.parse import urljoin, urlparse + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +#: MIME-ish AI Catalog entry type identifying an agent-skills discovery index. +SKILL_SET_ENTRY_TYPE = "application/agent-skills+json" + +#: Nested AI Catalog entry type (sub-catalogs) — followed one level deep. +CATALOG_ENTRY_TYPE = "application/ai-catalog+json" + +#: Namespaced AI Catalog extension carrying set-level usage intent. +HERMES_SET_EXTENSION = "io.hermes.skill-set" + +#: ``$schema`` URIs this client knows how to process (agentskills #254). +KNOWN_INDEX_SCHEMAS = frozenset({ + "https://schemas.agentskills.io/discovery/0.2.0/schema.json", +}) + +#: Well-known path for AI Catalog discovery. +AI_CATALOG_WELL_KNOWN = "/.well-known/ai-catalog.json" + +_DIGEST_RE = re.compile(r"^sha256:([0-9a-f]{64})$") + +# Archive safety caps (decompression-bomb guard per #254 "Archive safety"). +MAX_ARCHIVE_BYTES = 20 * 1024 * 1024 # compressed artifact cap +MAX_UNPACKED_BYTES = 50 * 1024 * 1024 # total decompressed cap +MAX_MEMBER_BYTES = 5 * 1024 * 1024 # single decompressed file cap +MAX_ARCHIVE_MEMBERS = 500 + +#: Max skills a single set may enumerate (sanity cap for the prototype). +MAX_SET_MEMBERS = 50 + + +# --------------------------------------------------------------------------- +# Errors +# --------------------------------------------------------------------------- + +class SkillSetError(Exception): + """Base error for skill-set catalog operations.""" + + +class SchemaError(SkillSetError): + """Unrecognized or missing ``$schema`` — per #254, warn and stop.""" + + +class DigestError(SkillSetError): + """Artifact digest missing, malformed, or mismatched.""" + + +class ArchiveSafetyError(SkillSetError): + """Archive violated safety rules (traversal, links, bomb, structure).""" + + +# --------------------------------------------------------------------------- +# Data models +# --------------------------------------------------------------------------- + +@dataclass +class SkillSetInfo: + """A skill set discovered in an AI Catalog (or a bare #254 index).""" + name: str # display name (catalog displayName or derived) + description: str + index_url: str # resolved URL of the #254 index.json + identifier: str = "" # catalog entry identifier (urn:...), if any + command: str = "" # io.hermes.skill-set suggested alias + instruction: str = "" # io.hermes.skill-set shared preamble + catalog_url: str = "" # catalog this was discovered from, if any + + +@dataclass +class SkillSetMember: + """One skill entry from a #254 discovery index.""" + name: str + description: str + type: str # "skill-md" | "archive" + url: str # resolved absolute URL + digest: str # "sha256:" + + +@dataclass +class ResolvedSkillSet: + """A fully parsed set: info + members, ready to fetch.""" + info: SkillSetInfo + members: List[SkillSetMember] = field(default_factory=list) + skipped: List[str] = field(default_factory=list) # entries skipped w/ reason + + +# --------------------------------------------------------------------------- +# HTTP (indirection point — tests monkeypatch _http_get_bytes) +# --------------------------------------------------------------------------- + +def _http_get_bytes(url: str, *, timeout: int = 30) -> Optional[bytes]: + """SSRF-guarded GET returning raw bytes, or None on any failure.""" + from tools.skills_hub import _guarded_http_get + resp = _guarded_http_get(url, timeout=timeout) + if resp is None or resp.status_code != 200: + return None + return resp.content + + +def _http_get_json(url: str, *, timeout: int = 20) -> Optional[Any]: + raw = _http_get_bytes(url, timeout=timeout) + if raw is None: + return None + try: + return json.loads(raw.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + logger.warning("Skill-set fetch: invalid JSON at %s", url) + return None + + +# --------------------------------------------------------------------------- +# Digest verification +# --------------------------------------------------------------------------- + +def verify_digest(content: bytes, digest: str, *, what: str = "artifact") -> None: + """Verify ``content`` against a ``sha256:`` digest string. + + Raises :class:`DigestError` on missing/malformed/mismatched digests — + #254 makes the digest required and forbids using unverified content. + """ + if not isinstance(digest, str) or not digest: + raise DigestError(f"{what}: missing digest (required by the discovery spec)") + m = _DIGEST_RE.match(digest.strip()) + if not m: + raise DigestError( + f"{what}: malformed digest {digest!r} (expected sha256:<64 lowercase hex>)" + ) + actual = hashlib.sha256(content).hexdigest() + if actual != m.group(1): + raise DigestError( + f"{what}: digest mismatch — index says sha256:{m.group(1)}, " + f"downloaded content is sha256:{actual}. Refusing to use it." + ) + + +def compute_digest(content: bytes) -> str: + """Format a #254 digest string for raw bytes (publisher-side helper).""" + return f"sha256:{hashlib.sha256(content).hexdigest()}" + + +# --------------------------------------------------------------------------- +# AI Catalog parsing +# --------------------------------------------------------------------------- + +def catalog_url_for(origin_or_url: str) -> str: + """Normalize user input to an AI Catalog URL. + + ``https://example.com`` -> ``https://example.com/.well-known/ai-catalog.json``; + anything already ending in ``.json`` is returned as-is. + """ + url = origin_or_url.strip().rstrip("/") + if url.endswith(".json"): + return origin_or_url.strip() + return url + AI_CATALOG_WELL_KNOWN + + +def _entry_extensions(entry: Dict[str, Any]) -> Dict[str, Any]: + ext = entry.get("extensions") + return ext if isinstance(ext, dict) else {} + + +def discover_skill_sets(catalog_url: str, *, _depth: int = 0) -> List[SkillSetInfo]: + """Fetch an AI Catalog and return every skill-set entry in it. + + Follows ``application/ai-catalog+json`` sub-catalog entries one level + deep (AI Catalog is nestable; the prototype bounds recursion at 1). + """ + data = _http_get_json(catalog_url) + if not isinstance(data, dict): + raise SkillSetError(f"Could not fetch AI Catalog at {catalog_url}") + + entries = data.get("entries") + if not isinstance(entries, list): + raise SkillSetError( + f"{catalog_url} is not an AI Catalog (no 'entries' array)" + ) + + sets: List[SkillSetInfo] = [] + for entry in entries: + if not isinstance(entry, dict): + continue + etype = entry.get("type") + url = entry.get("url") + if not isinstance(url, str) or not url: + continue + resolved = urljoin(catalog_url, url) + + if etype == SKILL_SET_ENTRY_TYPE: + hermes_ext = _entry_extensions(entry).get(HERMES_SET_EXTENSION) + hermes_ext = hermes_ext if isinstance(hermes_ext, dict) else {} + sets.append(SkillSetInfo( + name=str(entry.get("displayName") or entry.get("identifier") or resolved), + description=str(entry.get("description") or ""), + index_url=resolved, + identifier=str(entry.get("identifier") or ""), + command=str(hermes_ext.get("command") or ""), + instruction=str(hermes_ext.get("instruction") or ""), + catalog_url=catalog_url, + )) + elif etype == CATALOG_ENTRY_TYPE and _depth < 1: + try: + sets.extend(discover_skill_sets(resolved, _depth=_depth + 1)) + except SkillSetError as exc: + logger.warning("Skipping unreachable sub-catalog %s: %s", resolved, exc) + + return sets + + +# --------------------------------------------------------------------------- +# #254 discovery index parsing +# --------------------------------------------------------------------------- + +def resolve_skill_set(info: SkillSetInfo) -> ResolvedSkillSet: + """Fetch and validate the #254 index a :class:`SkillSetInfo` points at.""" + data = _http_get_json(info.index_url) + if not isinstance(data, dict): + raise SkillSetError(f"Could not fetch skill index at {info.index_url}") + + schema = data.get("$schema") + if not isinstance(schema, str) or schema not in KNOWN_INDEX_SCHEMAS: + # #254: "Clients encountering an unrecognized or absent $schema + # should warn the user and should not process the index." + raise SchemaError( + f"Unrecognized index $schema {schema!r} at {info.index_url}. " + f"Known: {', '.join(sorted(KNOWN_INDEX_SCHEMAS))}" + ) + + raw_skills = data.get("skills") + if not isinstance(raw_skills, list): + raise SkillSetError(f"Index at {info.index_url} has no 'skills' array") + if len(raw_skills) > MAX_SET_MEMBERS: + raise SkillSetError( + f"Index enumerates {len(raw_skills)} skills (cap: {MAX_SET_MEMBERS})" + ) + + members: List[SkillSetMember] = [] + skipped: List[str] = [] + for entry in raw_skills: + if not isinstance(entry, dict): + skipped.append("(non-object entry)") + continue + name = entry.get("name") + etype = entry.get("type") + url = entry.get("url") + digest = entry.get("digest") + if not isinstance(name, str) or not name: + skipped.append("(entry without a name)") + continue + if etype not in ("skill-md", "archive"): + # #254: skip entries with an unrecognized type and warn. + skipped.append(f"{name} (unrecognized type {etype!r})") + continue + if not isinstance(url, str) or not url: + skipped.append(f"{name} (missing url)") + continue + members.append(SkillSetMember( + name=name, + description=str(entry.get("description") or ""), + type=etype, + url=urljoin(info.index_url, url), + digest=str(digest or ""), + )) + + return ResolvedSkillSet(info=info, members=members, skipped=skipped) + + +def resolve_bare_index(index_url: str, *, name: str = "") -> ResolvedSkillSet: + """Treat a raw #254 index URL as a set (no AI Catalog wrapper). + + Used when the user hands us an index.json directly. The set name falls + back to the host name; no extension metadata is available on this path. + """ + host = urlparse(index_url).netloc or index_url + info = SkillSetInfo( + name=name or host, + description=f"Skill set from {host}", + index_url=index_url, + ) + return resolve_skill_set(info) + + +# --------------------------------------------------------------------------- +# Member fetching (skill-md + archive) -> SkillBundle +# --------------------------------------------------------------------------- + +def _safe_member_path(raw_name: str) -> str: + """Validate an archive member path per #254 archive-safety rules.""" + name = raw_name.replace("\\", "/") + if name.startswith("/") or re.match(r"^[A-Za-z]:", name): + raise ArchiveSafetyError(f"absolute path in archive: {raw_name!r}") + normalized = posixpath.normpath(name) + if normalized.startswith("..") or "/../" in f"/{normalized}/": + raise ArchiveSafetyError(f"path traversal in archive: {raw_name!r}") + if normalized in (".", ""): + raise ArchiveSafetyError(f"empty archive member path: {raw_name!r}") + # Reuse the hub-wide validator for charset / depth constraints. + from tools.skills_hub import _validate_bundle_rel_path + return _validate_bundle_rel_path(normalized) + + +def _decode_member(raw: bytes) -> Union[str, bytes]: + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + return raw + + +def _extract_tar_gz(content: bytes) -> Dict[str, Union[str, bytes]]: + files: Dict[str, Union[str, bytes]] = {} + total = 0 + try: + tf = tarfile.open(fileobj=io.BytesIO(content), mode="r:gz") + except tarfile.TarError as exc: + raise ArchiveSafetyError(f"invalid .tar.gz archive: {exc}") from exc + with tf: + members = tf.getmembers() + if len(members) > MAX_ARCHIVE_MEMBERS: + raise ArchiveSafetyError( + f"archive has {len(members)} members (cap: {MAX_ARCHIVE_MEMBERS})" + ) + for m in members: + if m.issym() or m.islnk(): + # #254: reject symlinks/hardlinks outright (resolving them + # safely is not worth it for skill payloads). + raise ArchiveSafetyError(f"link member in archive: {m.name!r}") + if not m.isfile(): + continue + safe = _safe_member_path(m.name) + if m.size > MAX_MEMBER_BYTES: + raise ArchiveSafetyError( + f"archive member {safe} is {m.size} bytes (cap: {MAX_MEMBER_BYTES})" + ) + total += m.size + if total > MAX_UNPACKED_BYTES: + raise ArchiveSafetyError("archive exceeds decompressed size cap") + fh = tf.extractfile(m) + if fh is None: + continue + raw = fh.read(MAX_MEMBER_BYTES + 1) + if len(raw) > MAX_MEMBER_BYTES: + raise ArchiveSafetyError(f"archive member {safe} exceeded size cap") + files[safe] = _decode_member(raw) + return files + + +def _extract_zip(content: bytes) -> Dict[str, Union[str, bytes]]: + files: Dict[str, Union[str, bytes]] = {} + total = 0 + try: + zf = zipfile.ZipFile(io.BytesIO(content)) + except zipfile.BadZipFile as exc: + raise ArchiveSafetyError(f"invalid .zip archive: {exc}") from exc + with zf: + infos = zf.infolist() + if len(infos) > MAX_ARCHIVE_MEMBERS: + raise ArchiveSafetyError( + f"archive has {len(infos)} members (cap: {MAX_ARCHIVE_MEMBERS})" + ) + for zinfo in infos: + if zinfo.is_dir(): + continue + # Zip symlinks encode the link mode in external_attr's high bits. + if (zinfo.external_attr >> 16) & 0o170000 == 0o120000: + raise ArchiveSafetyError(f"symlink member in archive: {zinfo.filename!r}") + safe = _safe_member_path(zinfo.filename) + if zinfo.file_size > MAX_MEMBER_BYTES: + raise ArchiveSafetyError( + f"archive member {safe} is {zinfo.file_size} bytes " + f"(cap: {MAX_MEMBER_BYTES})" + ) + total += zinfo.file_size + if total > MAX_UNPACKED_BYTES: + raise ArchiveSafetyError("archive exceeds decompressed size cap") + with zf.open(zinfo) as fh: + raw = fh.read(MAX_MEMBER_BYTES + 1) + if len(raw) > MAX_MEMBER_BYTES: + raise ArchiveSafetyError(f"archive member {safe} exceeded size cap") + files[safe] = _decode_member(raw) + return files + + +def _archive_format(url: str) -> str: + path = urlparse(url).path.lower() + if path.endswith((".tar.gz", ".tgz")): + return "tar.gz" + if path.endswith(".zip"): + return "zip" + raise ArchiveSafetyError( + f"unsupported archive format for {url} (expected .tar.gz or .zip)" + ) + + +def fetch_member(member: SkillSetMember, *, set_info: SkillSetInfo): + """Download, digest-verify, and unpack one set member. + + Returns a ``tools.skills_hub.SkillBundle`` ready for the standard + quarantine -> scan -> install pipeline. + """ + from tools.skills_hub import SkillBundle, _validate_skill_name + + skill_name = _validate_skill_name(member.name) + + content = _http_get_bytes(member.url) + if content is None: + raise SkillSetError(f"{member.name}: could not download {member.url}") + if len(content) > MAX_ARCHIVE_BYTES: + raise ArchiveSafetyError( + f"{member.name}: artifact is {len(content)} bytes (cap: {MAX_ARCHIVE_BYTES})" + ) + + verify_digest(content, member.digest, what=member.name) + + if member.type == "skill-md": + try: + files: Dict[str, Union[str, bytes]] = { + "SKILL.md": content.decode("utf-8") + } + except UnicodeDecodeError as exc: + raise SkillSetError(f"{member.name}: SKILL.md is not valid UTF-8") from exc + else: + files = ( + _extract_tar_gz(content) + if _archive_format(member.url) == "tar.gz" + else _extract_zip(content) + ) + if "SKILL.md" not in files: + # #254: "The archive must contain SKILL.md at the root." + raise ArchiveSafetyError( + f"{member.name}: archive has no SKILL.md at its root" + ) + + return SkillBundle( + name=skill_name, + files=files, + source="skill-set", + identifier=f"skill-set:{set_info.index_url}#{skill_name}", + trust_level="community", + metadata={ + "set_name": set_info.name, + "index_url": set_info.index_url, + "catalog_url": set_info.catalog_url, + "artifact_url": member.url, + "artifact_type": member.type, + "digest": member.digest, + "source_url": member.url, + }, + )