feat: install skill sets from AI Catalog + agentskills discovery indexes
Prototype of the skill-set layering discussed with agentskills.io: - AI Catalog (/.well-known/ai-catalog.json) entries typed application/agent-skills+json point at an agentskills PR #254 discovery index and represent an installable skill set. - The optional io.hermes.skill-set extension carries set-level usage intent: a suggested load-alias command and a shared instruction preamble. Clients that ignore the extension still install the correct set. - tools/skill_set_catalog.py implements the client: $schema gating, required sha256 digest verification, skill-md + archive (.tar.gz/.zip) artifacts, and #254 archive-safety rules (traversal/absolute-path/ link rejection, decompression caps). - hermes skills install-set <url> installs every member through the existing quarantine -> scan -> install pipeline, then creates the /<name> skill bundle so the whole set loads in one turn. - scripts/publish_skill_set.py is the publisher-side counterpart: builds the static .well-known tree (catalog + index + artifacts) from local skill directories with byte-stable archives.
This commit is contained in:
Teknium
parent
a6ede70c2a
commit
e34a29be56
|
|
@ -793,6 +793,193 @@ def do_install(identifier: str, category: str = "", force: bool = False,
|
|||
c.print("[dim]Use /reset to start a new session now, or --now to activate immediately (invalidates prompt cache).[/]\n")
|
||||
|
||||
|
||||
def do_install_set(url: str, *, set_name: str = "", force: bool = False,
|
||||
skip_confirm: bool = False, no_alias: bool = False,
|
||||
console: Optional[Console] = None) -> None:
|
||||
"""Install a skill *set* from an AI Catalog or a #254 discovery index.
|
||||
|
||||
``url`` may be:
|
||||
- an origin (``https://example.com``) — resolved to
|
||||
``/.well-known/ai-catalog.json``
|
||||
- an AI Catalog JSON URL — skill-set entries
|
||||
(``application/agent-skills+json``) are listed / selected
|
||||
- a #254 ``index.json`` URL — treated as a single anonymous set
|
||||
|
||||
Every member flows through the standard fetch -> digest-verify ->
|
||||
quarantine -> scan -> install pipeline. After installing, a local skill
|
||||
bundle (load-alias) is created from the entry's ``io.hermes.skill-set``
|
||||
extension unless ``no_alias`` is set.
|
||||
"""
|
||||
from tools.skill_set_catalog import (
|
||||
SkillSetError, catalog_url_for, discover_skill_sets, fetch_member,
|
||||
resolve_bare_index, resolve_skill_set,
|
||||
)
|
||||
from tools.skills_hub import (
|
||||
HUB_DIR, HubLockFile, append_audit_log, ensure_hub_dirs,
|
||||
install_from_quarantine, quarantine_bundle, source_url_for_bundle,
|
||||
)
|
||||
from tools.skills_guard import (
|
||||
format_scan_report, scan_skill_cached, should_allow_install,
|
||||
)
|
||||
|
||||
c = console or _console
|
||||
ensure_hub_dirs()
|
||||
|
||||
# --- Resolve the set -------------------------------------------------
|
||||
try:
|
||||
if url.rstrip("/").endswith("index.json"):
|
||||
resolved = resolve_bare_index(url, name=set_name)
|
||||
else:
|
||||
catalog_url = catalog_url_for(url)
|
||||
c.print(f"\n[bold]Fetching catalog:[/] {catalog_url}")
|
||||
sets = discover_skill_sets(catalog_url)
|
||||
if not sets:
|
||||
c.print("[bold red]Error:[/] catalog has no skill-set entries "
|
||||
"(type: application/agent-skills+json).\n")
|
||||
return
|
||||
chosen = None
|
||||
if set_name:
|
||||
wanted = set_name.strip().lower()
|
||||
for s in sets:
|
||||
if wanted in (s.name.strip().lower(),
|
||||
s.command.strip().lower(),
|
||||
s.identifier.strip().lower()):
|
||||
chosen = s
|
||||
break
|
||||
if chosen is None:
|
||||
c.print(f"[bold red]Error:[/] no skill set named '{set_name}'. "
|
||||
f"Available: {', '.join(s.name for s in sets)}\n")
|
||||
return
|
||||
elif len(sets) == 1:
|
||||
chosen = sets[0]
|
||||
else:
|
||||
c.print("\n[bold]Skill sets in this catalog:[/]")
|
||||
for s in sets:
|
||||
c.print(f" • [cyan]{s.name}[/] — {s.description or '(no description)'}")
|
||||
c.print("\nRe-run with [bold]--set <name>[/] to pick one.\n")
|
||||
return
|
||||
resolved = resolve_skill_set(chosen)
|
||||
except SkillSetError as exc:
|
||||
c.print(f"[bold red]Error:[/] {exc}\n")
|
||||
return
|
||||
|
||||
info = resolved.info
|
||||
c.print(f"\n[bold]Skill set:[/] {info.name}")
|
||||
if info.description:
|
||||
c.print(f" {info.description}")
|
||||
c.print(f" [dim]Index: {info.index_url}[/]")
|
||||
c.print(f" [bold]{len(resolved.members)}[/] member skill(s): "
|
||||
f"{', '.join(m.name for m in resolved.members)}")
|
||||
for note in resolved.skipped:
|
||||
c.print(f" [yellow]Skipped:[/] {note}")
|
||||
if not resolved.members:
|
||||
c.print("[bold red]Error:[/] set has no installable members.\n")
|
||||
return
|
||||
|
||||
if not force and not skip_confirm:
|
||||
c.print(Panel(
|
||||
"[bold yellow]You are installing third-party skills at your own risk.[/]\n\n"
|
||||
"Each member is digest-verified and security-scanned before install,\n"
|
||||
"but you should review the installed files before use.",
|
||||
title="Disclaimer", border_style="yellow",
|
||||
))
|
||||
c.print(f"[bold]Install all {len(resolved.members)} skills from "
|
||||
f"'{info.name}'?[/]")
|
||||
try:
|
||||
answer = input("Confirm [y/N]: ").strip().lower()
|
||||
except (EOFError, KeyboardInterrupt):
|
||||
answer = "n"
|
||||
if answer not in {"y", "yes"}:
|
||||
c.print("[dim]Installation cancelled.[/]\n")
|
||||
return
|
||||
|
||||
# --- Install each member through the standard pipeline ----------------
|
||||
lock = HubLockFile()
|
||||
installed: List[str] = []
|
||||
failed: List[str] = []
|
||||
for member in resolved.members:
|
||||
c.print(f"\n[bold]── {member.name} ──[/]")
|
||||
try:
|
||||
bundle = fetch_member(member, set_info=info)
|
||||
except SkillSetError as exc:
|
||||
c.print(f"[bold red]Failed:[/] {exc}")
|
||||
failed.append(member.name)
|
||||
continue
|
||||
|
||||
if lock.get_installed(bundle.name) and not force:
|
||||
c.print(f"[yellow]Already installed — skipping.[/] (--force to reinstall)")
|
||||
installed.append(bundle.name) # still usable by the alias
|
||||
continue
|
||||
|
||||
try:
|
||||
q_path = quarantine_bundle(bundle)
|
||||
except ValueError as exc:
|
||||
c.print(f"[bold red]Blocked:[/] {exc}")
|
||||
append_audit_log("BLOCKED", bundle.name, bundle.source,
|
||||
bundle.trust_level, "invalid_path", str(exc))
|
||||
failed.append(member.name)
|
||||
continue
|
||||
|
||||
result, scan_provenance = scan_skill_cached(
|
||||
q_path, source=bundle.identifier,
|
||||
source_url=source_url_for_bundle(bundle),
|
||||
cache_dir=HUB_DIR / "scan-cache",
|
||||
)
|
||||
c.print(format_scan_report(result))
|
||||
allowed, reason = should_allow_install(result, force=force)
|
||||
if not allowed:
|
||||
c.print(f"[bold red]Blocked:[/] {reason}")
|
||||
shutil.rmtree(q_path, ignore_errors=True)
|
||||
append_audit_log("BLOCKED", bundle.name, bundle.source,
|
||||
bundle.trust_level, result.verdict,
|
||||
f"{len(result.findings)}_findings")
|
||||
failed.append(member.name)
|
||||
continue
|
||||
|
||||
try:
|
||||
install_dir = install_from_quarantine(
|
||||
q_path, bundle.name, "", bundle, result, scan_provenance,
|
||||
)
|
||||
except ValueError as exc:
|
||||
c.print(f"[bold red]Blocked:[/] {exc}")
|
||||
shutil.rmtree(q_path, ignore_errors=True)
|
||||
failed.append(member.name)
|
||||
continue
|
||||
c.print(f"[bold green]Installed:[/] {install_dir.name}")
|
||||
installed.append(bundle.name)
|
||||
|
||||
# --- Create the load-alias (skill bundle) ------------------------------
|
||||
c.print()
|
||||
if failed:
|
||||
c.print(f"[yellow]{len(failed)} member(s) failed:[/] {', '.join(failed)}")
|
||||
if not installed:
|
||||
c.print("[bold red]No skills installed — not creating a bundle alias.[/]\n")
|
||||
return
|
||||
|
||||
if no_alias:
|
||||
c.print(f"[bold green]Done.[/] Installed {len(installed)} skill(s).\n")
|
||||
return
|
||||
|
||||
alias = info.command or info.name
|
||||
try:
|
||||
from agent.skill_bundles import save_bundle
|
||||
bundle_path = save_bundle(
|
||||
alias, installed,
|
||||
description=info.description,
|
||||
instruction=info.instruction,
|
||||
overwrite=True,
|
||||
)
|
||||
from agent.skill_bundles import _slugify as _bundle_slug
|
||||
c.print(f"[bold green]Done.[/] Installed {len(installed)} skill(s) and "
|
||||
f"created the [bold]/{_bundle_slug(alias)}[/] bundle "
|
||||
f"([dim]{bundle_path}[/]).")
|
||||
c.print("[dim]Invoke it in chat to load the whole set in one turn. "
|
||||
"New skills appear next session (or /reset now).[/]\n")
|
||||
except (ValueError, OSError) as exc:
|
||||
c.print(f"[yellow]Installed {len(installed)} skill(s), but could not "
|
||||
f"create the bundle alias: {exc}[/]\n")
|
||||
|
||||
|
||||
def do_inspect(identifier: str, console: Optional[Console] = None) -> None:
|
||||
"""Preview a skill's SKILL.md content without installing."""
|
||||
from tools.skills_hub import GitHubAuth, create_source_router
|
||||
|
|
@ -1735,6 +1922,11 @@ def skills_command(args) -> None:
|
|||
do_install(args.identifier, category=args.category, force=args.force,
|
||||
skip_confirm=getattr(args, "yes", False),
|
||||
name_override=getattr(args, "name", "") or "")
|
||||
elif action == "install-set":
|
||||
do_install_set(args.url, set_name=getattr(args, "set_name", "") or "",
|
||||
force=getattr(args, "force", False),
|
||||
skip_confirm=getattr(args, "yes", False),
|
||||
no_alias=getattr(args, "no_alias", False))
|
||||
elif action == "inspect":
|
||||
do_inspect(args.identifier)
|
||||
elif action == "list":
|
||||
|
|
@ -1788,7 +1980,7 @@ def skills_command(args) -> None:
|
|||
return
|
||||
do_tap(tap_action, repo=repo)
|
||||
else:
|
||||
_console.print("Usage: hermes skills [browse|search|install|inspect|list|list-modified|diff|check|update|audit|uninstall|reset|opt-out|opt-in|publish|snapshot|tap]\n")
|
||||
_console.print("Usage: hermes skills [browse|search|install|install-set|inspect|list|list-modified|diff|check|update|audit|uninstall|reset|opt-out|opt-in|publish|snapshot|tap]\n")
|
||||
_console.print("Run 'hermes skills <command> --help' for details.\n")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -108,6 +108,38 @@ def build_skills_parser(subparsers, *, cmd_skills: Callable) -> None:
|
|||
help="Skip confirmation prompt (needed in TUI mode)",
|
||||
)
|
||||
|
||||
skills_install_set = skills_subparsers.add_parser(
|
||||
"install-set",
|
||||
help="Install a skill set from an AI Catalog or agent-skills discovery index",
|
||||
description=(
|
||||
"Install a group of skills published as a set. Accepts an origin "
|
||||
"(https://example.com — resolved to /.well-known/ai-catalog.json), "
|
||||
"an AI Catalog JSON URL, or a direct agent-skills index.json URL. "
|
||||
"Members are digest-verified and security-scanned individually; "
|
||||
"a /<name> bundle alias is created so the whole set loads in one turn."
|
||||
),
|
||||
)
|
||||
skills_install_set.add_argument(
|
||||
"url",
|
||||
help="Origin, AI Catalog URL, or agent-skills index.json URL",
|
||||
)
|
||||
skills_install_set.add_argument(
|
||||
"--set", dest="set_name", default="",
|
||||
help="Which skill set to install when the catalog has several",
|
||||
)
|
||||
skills_install_set.add_argument(
|
||||
"--no-alias", action="store_true",
|
||||
help="Install the skills but skip creating the /<name> bundle alias",
|
||||
)
|
||||
skills_install_set.add_argument(
|
||||
"--force", action="store_true",
|
||||
help="Reinstall members that are already installed",
|
||||
)
|
||||
skills_install_set.add_argument(
|
||||
"--yes", "-y", action="store_true",
|
||||
help="Skip confirmation prompt",
|
||||
)
|
||||
|
||||
skills_inspect = skills_subparsers.add_parser(
|
||||
"inspect", help="Preview a skill without installing"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,186 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Build a publishable skill-set catalog from local skill directories.
|
||||
|
||||
Publisher-side companion to ``hermes skills install-set``. Given one or more
|
||||
skill directories (each containing a ``SKILL.md``), this produces a static
|
||||
site tree implementing the agentskills #254 discovery index plus an AI
|
||||
Catalog (https://ai-catalog.io) wrapper entry:
|
||||
|
||||
out/
|
||||
├── .well-known/
|
||||
│ ├── ai-catalog.json # AI Catalog with one skill-set entry
|
||||
│ └── agent-skills/
|
||||
│ ├── index.json # #254 discovery index
|
||||
│ ├── <single-file-skill>/SKILL.md # type: skill-md
|
||||
│ └── <multi-file-skill>.tar.gz # type: archive
|
||||
└── (serve this directory over HTTPS)
|
||||
|
||||
Skills with only a SKILL.md are published as ``type: "skill-md"``; skills
|
||||
with supporting files (scripts/, references/, ...) are packed into a
|
||||
``.tar.gz`` with SKILL.md at the archive root, per the spec. Every artifact
|
||||
gets a required ``sha256:`` digest.
|
||||
|
||||
Usage:
|
||||
python scripts/publish_skill_set.py \
|
||||
--name "Backend Dev" \
|
||||
--command backend-dev \
|
||||
--description "Everything for backend feature work." \
|
||||
--instruction "Prefer TDD. Run the linter before opening a PR." \
|
||||
--host-name "Example Corp" \
|
||||
--out ./public \
|
||||
path/to/skill-a path/to/skill-b ...
|
||||
|
||||
Then serve ``./public`` at your origin and install with:
|
||||
hermes skills install-set https://your-origin.example
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
INDEX_SCHEMA = "https://schemas.agentskills.io/discovery/0.2.0/schema.json"
|
||||
SKILL_SET_ENTRY_TYPE = "application/agent-skills+json"
|
||||
HERMES_SET_EXTENSION = "io.hermes.skill-set"
|
||||
|
||||
|
||||
def _digest(content: bytes) -> str:
|
||||
return f"sha256:{hashlib.sha256(content).hexdigest()}"
|
||||
|
||||
|
||||
def _frontmatter_field(skill_md: str, field: str) -> str:
|
||||
"""Cheap YAML frontmatter single-field read (name/description)."""
|
||||
lines = skill_md.splitlines()
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return ""
|
||||
for line in lines[1:]:
|
||||
if line.strip() == "---":
|
||||
break
|
||||
if line.startswith(f"{field}:"):
|
||||
return line.split(":", 1)[1].strip().strip("\"'")
|
||||
return ""
|
||||
|
||||
|
||||
def _pack_tar_gz(skill_dir: Path) -> bytes:
|
||||
"""Deterministic tar.gz of a skill directory, SKILL.md at archive root."""
|
||||
buf = io.BytesIO()
|
||||
# mtime=0 + sorted names -> byte-stable archives, so digests only change
|
||||
# when content changes (plays nice with #254's digest-based caching).
|
||||
with gzip.GzipFile(fileobj=buf, mode="wb", mtime=0) as gz:
|
||||
with tarfile.open(fileobj=gz, mode="w") as tf:
|
||||
for path in sorted(skill_dir.rglob("*")):
|
||||
if not path.is_file() or path.is_symlink():
|
||||
continue
|
||||
rel = path.relative_to(skill_dir).as_posix()
|
||||
info = tarfile.TarInfo(name=rel)
|
||||
data = path.read_bytes()
|
||||
info.size = len(data)
|
||||
info.mtime = 0
|
||||
info.mode = 0o644
|
||||
tf.addfile(info, io.BytesIO(data))
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=(__doc__ or "").split("\n", 1)[0])
|
||||
ap.add_argument("skill_dirs", nargs="+", type=Path,
|
||||
help="Skill directories (each must contain SKILL.md)")
|
||||
ap.add_argument("--name", required=True, help="Skill set display name")
|
||||
ap.add_argument("--description", default="", help="Skill set description")
|
||||
ap.add_argument("--command", default="",
|
||||
help="Suggested load-alias (io.hermes.skill-set extension)")
|
||||
ap.add_argument("--instruction", default="",
|
||||
help="Shared instruction preamble (io.hermes.skill-set extension)")
|
||||
ap.add_argument("--host-name", default="", help="AI Catalog host displayName")
|
||||
ap.add_argument("--host-id", default="", help="AI Catalog host identifier (e.g. did:web:...)")
|
||||
ap.add_argument("--out", type=Path, default=Path("public"), help="Output directory")
|
||||
args = ap.parse_args()
|
||||
|
||||
well_known = args.out / ".well-known"
|
||||
skills_dir = well_known / "agent-skills"
|
||||
skills_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
entries = []
|
||||
for skill_dir in args.skill_dirs:
|
||||
skill_md_path = skill_dir / "SKILL.md"
|
||||
if not skill_md_path.is_file():
|
||||
print(f"error: {skill_dir} has no SKILL.md", file=sys.stderr)
|
||||
return 1
|
||||
skill_md = skill_md_path.read_text(encoding="utf-8")
|
||||
name = _frontmatter_field(skill_md, "name") or skill_dir.name
|
||||
description = _frontmatter_field(skill_md, "description")
|
||||
|
||||
extra_files = [p for p in skill_dir.rglob("*")
|
||||
if p.is_file() and p != skill_md_path]
|
||||
if extra_files:
|
||||
artifact = _pack_tar_gz(skill_dir)
|
||||
(skills_dir / f"{name}.tar.gz").write_bytes(artifact)
|
||||
entries.append({
|
||||
"name": name,
|
||||
"type": "archive",
|
||||
"description": description,
|
||||
"url": f"/.well-known/agent-skills/{name}.tar.gz",
|
||||
"digest": _digest(artifact),
|
||||
})
|
||||
print(f" archive {name} ({len(extra_files) + 1} files, "
|
||||
f"{len(artifact):,} bytes)")
|
||||
else:
|
||||
content = skill_md.encode("utf-8")
|
||||
dest = skills_dir / name
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
(dest / "SKILL.md").write_bytes(content)
|
||||
entries.append({
|
||||
"name": name,
|
||||
"type": "skill-md",
|
||||
"description": description,
|
||||
"url": f"/.well-known/agent-skills/{name}/SKILL.md",
|
||||
"digest": _digest(content),
|
||||
})
|
||||
print(f" skill-md {name}")
|
||||
|
||||
index = {"$schema": INDEX_SCHEMA, "skills": entries}
|
||||
(skills_dir / "index.json").write_text(
|
||||
json.dumps(index, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
set_entry = {
|
||||
"identifier": f"urn:air:{args.host_id or 'example'}:skill-set:"
|
||||
f"{args.command or args.name.lower().replace(' ', '-')}",
|
||||
"displayName": args.name,
|
||||
"description": args.description,
|
||||
"type": SKILL_SET_ENTRY_TYPE,
|
||||
"url": "/.well-known/agent-skills/index.json",
|
||||
}
|
||||
ext: dict = {}
|
||||
if args.command:
|
||||
ext["command"] = args.command
|
||||
if args.instruction:
|
||||
ext["instruction"] = args.instruction
|
||||
if ext:
|
||||
set_entry["extensions"] = {HERMES_SET_EXTENSION: ext}
|
||||
|
||||
catalog = {
|
||||
"specVersion": "1.0",
|
||||
"host": {
|
||||
"displayName": args.host_name or args.name,
|
||||
**({"identifier": args.host_id} if args.host_id else {}),
|
||||
},
|
||||
"entries": [set_entry],
|
||||
}
|
||||
(well_known / "ai-catalog.json").write_text(
|
||||
json.dumps(catalog, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
print(f"\nWrote {args.out}/.well-known/ai-catalog.json "
|
||||
f"and agent-skills/index.json ({len(entries)} skills).")
|
||||
print("Serve the output directory at your origin, then:")
|
||||
print(" hermes skills install-set https://<your-origin>")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -0,0 +1,458 @@
|
|||
"""Tests for tools/skill_set_catalog.py — the #254 + AI Catalog skill-set client.
|
||||
|
||||
Unit tests cover schema gating, digest verification, archive safety, and
|
||||
catalog/index parsing with mocked HTTP. The E2E test at the bottom runs a
|
||||
real local HTTP server built by scripts/publish_skill_set.py and exercises
|
||||
discover -> resolve -> fetch against actual bytes on the wire.
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import tarfile
|
||||
import zipfile
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from tools.skill_set_catalog import (
|
||||
ArchiveSafetyError,
|
||||
DigestError,
|
||||
SchemaError,
|
||||
SkillSetError,
|
||||
SkillSetInfo,
|
||||
catalog_url_for,
|
||||
compute_digest,
|
||||
discover_skill_sets,
|
||||
fetch_member,
|
||||
resolve_bare_index,
|
||||
resolve_skill_set,
|
||||
verify_digest,
|
||||
SkillSetMember,
|
||||
KNOWN_INDEX_SCHEMAS,
|
||||
HERMES_SET_EXTENSION,
|
||||
SKILL_SET_ENTRY_TYPE,
|
||||
)
|
||||
|
||||
SCHEMA = next(iter(KNOWN_INDEX_SCHEMAS))
|
||||
BASE = "https://skills.example.com"
|
||||
INDEX_URL = f"{BASE}/.well-known/agent-skills/index.json"
|
||||
CATALOG_URL = f"{BASE}/.well-known/ai-catalog.json"
|
||||
|
||||
SKILL_MD = (
|
||||
"---\nname: code-review\ndescription: Review code.\n---\n\n# Code Review Skill\n"
|
||||
)
|
||||
|
||||
|
||||
def _tar_gz(files: dict) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
with gzip.GzipFile(fileobj=buf, mode="wb", mtime=0) as gz:
|
||||
with tarfile.open(fileobj=gz, mode="w") as tf:
|
||||
for name, data in files.items():
|
||||
raw = data.encode() if isinstance(data, str) else data
|
||||
info = tarfile.TarInfo(name=name)
|
||||
info.size = len(raw)
|
||||
tf.addfile(info, io.BytesIO(raw))
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _zip(files: dict) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w") as zf:
|
||||
for name, data in files.items():
|
||||
zf.writestr(name, data)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _serve(pages: dict):
|
||||
"""Patch the module HTTP layer with a URL -> bytes dict."""
|
||||
def fake_get(url, *, timeout=30):
|
||||
val = pages.get(url)
|
||||
if val is None:
|
||||
return None
|
||||
return val.encode() if isinstance(val, str) else val
|
||||
return patch("tools.skill_set_catalog._http_get_bytes", side_effect=fake_get)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Digest verification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestDigest:
|
||||
def test_roundtrip(self):
|
||||
content = b"hello skills"
|
||||
verify_digest(content, compute_digest(content)) # no raise
|
||||
|
||||
def test_mismatch_rejected(self):
|
||||
with pytest.raises(DigestError, match="mismatch"):
|
||||
verify_digest(b"tampered", compute_digest(b"original"))
|
||||
|
||||
def test_missing_digest_rejected(self):
|
||||
with pytest.raises(DigestError, match="missing digest"):
|
||||
verify_digest(b"x", "")
|
||||
|
||||
@pytest.mark.parametrize("bad", [
|
||||
"sha256:short",
|
||||
"sha256:" + "G" * 64, # non-hex
|
||||
"sha256:" + "A" * 64, # uppercase — spec says lowercase
|
||||
"md5:" + "a" * 32,
|
||||
"a" * 64, # bare hex without prefix
|
||||
])
|
||||
def test_malformed_digest_rejected(self, bad):
|
||||
with pytest.raises(DigestError, match="malformed"):
|
||||
verify_digest(b"x", bad)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AI Catalog discovery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _catalog(entries) -> str:
|
||||
return json.dumps({
|
||||
"specVersion": "1.0",
|
||||
"host": {"displayName": "Example"},
|
||||
"entries": entries,
|
||||
})
|
||||
|
||||
|
||||
class TestCatalogDiscovery:
|
||||
def test_catalog_url_for_origin(self):
|
||||
assert catalog_url_for("https://x.example") == \
|
||||
"https://x.example/.well-known/ai-catalog.json"
|
||||
assert catalog_url_for("https://x.example/custom/cat.json") == \
|
||||
"https://x.example/custom/cat.json"
|
||||
|
||||
def test_finds_skill_set_entries_with_extension(self):
|
||||
pages = {CATALOG_URL: _catalog([
|
||||
{
|
||||
"identifier": "urn:air:example:skill-set:backend",
|
||||
"displayName": "Backend Dev",
|
||||
"description": "Backend feature work.",
|
||||
"type": SKILL_SET_ENTRY_TYPE,
|
||||
"url": "/.well-known/agent-skills/index.json",
|
||||
"extensions": {HERMES_SET_EXTENSION: {
|
||||
"command": "backend-dev",
|
||||
"instruction": "Prefer TDD.",
|
||||
}},
|
||||
},
|
||||
{"identifier": "urn:air:example:mcp:weather",
|
||||
"type": "application/mcp-server-card+json",
|
||||
"url": "https://api.example.com/mcp"},
|
||||
])}
|
||||
with _serve(pages):
|
||||
sets = discover_skill_sets(CATALOG_URL)
|
||||
assert len(sets) == 1
|
||||
s = sets[0]
|
||||
assert s.name == "Backend Dev"
|
||||
assert s.index_url == INDEX_URL
|
||||
assert s.command == "backend-dev"
|
||||
assert s.instruction == "Prefer TDD."
|
||||
|
||||
def test_entry_without_extension_still_discovered(self):
|
||||
pages = {CATALOG_URL: _catalog([
|
||||
{"displayName": "Plain Set", "type": SKILL_SET_ENTRY_TYPE,
|
||||
"url": "/.well-known/agent-skills/index.json"},
|
||||
])}
|
||||
with _serve(pages):
|
||||
sets = discover_skill_sets(CATALOG_URL)
|
||||
assert len(sets) == 1
|
||||
assert sets[0].command == ""
|
||||
assert sets[0].instruction == ""
|
||||
|
||||
def test_follows_sub_catalog_one_level(self):
|
||||
sub_url = f"{BASE}/catalogs/eng.json"
|
||||
pages = {
|
||||
CATALOG_URL: _catalog([
|
||||
{"displayName": "Engineering", "type": "application/ai-catalog+json",
|
||||
"url": "/catalogs/eng.json"},
|
||||
]),
|
||||
sub_url: _catalog([
|
||||
{"displayName": "Backend Dev", "type": SKILL_SET_ENTRY_TYPE,
|
||||
"url": "/.well-known/agent-skills/index.json"},
|
||||
]),
|
||||
}
|
||||
with _serve(pages):
|
||||
sets = discover_skill_sets(CATALOG_URL)
|
||||
assert [s.name for s in sets] == ["Backend Dev"]
|
||||
assert sets[0].index_url == INDEX_URL
|
||||
|
||||
def test_not_a_catalog_raises(self):
|
||||
with _serve({CATALOG_URL: json.dumps({"skills": []})}):
|
||||
with pytest.raises(SkillSetError, match="not an AI Catalog"):
|
||||
discover_skill_sets(CATALOG_URL)
|
||||
|
||||
def test_unreachable_catalog_raises(self):
|
||||
with _serve({}):
|
||||
with pytest.raises(SkillSetError, match="Could not fetch"):
|
||||
discover_skill_sets(CATALOG_URL)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #254 index resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _index(skills, schema: "str | None" = SCHEMA) -> str:
|
||||
payload = {"skills": skills}
|
||||
if schema is not None:
|
||||
payload["$schema"] = schema
|
||||
return json.dumps(payload)
|
||||
|
||||
|
||||
def _info() -> SkillSetInfo:
|
||||
return SkillSetInfo(name="Test Set", description="", index_url=INDEX_URL)
|
||||
|
||||
|
||||
class TestIndexResolution:
|
||||
def test_members_parsed_and_urls_resolved(self):
|
||||
pages = {INDEX_URL: _index([
|
||||
{"name": "code-review", "type": "skill-md",
|
||||
"description": "Review code.",
|
||||
"url": "code-review/SKILL.md", "digest": compute_digest(b"x")},
|
||||
{"name": "wrangler", "type": "archive",
|
||||
"description": "Deploy workers.",
|
||||
"url": "/.well-known/agent-skills/wrangler.tar.gz",
|
||||
"digest": compute_digest(b"y")},
|
||||
])}
|
||||
with _serve(pages):
|
||||
resolved = resolve_skill_set(_info())
|
||||
assert [m.name for m in resolved.members] == ["code-review", "wrangler"]
|
||||
# Relative resolved against index directory; path-absolute against origin.
|
||||
assert resolved.members[0].url == \
|
||||
f"{BASE}/.well-known/agent-skills/code-review/SKILL.md"
|
||||
assert resolved.members[1].url == \
|
||||
f"{BASE}/.well-known/agent-skills/wrangler.tar.gz"
|
||||
|
||||
def test_unknown_schema_refused(self):
|
||||
pages = {INDEX_URL: _index([], schema="https://example.com/other/1.0.json")}
|
||||
with _serve(pages):
|
||||
with pytest.raises(SchemaError, match="Unrecognized index"):
|
||||
resolve_skill_set(_info())
|
||||
|
||||
def test_absent_schema_refused(self):
|
||||
pages = {INDEX_URL: _index([], schema=None)}
|
||||
with _serve(pages):
|
||||
with pytest.raises(SchemaError):
|
||||
resolve_skill_set(_info())
|
||||
|
||||
def test_unrecognized_type_skipped_with_warning(self):
|
||||
pages = {INDEX_URL: _index([
|
||||
{"name": "good", "type": "skill-md", "url": "good/SKILL.md",
|
||||
"digest": compute_digest(b"x")},
|
||||
{"name": "weird", "type": "oci-image", "url": "weird.oci",
|
||||
"digest": compute_digest(b"y")},
|
||||
])}
|
||||
with _serve(pages):
|
||||
resolved = resolve_skill_set(_info())
|
||||
assert [m.name for m in resolved.members] == ["good"]
|
||||
assert any("weird" in s for s in resolved.skipped)
|
||||
|
||||
def test_bare_index_fallback_name(self):
|
||||
pages = {INDEX_URL: _index([])}
|
||||
with _serve(pages):
|
||||
resolved = resolve_bare_index(INDEX_URL)
|
||||
assert resolved.info.name == "skills.example.com"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Member fetching — digest + archive safety
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _member(name="code-review", mtype="skill-md", url=None, digest=""):
|
||||
return SkillSetMember(
|
||||
name=name, description="", type=mtype,
|
||||
url=url or f"{BASE}/.well-known/agent-skills/{name}/SKILL.md",
|
||||
digest=digest,
|
||||
)
|
||||
|
||||
|
||||
class TestFetchMember:
|
||||
def test_skill_md_happy_path(self):
|
||||
content = SKILL_MD.encode()
|
||||
m = _member(digest=compute_digest(content))
|
||||
with _serve({m.url: content}):
|
||||
bundle = fetch_member(m, set_info=_info())
|
||||
assert bundle.name == "code-review"
|
||||
assert bundle.files == {"SKILL.md": SKILL_MD}
|
||||
assert bundle.source == "skill-set"
|
||||
assert bundle.metadata["digest"] == m.digest
|
||||
|
||||
def test_tampered_content_rejected(self):
|
||||
m = _member(digest=compute_digest(SKILL_MD.encode()))
|
||||
with _serve({m.url: b"---\nname: evil\n---\nrm -rf /"}):
|
||||
with pytest.raises(DigestError, match="mismatch"):
|
||||
fetch_member(m, set_info=_info())
|
||||
|
||||
def test_archive_happy_path(self):
|
||||
artifact = _tar_gz({
|
||||
"SKILL.md": SKILL_MD,
|
||||
"scripts/deploy.sh": "#!/bin/sh\necho hi\n",
|
||||
"references/API.md": "# API\n",
|
||||
})
|
||||
url = f"{BASE}/.well-known/agent-skills/wrangler.tar.gz"
|
||||
m = _member("wrangler", "archive", url, compute_digest(artifact))
|
||||
with _serve({url: artifact}):
|
||||
bundle = fetch_member(m, set_info=_info())
|
||||
assert set(bundle.files) == {"SKILL.md", "scripts/deploy.sh",
|
||||
"references/API.md"}
|
||||
|
||||
def test_zip_happy_path(self):
|
||||
artifact = _zip({"SKILL.md": SKILL_MD, "references/NOTES.md": "notes"})
|
||||
url = f"{BASE}/.well-known/agent-skills/z.zip"
|
||||
m = _member("zskill", "archive", url, compute_digest(artifact))
|
||||
with _serve({url: artifact}):
|
||||
bundle = fetch_member(m, set_info=_info())
|
||||
assert set(bundle.files) == {"SKILL.md", "references/NOTES.md"}
|
||||
|
||||
def test_archive_without_root_skill_md_rejected(self):
|
||||
artifact = _tar_gz({"nested/SKILL.md": SKILL_MD})
|
||||
url = f"{BASE}/.well-known/agent-skills/bad.tar.gz"
|
||||
m = _member("bad", "archive", url, compute_digest(artifact))
|
||||
with _serve({url: artifact}):
|
||||
with pytest.raises(ArchiveSafetyError, match="no SKILL.md at its root"):
|
||||
fetch_member(m, set_info=_info())
|
||||
|
||||
def test_path_traversal_rejected(self):
|
||||
artifact = _tar_gz({"SKILL.md": SKILL_MD, "../../evil.sh": "boom"})
|
||||
url = f"{BASE}/.well-known/agent-skills/trav.tar.gz"
|
||||
m = _member("trav", "archive", url, compute_digest(artifact))
|
||||
with _serve({url: artifact}):
|
||||
with pytest.raises(ArchiveSafetyError):
|
||||
fetch_member(m, set_info=_info())
|
||||
|
||||
def test_absolute_path_rejected(self):
|
||||
artifact = _tar_gz({"SKILL.md": SKILL_MD, "/etc/cron.d/evil": "boom"})
|
||||
url = f"{BASE}/.well-known/agent-skills/abs.tar.gz"
|
||||
m = _member("abs", "archive", url, compute_digest(artifact))
|
||||
with _serve({url: artifact}):
|
||||
with pytest.raises(ArchiveSafetyError):
|
||||
fetch_member(m, set_info=_info())
|
||||
|
||||
def test_symlink_member_rejected(self):
|
||||
buf = io.BytesIO()
|
||||
with gzip.GzipFile(fileobj=buf, mode="wb", mtime=0) as gz:
|
||||
with tarfile.open(fileobj=gz, mode="w") as tf:
|
||||
data = SKILL_MD.encode()
|
||||
info = tarfile.TarInfo(name="SKILL.md")
|
||||
info.size = len(data)
|
||||
tf.addfile(info, io.BytesIO(data))
|
||||
link = tarfile.TarInfo(name="creds")
|
||||
link.type = tarfile.SYMTYPE
|
||||
link.linkname = "/home/user/.ssh/id_rsa"
|
||||
tf.addfile(link)
|
||||
artifact = buf.getvalue()
|
||||
url = f"{BASE}/.well-known/agent-skills/lnk.tar.gz"
|
||||
m = _member("lnk", "archive", url, compute_digest(artifact))
|
||||
with _serve({url: artifact}):
|
||||
with pytest.raises(ArchiveSafetyError, match="link member"):
|
||||
fetch_member(m, set_info=_info())
|
||||
|
||||
def test_decompression_bomb_rejected(self):
|
||||
# 20MB of zeros compresses tiny but exceeds the per-member cap.
|
||||
artifact = _tar_gz({"SKILL.md": SKILL_MD, "big.bin": b"\0" * (6 * 1024 * 1024)})
|
||||
url = f"{BASE}/.well-known/agent-skills/bomb.tar.gz"
|
||||
m = _member("bomb", "archive", url, compute_digest(artifact))
|
||||
with _serve({url: artifact}):
|
||||
with pytest.raises(ArchiveSafetyError):
|
||||
fetch_member(m, set_info=_info())
|
||||
|
||||
def test_unsupported_archive_extension_rejected(self):
|
||||
url = f"{BASE}/.well-known/agent-skills/skill.rar"
|
||||
m = _member("rarred", "archive", url, compute_digest(b"data"))
|
||||
with _serve({url: b"data"}):
|
||||
with pytest.raises(ArchiveSafetyError, match="unsupported archive format"):
|
||||
fetch_member(m, set_info=_info())
|
||||
|
||||
def test_unsafe_skill_name_rejected(self):
|
||||
m = _member("../escape", digest=compute_digest(b"x"))
|
||||
with _serve({m.url: b"x"}):
|
||||
with pytest.raises(ValueError):
|
||||
fetch_member(m, set_info=_info())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# E2E: publisher script -> real HTTP server -> full client flow
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestEndToEnd:
|
||||
@pytest.fixture()
|
||||
def published_site(self, tmp_path):
|
||||
"""Build a real publisher tree with scripts/publish_skill_set.py."""
|
||||
import subprocess
|
||||
import sys as _sys
|
||||
from pathlib import Path as _P
|
||||
|
||||
repo_root = _P(__file__).resolve().parents[2]
|
||||
script = repo_root / "scripts" / "publish_skill_set.py"
|
||||
|
||||
# One single-file skill, one multi-file skill.
|
||||
s1 = tmp_path / "src" / "code-review"
|
||||
s1.mkdir(parents=True)
|
||||
(s1 / "SKILL.md").write_text(SKILL_MD)
|
||||
s2 = tmp_path / "src" / "deploy-tool"
|
||||
(s2 / "scripts").mkdir(parents=True)
|
||||
(s2 / "SKILL.md").write_text(
|
||||
"---\nname: deploy-tool\ndescription: Deploy things.\n---\n\n# Deploy\n")
|
||||
(s2 / "scripts" / "run.sh").write_text("#!/bin/sh\necho deploy\n")
|
||||
|
||||
out = tmp_path / "public"
|
||||
subprocess.run(
|
||||
[_sys.executable, str(script),
|
||||
"--name", "Backend Dev", "--command", "backend-dev",
|
||||
"--description", "Backend feature work.",
|
||||
"--instruction", "Prefer TDD.",
|
||||
"--out", str(out), str(s1), str(s2)],
|
||||
check=True, capture_output=True, text=True,
|
||||
)
|
||||
return out
|
||||
|
||||
def test_full_flow_over_real_http(self, published_site):
|
||||
import http.server
|
||||
import threading
|
||||
|
||||
handler = type("H", (http.server.SimpleHTTPRequestHandler,), {
|
||||
"directory": str(published_site),
|
||||
"log_message": lambda self, *a: None,
|
||||
})
|
||||
httpd = http.server.ThreadingHTTPServer(
|
||||
("127.0.0.1", 0),
|
||||
lambda *a, **kw: handler(*a, directory=str(published_site), **kw),
|
||||
)
|
||||
port = httpd.server_address[1]
|
||||
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
origin = f"http://127.0.0.1:{port}"
|
||||
|
||||
# Bypass the SSRF guard for the loopback test server only.
|
||||
def local_get(url, *, timeout=30):
|
||||
import httpx
|
||||
resp = httpx.get(url, timeout=timeout)
|
||||
return resp.content if resp.status_code == 200 else None
|
||||
|
||||
with patch("tools.skill_set_catalog._http_get_bytes",
|
||||
side_effect=local_get):
|
||||
sets = discover_skill_sets(catalog_url_for(origin))
|
||||
assert len(sets) == 1
|
||||
info = sets[0]
|
||||
assert info.name == "Backend Dev"
|
||||
assert info.command == "backend-dev"
|
||||
assert info.instruction == "Prefer TDD."
|
||||
|
||||
resolved = resolve_skill_set(info)
|
||||
assert {m.name for m in resolved.members} == \
|
||||
{"code-review", "deploy-tool"}
|
||||
types = {m.name: m.type for m in resolved.members}
|
||||
assert types["code-review"] == "skill-md"
|
||||
assert types["deploy-tool"] == "archive"
|
||||
|
||||
bundles = {m.name: fetch_member(m, set_info=info)
|
||||
for m in resolved.members}
|
||||
assert bundles["code-review"].files["SKILL.md"] == SKILL_MD
|
||||
assert "scripts/run.sh" in bundles["deploy-tool"].files
|
||||
# Digest verification ran on real bytes for every member.
|
||||
for b in bundles.values():
|
||||
assert b.metadata["digest"].startswith("sha256:")
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
|
|
@ -0,0 +1,504 @@
|
|||
"""Skill-set catalog client — prototype for the agentskills #254 + AI Catalog layering.
|
||||
|
||||
Implements the design discussed with agentskills.io:
|
||||
|
||||
1. **AI Catalog** (https://ai-catalog.io) — a typed discovery index served at
|
||||
``/.well-known/ai-catalog.json``. Entries with
|
||||
``type: "application/agent-skills+json"`` point at an agent-skills
|
||||
discovery index and represent an installable *skill set*.
|
||||
|
||||
2. **agentskills PR #254 discovery index** — the index the entry points at:
|
||||
``{"$schema": ..., "skills": [{name, description, type, url, digest}]}``
|
||||
where ``type`` is ``"skill-md"`` (single file) or ``"archive"``
|
||||
(``.tar.gz`` / ``.zip`` with SKILL.md at the archive root).
|
||||
|
||||
3. **``io.hermes.skill-set`` extension** — optional namespaced metadata on
|
||||
the AI Catalog entry carrying set-level usage intent::
|
||||
|
||||
"extensions": {
|
||||
"io.hermes.skill-set": {
|
||||
"command": "backend-dev",
|
||||
"instruction": "Prefer TDD. Run the linter before opening a PR."
|
||||
}
|
||||
}
|
||||
|
||||
``command`` is the suggested local load-alias (Hermes creates a skill
|
||||
bundle so ``/backend-dev`` loads every member in one turn);
|
||||
``instruction`` is a shared preamble injected above the member skills.
|
||||
Clients that don't recognize the extension still install the correct
|
||||
set — they only miss the alias/preamble sugar.
|
||||
|
||||
Security posture matches the rest of the skills hub: all HTTP goes through
|
||||
the SSRF-guarded fetcher, member artifacts are digest-verified (SHA-256,
|
||||
required by #254), archive extraction rejects path traversal / absolute
|
||||
paths / symlinks / hardlinks and caps decompressed size, and every fetched
|
||||
skill flows through the existing quarantine -> scan -> install pipeline.
|
||||
|
||||
This module only *fetches and validates*; installation is orchestrated by
|
||||
``hermes_cli.skills_hub.do_install_set``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import posixpath
|
||||
import re
|
||||
import tarfile
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
#: MIME-ish AI Catalog entry type identifying an agent-skills discovery index.
|
||||
SKILL_SET_ENTRY_TYPE = "application/agent-skills+json"
|
||||
|
||||
#: Nested AI Catalog entry type (sub-catalogs) — followed one level deep.
|
||||
CATALOG_ENTRY_TYPE = "application/ai-catalog+json"
|
||||
|
||||
#: Namespaced AI Catalog extension carrying set-level usage intent.
|
||||
HERMES_SET_EXTENSION = "io.hermes.skill-set"
|
||||
|
||||
#: ``$schema`` URIs this client knows how to process (agentskills #254).
|
||||
KNOWN_INDEX_SCHEMAS = frozenset({
|
||||
"https://schemas.agentskills.io/discovery/0.2.0/schema.json",
|
||||
})
|
||||
|
||||
#: Well-known path for AI Catalog discovery.
|
||||
AI_CATALOG_WELL_KNOWN = "/.well-known/ai-catalog.json"
|
||||
|
||||
_DIGEST_RE = re.compile(r"^sha256:([0-9a-f]{64})$")
|
||||
|
||||
# Archive safety caps (decompression-bomb guard per #254 "Archive safety").
|
||||
MAX_ARCHIVE_BYTES = 20 * 1024 * 1024 # compressed artifact cap
|
||||
MAX_UNPACKED_BYTES = 50 * 1024 * 1024 # total decompressed cap
|
||||
MAX_MEMBER_BYTES = 5 * 1024 * 1024 # single decompressed file cap
|
||||
MAX_ARCHIVE_MEMBERS = 500
|
||||
|
||||
#: Max skills a single set may enumerate (sanity cap for the prototype).
|
||||
MAX_SET_MEMBERS = 50
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Errors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class SkillSetError(Exception):
|
||||
"""Base error for skill-set catalog operations."""
|
||||
|
||||
|
||||
class SchemaError(SkillSetError):
|
||||
"""Unrecognized or missing ``$schema`` — per #254, warn and stop."""
|
||||
|
||||
|
||||
class DigestError(SkillSetError):
|
||||
"""Artifact digest missing, malformed, or mismatched."""
|
||||
|
||||
|
||||
class ArchiveSafetyError(SkillSetError):
|
||||
"""Archive violated safety rules (traversal, links, bomb, structure)."""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class SkillSetInfo:
|
||||
"""A skill set discovered in an AI Catalog (or a bare #254 index)."""
|
||||
name: str # display name (catalog displayName or derived)
|
||||
description: str
|
||||
index_url: str # resolved URL of the #254 index.json
|
||||
identifier: str = "" # catalog entry identifier (urn:...), if any
|
||||
command: str = "" # io.hermes.skill-set suggested alias
|
||||
instruction: str = "" # io.hermes.skill-set shared preamble
|
||||
catalog_url: str = "" # catalog this was discovered from, if any
|
||||
|
||||
|
||||
@dataclass
|
||||
class SkillSetMember:
|
||||
"""One skill entry from a #254 discovery index."""
|
||||
name: str
|
||||
description: str
|
||||
type: str # "skill-md" | "archive"
|
||||
url: str # resolved absolute URL
|
||||
digest: str # "sha256:<hex>"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResolvedSkillSet:
|
||||
"""A fully parsed set: info + members, ready to fetch."""
|
||||
info: SkillSetInfo
|
||||
members: List[SkillSetMember] = field(default_factory=list)
|
||||
skipped: List[str] = field(default_factory=list) # entries skipped w/ reason
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HTTP (indirection point — tests monkeypatch _http_get_bytes)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _http_get_bytes(url: str, *, timeout: int = 30) -> Optional[bytes]:
|
||||
"""SSRF-guarded GET returning raw bytes, or None on any failure."""
|
||||
from tools.skills_hub import _guarded_http_get
|
||||
resp = _guarded_http_get(url, timeout=timeout)
|
||||
if resp is None or resp.status_code != 200:
|
||||
return None
|
||||
return resp.content
|
||||
|
||||
|
||||
def _http_get_json(url: str, *, timeout: int = 20) -> Optional[Any]:
|
||||
raw = _http_get_bytes(url, timeout=timeout)
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError):
|
||||
logger.warning("Skill-set fetch: invalid JSON at %s", url)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Digest verification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def verify_digest(content: bytes, digest: str, *, what: str = "artifact") -> None:
|
||||
"""Verify ``content`` against a ``sha256:<hex>`` digest string.
|
||||
|
||||
Raises :class:`DigestError` on missing/malformed/mismatched digests —
|
||||
#254 makes the digest required and forbids using unverified content.
|
||||
"""
|
||||
if not isinstance(digest, str) or not digest:
|
||||
raise DigestError(f"{what}: missing digest (required by the discovery spec)")
|
||||
m = _DIGEST_RE.match(digest.strip())
|
||||
if not m:
|
||||
raise DigestError(
|
||||
f"{what}: malformed digest {digest!r} (expected sha256:<64 lowercase hex>)"
|
||||
)
|
||||
actual = hashlib.sha256(content).hexdigest()
|
||||
if actual != m.group(1):
|
||||
raise DigestError(
|
||||
f"{what}: digest mismatch — index says sha256:{m.group(1)}, "
|
||||
f"downloaded content is sha256:{actual}. Refusing to use it."
|
||||
)
|
||||
|
||||
|
||||
def compute_digest(content: bytes) -> str:
|
||||
"""Format a #254 digest string for raw bytes (publisher-side helper)."""
|
||||
return f"sha256:{hashlib.sha256(content).hexdigest()}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AI Catalog parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def catalog_url_for(origin_or_url: str) -> str:
|
||||
"""Normalize user input to an AI Catalog URL.
|
||||
|
||||
``https://example.com`` -> ``https://example.com/.well-known/ai-catalog.json``;
|
||||
anything already ending in ``.json`` is returned as-is.
|
||||
"""
|
||||
url = origin_or_url.strip().rstrip("/")
|
||||
if url.endswith(".json"):
|
||||
return origin_or_url.strip()
|
||||
return url + AI_CATALOG_WELL_KNOWN
|
||||
|
||||
|
||||
def _entry_extensions(entry: Dict[str, Any]) -> Dict[str, Any]:
|
||||
ext = entry.get("extensions")
|
||||
return ext if isinstance(ext, dict) else {}
|
||||
|
||||
|
||||
def discover_skill_sets(catalog_url: str, *, _depth: int = 0) -> List[SkillSetInfo]:
|
||||
"""Fetch an AI Catalog and return every skill-set entry in it.
|
||||
|
||||
Follows ``application/ai-catalog+json`` sub-catalog entries one level
|
||||
deep (AI Catalog is nestable; the prototype bounds recursion at 1).
|
||||
"""
|
||||
data = _http_get_json(catalog_url)
|
||||
if not isinstance(data, dict):
|
||||
raise SkillSetError(f"Could not fetch AI Catalog at {catalog_url}")
|
||||
|
||||
entries = data.get("entries")
|
||||
if not isinstance(entries, list):
|
||||
raise SkillSetError(
|
||||
f"{catalog_url} is not an AI Catalog (no 'entries' array)"
|
||||
)
|
||||
|
||||
sets: List[SkillSetInfo] = []
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
etype = entry.get("type")
|
||||
url = entry.get("url")
|
||||
if not isinstance(url, str) or not url:
|
||||
continue
|
||||
resolved = urljoin(catalog_url, url)
|
||||
|
||||
if etype == SKILL_SET_ENTRY_TYPE:
|
||||
hermes_ext = _entry_extensions(entry).get(HERMES_SET_EXTENSION)
|
||||
hermes_ext = hermes_ext if isinstance(hermes_ext, dict) else {}
|
||||
sets.append(SkillSetInfo(
|
||||
name=str(entry.get("displayName") or entry.get("identifier") or resolved),
|
||||
description=str(entry.get("description") or ""),
|
||||
index_url=resolved,
|
||||
identifier=str(entry.get("identifier") or ""),
|
||||
command=str(hermes_ext.get("command") or ""),
|
||||
instruction=str(hermes_ext.get("instruction") or ""),
|
||||
catalog_url=catalog_url,
|
||||
))
|
||||
elif etype == CATALOG_ENTRY_TYPE and _depth < 1:
|
||||
try:
|
||||
sets.extend(discover_skill_sets(resolved, _depth=_depth + 1))
|
||||
except SkillSetError as exc:
|
||||
logger.warning("Skipping unreachable sub-catalog %s: %s", resolved, exc)
|
||||
|
||||
return sets
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #254 discovery index parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def resolve_skill_set(info: SkillSetInfo) -> ResolvedSkillSet:
|
||||
"""Fetch and validate the #254 index a :class:`SkillSetInfo` points at."""
|
||||
data = _http_get_json(info.index_url)
|
||||
if not isinstance(data, dict):
|
||||
raise SkillSetError(f"Could not fetch skill index at {info.index_url}")
|
||||
|
||||
schema = data.get("$schema")
|
||||
if not isinstance(schema, str) or schema not in KNOWN_INDEX_SCHEMAS:
|
||||
# #254: "Clients encountering an unrecognized or absent $schema
|
||||
# should warn the user and should not process the index."
|
||||
raise SchemaError(
|
||||
f"Unrecognized index $schema {schema!r} at {info.index_url}. "
|
||||
f"Known: {', '.join(sorted(KNOWN_INDEX_SCHEMAS))}"
|
||||
)
|
||||
|
||||
raw_skills = data.get("skills")
|
||||
if not isinstance(raw_skills, list):
|
||||
raise SkillSetError(f"Index at {info.index_url} has no 'skills' array")
|
||||
if len(raw_skills) > MAX_SET_MEMBERS:
|
||||
raise SkillSetError(
|
||||
f"Index enumerates {len(raw_skills)} skills (cap: {MAX_SET_MEMBERS})"
|
||||
)
|
||||
|
||||
members: List[SkillSetMember] = []
|
||||
skipped: List[str] = []
|
||||
for entry in raw_skills:
|
||||
if not isinstance(entry, dict):
|
||||
skipped.append("(non-object entry)")
|
||||
continue
|
||||
name = entry.get("name")
|
||||
etype = entry.get("type")
|
||||
url = entry.get("url")
|
||||
digest = entry.get("digest")
|
||||
if not isinstance(name, str) or not name:
|
||||
skipped.append("(entry without a name)")
|
||||
continue
|
||||
if etype not in ("skill-md", "archive"):
|
||||
# #254: skip entries with an unrecognized type and warn.
|
||||
skipped.append(f"{name} (unrecognized type {etype!r})")
|
||||
continue
|
||||
if not isinstance(url, str) or not url:
|
||||
skipped.append(f"{name} (missing url)")
|
||||
continue
|
||||
members.append(SkillSetMember(
|
||||
name=name,
|
||||
description=str(entry.get("description") or ""),
|
||||
type=etype,
|
||||
url=urljoin(info.index_url, url),
|
||||
digest=str(digest or ""),
|
||||
))
|
||||
|
||||
return ResolvedSkillSet(info=info, members=members, skipped=skipped)
|
||||
|
||||
|
||||
def resolve_bare_index(index_url: str, *, name: str = "") -> ResolvedSkillSet:
|
||||
"""Treat a raw #254 index URL as a set (no AI Catalog wrapper).
|
||||
|
||||
Used when the user hands us an index.json directly. The set name falls
|
||||
back to the host name; no extension metadata is available on this path.
|
||||
"""
|
||||
host = urlparse(index_url).netloc or index_url
|
||||
info = SkillSetInfo(
|
||||
name=name or host,
|
||||
description=f"Skill set from {host}",
|
||||
index_url=index_url,
|
||||
)
|
||||
return resolve_skill_set(info)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Member fetching (skill-md + archive) -> SkillBundle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _safe_member_path(raw_name: str) -> str:
|
||||
"""Validate an archive member path per #254 archive-safety rules."""
|
||||
name = raw_name.replace("\\", "/")
|
||||
if name.startswith("/") or re.match(r"^[A-Za-z]:", name):
|
||||
raise ArchiveSafetyError(f"absolute path in archive: {raw_name!r}")
|
||||
normalized = posixpath.normpath(name)
|
||||
if normalized.startswith("..") or "/../" in f"/{normalized}/":
|
||||
raise ArchiveSafetyError(f"path traversal in archive: {raw_name!r}")
|
||||
if normalized in (".", ""):
|
||||
raise ArchiveSafetyError(f"empty archive member path: {raw_name!r}")
|
||||
# Reuse the hub-wide validator for charset / depth constraints.
|
||||
from tools.skills_hub import _validate_bundle_rel_path
|
||||
return _validate_bundle_rel_path(normalized)
|
||||
|
||||
|
||||
def _decode_member(raw: bytes) -> Union[str, bytes]:
|
||||
try:
|
||||
return raw.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return raw
|
||||
|
||||
|
||||
def _extract_tar_gz(content: bytes) -> Dict[str, Union[str, bytes]]:
|
||||
files: Dict[str, Union[str, bytes]] = {}
|
||||
total = 0
|
||||
try:
|
||||
tf = tarfile.open(fileobj=io.BytesIO(content), mode="r:gz")
|
||||
except tarfile.TarError as exc:
|
||||
raise ArchiveSafetyError(f"invalid .tar.gz archive: {exc}") from exc
|
||||
with tf:
|
||||
members = tf.getmembers()
|
||||
if len(members) > MAX_ARCHIVE_MEMBERS:
|
||||
raise ArchiveSafetyError(
|
||||
f"archive has {len(members)} members (cap: {MAX_ARCHIVE_MEMBERS})"
|
||||
)
|
||||
for m in members:
|
||||
if m.issym() or m.islnk():
|
||||
# #254: reject symlinks/hardlinks outright (resolving them
|
||||
# safely is not worth it for skill payloads).
|
||||
raise ArchiveSafetyError(f"link member in archive: {m.name!r}")
|
||||
if not m.isfile():
|
||||
continue
|
||||
safe = _safe_member_path(m.name)
|
||||
if m.size > MAX_MEMBER_BYTES:
|
||||
raise ArchiveSafetyError(
|
||||
f"archive member {safe} is {m.size} bytes (cap: {MAX_MEMBER_BYTES})"
|
||||
)
|
||||
total += m.size
|
||||
if total > MAX_UNPACKED_BYTES:
|
||||
raise ArchiveSafetyError("archive exceeds decompressed size cap")
|
||||
fh = tf.extractfile(m)
|
||||
if fh is None:
|
||||
continue
|
||||
raw = fh.read(MAX_MEMBER_BYTES + 1)
|
||||
if len(raw) > MAX_MEMBER_BYTES:
|
||||
raise ArchiveSafetyError(f"archive member {safe} exceeded size cap")
|
||||
files[safe] = _decode_member(raw)
|
||||
return files
|
||||
|
||||
|
||||
def _extract_zip(content: bytes) -> Dict[str, Union[str, bytes]]:
|
||||
files: Dict[str, Union[str, bytes]] = {}
|
||||
total = 0
|
||||
try:
|
||||
zf = zipfile.ZipFile(io.BytesIO(content))
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise ArchiveSafetyError(f"invalid .zip archive: {exc}") from exc
|
||||
with zf:
|
||||
infos = zf.infolist()
|
||||
if len(infos) > MAX_ARCHIVE_MEMBERS:
|
||||
raise ArchiveSafetyError(
|
||||
f"archive has {len(infos)} members (cap: {MAX_ARCHIVE_MEMBERS})"
|
||||
)
|
||||
for zinfo in infos:
|
||||
if zinfo.is_dir():
|
||||
continue
|
||||
# Zip symlinks encode the link mode in external_attr's high bits.
|
||||
if (zinfo.external_attr >> 16) & 0o170000 == 0o120000:
|
||||
raise ArchiveSafetyError(f"symlink member in archive: {zinfo.filename!r}")
|
||||
safe = _safe_member_path(zinfo.filename)
|
||||
if zinfo.file_size > MAX_MEMBER_BYTES:
|
||||
raise ArchiveSafetyError(
|
||||
f"archive member {safe} is {zinfo.file_size} bytes "
|
||||
f"(cap: {MAX_MEMBER_BYTES})"
|
||||
)
|
||||
total += zinfo.file_size
|
||||
if total > MAX_UNPACKED_BYTES:
|
||||
raise ArchiveSafetyError("archive exceeds decompressed size cap")
|
||||
with zf.open(zinfo) as fh:
|
||||
raw = fh.read(MAX_MEMBER_BYTES + 1)
|
||||
if len(raw) > MAX_MEMBER_BYTES:
|
||||
raise ArchiveSafetyError(f"archive member {safe} exceeded size cap")
|
||||
files[safe] = _decode_member(raw)
|
||||
return files
|
||||
|
||||
|
||||
def _archive_format(url: str) -> str:
|
||||
path = urlparse(url).path.lower()
|
||||
if path.endswith((".tar.gz", ".tgz")):
|
||||
return "tar.gz"
|
||||
if path.endswith(".zip"):
|
||||
return "zip"
|
||||
raise ArchiveSafetyError(
|
||||
f"unsupported archive format for {url} (expected .tar.gz or .zip)"
|
||||
)
|
||||
|
||||
|
||||
def fetch_member(member: SkillSetMember, *, set_info: SkillSetInfo):
|
||||
"""Download, digest-verify, and unpack one set member.
|
||||
|
||||
Returns a ``tools.skills_hub.SkillBundle`` ready for the standard
|
||||
quarantine -> scan -> install pipeline.
|
||||
"""
|
||||
from tools.skills_hub import SkillBundle, _validate_skill_name
|
||||
|
||||
skill_name = _validate_skill_name(member.name)
|
||||
|
||||
content = _http_get_bytes(member.url)
|
||||
if content is None:
|
||||
raise SkillSetError(f"{member.name}: could not download {member.url}")
|
||||
if len(content) > MAX_ARCHIVE_BYTES:
|
||||
raise ArchiveSafetyError(
|
||||
f"{member.name}: artifact is {len(content)} bytes (cap: {MAX_ARCHIVE_BYTES})"
|
||||
)
|
||||
|
||||
verify_digest(content, member.digest, what=member.name)
|
||||
|
||||
if member.type == "skill-md":
|
||||
try:
|
||||
files: Dict[str, Union[str, bytes]] = {
|
||||
"SKILL.md": content.decode("utf-8")
|
||||
}
|
||||
except UnicodeDecodeError as exc:
|
||||
raise SkillSetError(f"{member.name}: SKILL.md is not valid UTF-8") from exc
|
||||
else:
|
||||
files = (
|
||||
_extract_tar_gz(content)
|
||||
if _archive_format(member.url) == "tar.gz"
|
||||
else _extract_zip(content)
|
||||
)
|
||||
if "SKILL.md" not in files:
|
||||
# #254: "The archive must contain SKILL.md at the root."
|
||||
raise ArchiveSafetyError(
|
||||
f"{member.name}: archive has no SKILL.md at its root"
|
||||
)
|
||||
|
||||
return SkillBundle(
|
||||
name=skill_name,
|
||||
files=files,
|
||||
source="skill-set",
|
||||
identifier=f"skill-set:{set_info.index_url}#{skill_name}",
|
||||
trust_level="community",
|
||||
metadata={
|
||||
"set_name": set_info.name,
|
||||
"index_url": set_info.index_url,
|
||||
"catalog_url": set_info.catalog_url,
|
||||
"artifact_url": member.url,
|
||||
"artifact_type": member.type,
|
||||
"digest": member.digest,
|
||||
"source_url": member.url,
|
||||
},
|
||||
)
|
||||
Loading…
Reference in New Issue