From 537722bf65ecbe06eda0c6aa527a54009b9d0153 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:30:07 -0700 Subject: [PATCH] Port from code-yeongyu/oh-my-openagent#6662: blocked-page-recovery research skill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit omo's ultimate-browsing engine added a 'surrogate retrieval tier' (PR #6662): when a page fetch is blocked by a WAF/paywall/rate-limit, it falls back to third-party copies (Wayback, archive.today, Jina Reader) with strict provenance labeling and validators that reject fake successes (dead Google Cache interstitials, AMP redirect stubs, rate-limit bodies). Hermes adaptation: a bundled research skill + stdlib-only script instead of a Python sub-engine — zero core-tool footprint, per the footprint ladder. Clean-room implementation (their repo is Sustainable Use License; nothing copied), keeping the good ideas: provenance contract (snapshot vs live), body validation over status codes, domain rotation for archive.today, API-first pivot guidance, and explicit skip of proxy relays (MITM). E2E tested: recovered a real 486KB Wayback snapshot with timestamp; validators reject redirect stubs, interstitial titles, and sub-floor bodies. --- .../research/blocked-page-recovery/SKILL.md | 137 ++++++++++ .../scripts/recover_page.py | 241 ++++++++++++++++++ website/docs/reference/skills-catalog.md | 1 + .../research-blocked-page-recovery.md | 155 +++++++++++ website/sidebars.ts | 1 + 5 files changed, 535 insertions(+) create mode 100644 skills/research/blocked-page-recovery/SKILL.md create mode 100644 skills/research/blocked-page-recovery/scripts/recover_page.py create mode 100644 website/docs/user-guide/skills/bundled/research/research-blocked-page-recovery.md diff --git a/skills/research/blocked-page-recovery/SKILL.md b/skills/research/blocked-page-recovery/SKILL.md new file mode 100644 index 0000000000000..d5e5378350b00 --- /dev/null +++ b/skills/research/blocked-page-recovery/SKILL.md @@ -0,0 +1,137 @@ +--- +name: blocked-page-recovery +description: "Recover blocked/paywalled/WAF'd pages via archive snapshots and reader fallbacks. Use when web_extract or the browser hits 403/429/challenge pages, paywalls, or bot-detection interstitials." +version: 1.0.0 +author: Hermes Agent +license: MIT +platforms: [linux, macos, windows] +metadata: + hermes: + tags: [Research, Archives, Wayback, Paywall, WAF, Fallback] + related_skills: [grounded-citations] +--- + +# Blocked-Page Recovery + +When a page won't fetch — 403/429, Cloudflare "Just a moment...", a paywall, +or a bot-detection interstitial — don't give up and don't loop on the same +URL. Third-party services often hold a **copy** of the page. Work down this +ladder, cheapest first. + +## The ladder + +``` +1. Wayback Machine — archive.org "available" API (snapshot + timestamp) +2. archive.today — domain rotation: archive.ph → .md → .li → .is +3. Jina Reader — only if JINA_API_KEY is set (live server-side render) +4. API-first pivot — look for /api/, /graphql, .json, or RSS on the same host +5. Real browser — browser tool as the last, most expensive resort +``` + +Run it in one shot with the bundled script: + +```bash +python3 scripts/recover_page.py "https://example.com/blocked-article" --json +``` + +The script tries each route in order, validates every body (see "Fake +successes" below), and prints the first genuine hit with its provenance. + +## Provenance discipline (non-negotiable) + +Every recovered copy carries a provenance you MUST preserve when citing: + +| Route | Provenance | How to cite | +|-------|-----------|-------------| +| Wayback / archive.today | `snapshot` | Cite WITH the snapshot date: "as archived 2026-08-06". Never present a snapshot as the live page — it may be stale. | +| Jina Reader | `live` | Server-side re-render of the live page; cite normally. | +| Live fetch / browser | `live` | Cite normally. | + +If the user needs *current* data (prices, availability, breaking news), a +snapshot is context, not an answer — say so explicitly and note its age. + +## Manual routes + +### 1. Wayback Machine (best provenance, try first) + +```bash +# Discovery: returns closest snapshot URL + timestamp as JSON +curl -sL "https://archive.org/wayback/available?url={URL}" +# Then fetch archived_snapshots.closest.url +``` + +For enumerating many snapshots (or recovering deleted pages), the CDX index: + +```bash +curl -sL "https://web.archive.org/cdx/search/cdx?url={URL}&output=json&limit=10" +``` + +CDX intermittently returns 503 under load — if it does, fall back to the +`available` API; don't retry-hammer it. + +Works for: any publicly crawled URL. Fails for: robots-blocked sites, +never-crawled URLs, JS-only SPAs (snapshots don't render). + +### 2. archive.today (paywalls, deleted content) + +User-submitted archives — often has paywalled news articles Wayback lacks. +Rate-limits aggressively (429) and rotates domains, so iterate: + +```bash +for d in archive.ph archive.md archive.li archive.is; do + curl -sL --max-time 20 "https://$d/newest/{URL}" -o /tmp/page.html \ + -w "%{http_code}" && break +done +``` + +**Validate the body, not the status code** — a 429 still ships several KB of +rate-limit HTML that looks like a success to a size check alone. + +### 3. Jina Reader (requires JINA_API_KEY) + +`r.jina.ai` re-renders the live page in a real browser server-side and +returns markdown. Anonymous access is dead (401 → Turnstile); a key is +required: + +```bash +curl -s -H "Authorization: Bearer $JINA_API_KEY" "https://r.jina.ai/{URL}" +``` + +Handles JS SPAs that archives can't. Skip this route entirely when the env +var is unset. + +### 4. API-first pivot + +WAFs protect the HTML surface far more aggressively than the data endpoints +behind it. After 2-3 blocked attempts on a site, stop fighting the HTML and +look for: + +- `/api/...`, `/graphql`, or `.json` variants of the page URL +- An RSS/Atom feed (`/feed`, `/rss`, `` in any copy + you did recover) +- A sitemap (`/sitemap.xml`) revealing canonical URLs that may not be gated + +## Fake successes — routes that LIE + +These return HTTP 200 with a plausible body that is NOT the page. The script +rejects them automatically; reject them manually too: + +- **Google Cache is dead** (since mid-2024). `webcache.googleusercontent.com` + returns 200 + tens of KB, but it's a Google Search interstitial with a JS + redirect, not a cache. Never use it. +- **AMP caches** (`*.cdn.ampproject.org`) mostly return a ~300-byte + `Redirecting` meta-refresh stub pointing back at the + original (blocked) URL. Treating that as success creates a fetch loop. +- **Rate-limit bodies**: archive.today 429 pages are multi-KB HTML. Check for + the target's actual content (title words, expected strings), not just size. + +Detection heuristics the script applies: body under a per-route byte floor; +meta-refresh/JS-redirect stubs whose target is the original host; interstitial +titles ("Just a moment", "Redirecting", "Google Search", "Attention Required"). + +## Proxy relays: don't + +Generic "web proxy" relays are man-in-the-middle by construction. Never send +cookies or Authorization headers through one, and don't use them for anything +the user will rely on — provenance is unverifiable. Prefer archives, which at +least timestamp their copies. diff --git a/skills/research/blocked-page-recovery/scripts/recover_page.py b/skills/research/blocked-page-recovery/scripts/recover_page.py new file mode 100644 index 0000000000000..0b358f0b4c4b6 --- /dev/null +++ b/skills/research/blocked-page-recovery/scripts/recover_page.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""Recover a blocked / paywalled / WAF'd page from third-party copies. + +Ladder (cheapest first): + 1. Wayback Machine "available" API -> dated snapshot (provenance: snapshot) + 2. archive.today domain rotation -> dated snapshot (provenance: snapshot) + 3. Jina Reader (JINA_API_KEY only) -> live re-render (provenance: live) + +Every candidate body is validated before being declared a win: byte floors, +redirect-stub detection (meta-refresh/JS pointing back at the original host), +and interstitial-title rejection. Fake 200s are the norm in this space. + +Stdlib only. Usage: + python3 recover_page.py URL [--json] [--out FILE] [--timeout N] + +Exit codes: 0 recovered, 1 nothing worked, 2 bad invocation. +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +USER_AGENT = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/126.0 Safari/537.36" +) + +ARCHIVE_TODAY_HOSTS = ["archive.ph", "archive.md", "archive.li", "archive.is"] + +# Titles that mean "this is not the page you asked for". +INTERSTITIAL_TITLES = ( + "just a moment", + "redirecting", + "google search", + "attention required", + "access denied", + "are you a robot", + "one more step", +) + +# Below these floors a body is a stub or an error page, not content. +MIN_BODY_BYTES = {"wayback": 3072, "archive_today": 3072, "jina": 512} + +REDIRECT_STUB_RE = re.compile( + r'http-equiv=["\']?refresh|window\.location|location\.replace', re.IGNORECASE +) +TITLE_RE = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) + + +def _fetch( + url: str, + timeout: int, + headers: dict | None = None, + retries_on_429: int = 2, +) -> tuple[int, bytes]: + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT, **(headers or {})}) + for attempt in range(retries_on_429 + 1): + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.status, resp.read() + except urllib.error.HTTPError as exc: + if exc.code == 429 and attempt < retries_on_429: + time.sleep(5 * (attempt + 1)) + continue + return exc.code, exc.read() if exc.fp else b"" + except (urllib.error.URLError, OSError, ValueError): + return 0, b"" + return 0, b"" + + +def _fetch_follow(url: str, timeout: int) -> tuple[int, bytes, str]: + """Like _fetch but also returns the final URL after redirects.""" + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.status, resp.read(), resp.geturl() + except urllib.error.HTTPError as exc: + return exc.code, exc.read() if exc.fp else b"", exc.geturl() or url + except (urllib.error.URLError, OSError, ValueError): + return 0, b"", url + + +def _page_title(body: bytes) -> str: + m = TITLE_RE.search(body[:65536].decode("utf-8", "replace")) + return re.sub(r"\s+", " ", m.group(1)).strip().lower() if m else "" + + +def validate(body: bytes, route: str, target_url: str) -> str | None: + """Return a rejection reason, or None if the body looks like real content.""" + floor = MIN_BODY_BYTES.get(route, 3072) + if len(body) < floor: + return f"body_too_small:{len(body)}<{floor}" + title = _page_title(body) + for marker in INTERSTITIAL_TITLES: + if marker in title: + return f"interstitial_title:{marker!r}" + # Redirect stub: small-ish page whose only job is bouncing back to the + # original (blocked) host — the classic AMP-cache failure mode. + if len(body) < 8192 and REDIRECT_STUB_RE.search(body.decode("utf-8", "replace")): + target_host = urllib.parse.urlsplit(target_url).hostname or "" + if target_host and target_host.encode() in body: + return "redirect_stub_to_origin" + return None + + +def try_wayback(url: str, timeout: int) -> dict | None: + snap_url = None + snap_ts = None + discovery = "https://archive.org/wayback/available?url=" + urllib.parse.quote(url, safe="") + status, raw = _fetch(discovery, timeout) + if status == 200: + try: + closest = json.loads(raw).get("archived_snapshots", {}).get("closest", {}) + except (json.JSONDecodeError, AttributeError): + closest = {} + if closest.get("available") and closest.get("url"): + snap_url = closest["url"].replace( + "http://web.archive.org", "https://web.archive.org" + ) + snap_ts = closest.get("timestamp") + if snap_url is None: + # Discovery API is rate-limited far more aggressively than snapshot + # serving. Fall back to the redirect form: /web/2/ bounces to + # the newest snapshot if one exists (404 page otherwise). + snap_url = "https://web.archive.org/web/2/" + url + status, body, final_url = _fetch_follow(snap_url, timeout) + if status != 200 or validate(body, "wayback", url): + return None + if snap_ts is None: + m = re.search(r"/web/(\d{14})", final_url) + snap_ts = m.group(1) if m else None + return { + "route": "wayback", + "provenance": "snapshot", + "snapshot_timestamp": snap_ts, + "source_url": final_url, + "body": body, + } + + +def try_archive_today(url: str, timeout: int) -> dict | None: + for host in ARCHIVE_TODAY_HOSTS: + fetch_url = f"https://{host}/newest/{url}" + status, body = _fetch(fetch_url, timeout) + if status != 200: + continue + if validate(body, "archive_today", url): + continue # 429 bodies and interstitials land here + return { + "route": f"archive_today:{host}", + "provenance": "snapshot", + "snapshot_timestamp": None, # archive.today embeds the date in-page + "source_url": fetch_url, + "body": body, + } + return None + + +def try_jina(url: str, timeout: int) -> dict | None: + key = os.environ.get("JINA_API_KEY") + if not key: + return None + status, body = _fetch( + "https://r.jina.ai/" + url, timeout, headers={"Authorization": f"Bearer {key}"} + ) + if status != 200 or validate(body, "jina", url): + return None + return { + "route": "jina_reader", + "provenance": "live", + "snapshot_timestamp": None, + "source_url": "https://r.jina.ai/" + url, + "body": body, + } + + +ROUTES = (try_wayback, try_archive_today, try_jina) + + +def recover(url: str, timeout: int = 25) -> dict | None: + for route_fn in ROUTES: + result = route_fn(url, timeout) + if result: + return result + return None + + +def main() -> int: + ap = argparse.ArgumentParser( + description="Recover a blocked / paywalled / WAF'd page from third-party copies." + ) + ap.add_argument("url") + ap.add_argument("--json", action="store_true", help="print metadata as JSON") + ap.add_argument("--out", help="write recovered body to this file") + ap.add_argument("--timeout", type=int, default=25) + args = ap.parse_args() + + if not args.url.startswith(("http://", "https://")): + print("error: URL must start with http:// or https://", file=sys.stderr) + return 2 + + result = recover(args.url, args.timeout) + if not result: + msg = {"recovered": False, "url": args.url, + "hint": "No archive copy found. Try the API-first pivot or the browser tool."} + print(json.dumps(msg, indent=2) if args.json else msg["hint"], file=sys.stderr) + return 1 + + body = result.pop("body") + result.update({"recovered": True, "url": args.url, "body_bytes": len(body)}) + if args.out: + with open(args.out, "wb") as fh: + fh.write(body) + result["saved_to"] = args.out + + if args.json: + print(json.dumps(result, indent=2)) + else: + for k, v in result.items(): + print(f"{k}: {v}") + if not args.out: + print("\n--- body (first 2000 chars) ---") + print(body[:2000].decode("utf-8", "replace")) + if result["provenance"] == "snapshot": + print( + "\nNOTE: this is an ARCHIVED SNAPSHOT, not the live page. " + "Cite it with its timestamp.", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/website/docs/reference/skills-catalog.md b/website/docs/reference/skills-catalog.md index fba5674e0a5d1..9b5c217094d16 100644 --- a/website/docs/reference/skills-catalog.md +++ b/website/docs/reference/skills-catalog.md @@ -120,6 +120,7 @@ If a skill is missing from this list but present in the repo, the catalog is reg | Skill | Description | Path | |-------|-------------|------| | [`arxiv`](/docs/user-guide/skills/bundled/research/research-arxiv) | Search arXiv papers by keyword, author, category, or ID. | `research/arxiv` | +| [`blocked-page-recovery`](/docs/user-guide/skills/bundled/research/research-blocked-page-recovery) | Recover blocked/paywalled/WAF'd pages via archive snapshots and reader fallbacks. Use when web_extract or the browser hits 403/429/challenge pages, paywalls, or bot-detection interstitials. | `research/blocked-page-recovery` | | [`blogwatcher`](/docs/user-guide/skills/bundled/research/research-blogwatcher) | Monitor blogs and RSS/Atom feeds via blogwatcher-cli tool. | `research/blogwatcher` | | [`competitor-news-monitor`](/docs/user-guide/skills/bundled/research/research-competitor-news-monitor) | Watch named companies for material news; cited digests. | `research/competitor-news-monitor` | | [`grounded-citations`](/docs/user-guide/skills/bundled/research/research-grounded-citations) | Ground answers and documents in cited, verifiable sources. | `research/grounded-citations` | diff --git a/website/docs/user-guide/skills/bundled/research/research-blocked-page-recovery.md b/website/docs/user-guide/skills/bundled/research/research-blocked-page-recovery.md new file mode 100644 index 0000000000000..60ab28f809b45 --- /dev/null +++ b/website/docs/user-guide/skills/bundled/research/research-blocked-page-recovery.md @@ -0,0 +1,155 @@ +--- +title: "Blocked Page Recovery — Recover blocked/paywalled/WAF'd pages via archive snapshots and reader fallbacks" +sidebar_label: "Blocked Page Recovery" +description: "Recover blocked/paywalled/WAF'd pages via archive snapshots and reader fallbacks" +--- + +{/* This page is auto-generated from the skill's SKILL.md by website/scripts/generate-skill-docs.py. Edit the source SKILL.md, not this page. */} + +# Blocked Page Recovery + +Recover blocked/paywalled/WAF'd pages via archive snapshots and reader fallbacks. Use when web_extract or the browser hits 403/429/challenge pages, paywalls, or bot-detection interstitials. + +## Skill metadata + +| | | +|---|---| +| Source | Bundled (installed by default) | +| Path | `skills/research/blocked-page-recovery` | +| Version | `1.0.0` | +| Author | Hermes Agent | +| License | MIT | +| Platforms | linux, macos, windows | +| Tags | `Research`, `Archives`, `Wayback`, `Paywall`, `WAF`, `Fallback` | +| Related skills | [`grounded-citations`](/docs/user-guide/skills/bundled/research/research-grounded-citations) | + +## Reference: full SKILL.md + +:::info +The following is the complete skill definition that Hermes loads when this skill is triggered. This is what the agent sees as instructions when the skill is active. +::: + +# Blocked-Page Recovery + +When a page won't fetch — 403/429, Cloudflare "Just a moment...", a paywall, +or a bot-detection interstitial — don't give up and don't loop on the same +URL. Third-party services often hold a **copy** of the page. Work down this +ladder, cheapest first. + +## The ladder + +``` +1. Wayback Machine — archive.org "available" API (snapshot + timestamp) +2. archive.today — domain rotation: archive.ph → .md → .li → .is +3. Jina Reader — only if JINA_API_KEY is set (live server-side render) +4. API-first pivot — look for /api/, /graphql, .json, or RSS on the same host +5. Real browser — browser tool as the last, most expensive resort +``` + +Run it in one shot with the bundled script: + +```bash +python3 scripts/recover_page.py "https://example.com/blocked-article" --json +``` + +The script tries each route in order, validates every body (see "Fake +successes" below), and prints the first genuine hit with its provenance. + +## Provenance discipline (non-negotiable) + +Every recovered copy carries a provenance you MUST preserve when citing: + +| Route | Provenance | How to cite | +|-------|-----------|-------------| +| Wayback / archive.today | `snapshot` | Cite WITH the snapshot date: "as archived 2026-08-06". Never present a snapshot as the live page — it may be stale. | +| Jina Reader | `live` | Server-side re-render of the live page; cite normally. | +| Live fetch / browser | `live` | Cite normally. | + +If the user needs *current* data (prices, availability, breaking news), a +snapshot is context, not an answer — say so explicitly and note its age. + +## Manual routes + +### 1. Wayback Machine (best provenance, try first) + +```bash +# Discovery: returns closest snapshot URL + timestamp as JSON +curl -sL "https://archive.org/wayback/available?url={URL}" +# Then fetch archived_snapshots.closest.url +``` + +For enumerating many snapshots (or recovering deleted pages), the CDX index: + +```bash +curl -sL "https://web.archive.org/cdx/search/cdx?url={URL}&output=json&limit=10" +``` + +CDX intermittently returns 503 under load — if it does, fall back to the +`available` API; don't retry-hammer it. + +Works for: any publicly crawled URL. Fails for: robots-blocked sites, +never-crawled URLs, JS-only SPAs (snapshots don't render). + +### 2. archive.today (paywalls, deleted content) + +User-submitted archives — often has paywalled news articles Wayback lacks. +Rate-limits aggressively (429) and rotates domains, so iterate: + +```bash +for d in archive.ph archive.md archive.li archive.is; do + curl -sL --max-time 20 "https://$d/newest/{URL}" -o /tmp/page.html \ + -w "%{http_code}" && break +done +``` + +**Validate the body, not the status code** — a 429 still ships several KB of +rate-limit HTML that looks like a success to a size check alone. + +### 3. Jina Reader (requires JINA_API_KEY) + +`r.jina.ai` re-renders the live page in a real browser server-side and +returns markdown. Anonymous access is dead (401 → Turnstile); a key is +required: + +```bash +curl -s -H "Authorization: Bearer $JINA_API_KEY" "https://r.jina.ai/{URL}" +``` + +Handles JS SPAs that archives can't. Skip this route entirely when the env +var is unset. + +### 4. API-first pivot + +WAFs protect the HTML surface far more aggressively than the data endpoints +behind it. After 2-3 blocked attempts on a site, stop fighting the HTML and +look for: + +- `/api/...`, `/graphql`, or `.json` variants of the page URL +- An RSS/Atom feed (`/feed`, `/rss`, `` in any copy + you did recover) +- A sitemap (`/sitemap.xml`) revealing canonical URLs that may not be gated + +## Fake successes — routes that LIE + +These return HTTP 200 with a plausible body that is NOT the page. The script +rejects them automatically; reject them manually too: + +- **Google Cache is dead** (since mid-2024). `webcache.googleusercontent.com` + returns 200 + tens of KB, but it's a Google Search interstitial with a JS + redirect, not a cache. Never use it. +- **AMP caches** (`*.cdn.ampproject.org`) mostly return a ~300-byte + `Redirecting` meta-refresh stub pointing back at the + original (blocked) URL. Treating that as success creates a fetch loop. +- **Rate-limit bodies**: archive.today 429 pages are multi-KB HTML. Check for + the target's actual content (title words, expected strings), not just size. + +Detection heuristics the script applies: body under a per-route byte floor; +meta-refresh/JS-redirect stubs whose target is the original host; interstitial +titles ("Just a moment", "Redirecting", "Google Search", "Attention Required"). + +## Proxy relays: don't + +Generic "web proxy" relays are man-in-the-middle by construction. Never send +cookies or Authorization headers through one, and don't use them for anything +the user will rely on — provenance is unverifiable. Prefer archives, which at +least timestamp their copies. diff --git a/website/sidebars.ts b/website/sidebars.ts index be7c116ccff11..26f9f71c1a292 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -283,6 +283,7 @@ const sidebars: SidebarsConfig = { collapsed: true, items: [ 'user-guide/skills/bundled/research/research-arxiv', + 'user-guide/skills/bundled/research/research-blocked-page-recovery', 'user-guide/skills/bundled/research/research-blogwatcher', 'user-guide/skills/bundled/research/research-competitor-news-monitor', 'user-guide/skills/bundled/research/research-grounded-citations',