feat(market-research): add play.google.com charts scraper

Replaces the AppBrain scraper (Cloudflare-blocked 403, confirmed dead).
play.google.com server-rendered chart HTML works: live-validated, real
Android packages + names + ratings from both /store/apps/top and
/store/apps/category/<CAT>. CLI: <top|category> [--category CAT]
[--region R] [--limit N] [--html-file F]; output source:"google-play",
entries {rank,name,package,rating}. Package link is the stable field;
title (Epkrse class) and rating are best-effort.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

plugins/market-research/skills/market-research/scripts/fetch-play-charts.py

@@ -1,20 +1,18 @@
 #!/usr/bin/env python3
-"""Fetch an AppBrain Play-store top-chart into a normalized JSON list.
+"""Fetch a play.google.com chart page into a normalized JSON list.
 
-Google Play has no public chart feed and renders via obfuscated batchexecute
+Google Play has no public chart feed, but its server-rendered HTML for
-JS. AppBrain (appbrain.com) publishes server-rendered HTML top-chart pages with
+/store/apps/top and /store/apps/category/<CAT> carries ranked app cards with
-REAL Android package names. This scrapes those rows. Stdlib-only, no pip.
+REAL Android package names (unlike Apple's RSS iOS bundle ids). Each card is an
+<a href="/store/apps/details?id=PKG"> wrapping the icon, then a
+<div class="Epkrse ">NAME</div> title, then a star aria-label. Play obfuscates
+the title class, so the package LINK (stable) is the load-bearing field; name
+and rating are best-effort. Stdlib-only, no pip.
 
-Unlike Apple's RSS (iOS bundle ids), entries here carry Android packages usable
+(AppBrain, the original plan's source, is Cloudflare-blocked 403 — dropped.)
-directly for a clone-app handoff.
 """
-import sys, json, re, argparse, urllib.request, ssl, subprocess, shutil
+import sys, json, re, html as _html, argparse, urllib.request, ssl, subprocess, shutil
 
-CHARTS = {
-    "popular": "https://www.appbrain.com/apps/popular/",
-    "top-grossing": "https://www.appbrain.com/apps/highest-grossing/",
-    "top-new": "https://www.appbrain.com/apps/new/",
-}
 UA = "Mozilla/5.0"
 
 def _http_get(url):
@@ -33,58 +31,66 @@ def _http_get(url):
             raise
         return out.stdout.decode("utf-8", "replace")
 
-# One app row: a link to /app/<slug>/<package> followed (within the same row
+LINK = re.compile(r'href="/store/apps/details\?id=([a-zA-Z0-9._]+)"')
-# block) by developer / category / rating / installs spans.
+NAME = re.compile(r'class="Epkrse[^"]*"[^>]*>([^<]+)<')
-ROW = re.compile(
+# rating is best-effort: search-style aria-label OR a star-icon followed by a number
-    r'href="/app/[^"/]+/(?P<package>[\w.]+)"[^>]*>(?P<name>[^<]+)</a>'
+RATE_ARIA = re.compile(r'aria-label="Rated\s+([\d.]+)\s+star')
-    r'(?P<rest>.*?)(?=href="/app/|</div>\s*</div>|$)', re.DOTALL)
+RATE_STAR = re.compile(r'>star</[^>]+>\s*([\d.]+)')
-DEV = re.compile(r'class="developer"[^>]*>([^<]+)<')
-CAT = re.compile(r'class="category"[^>]*>([^<]+)<')
-RAT = re.compile(r'class="rating"[^>]*>\s*([\d.]+)\s*<')
-INST = re.compile(r'class="installs"[^>]*>\s*([\d,]+\+)\s*<')
 
-def _first(rx, text):
+def _chart_url(chart, category, region):
-    m = rx.search(text)
+    if chart == "category":
-    return m.group(1).strip() if m else None
+        return f"https://play.google.com/store/apps/category/{category}?hl=en&gl={region}"
+    return f"https://play.google.com/store/apps/top?hl=en&gl={region}"
 
-def parse(html, chart, region, limit):
+def parse(html, chart_label, region, limit):
     entries = []
-    for i, m in enumerate(ROW.finditer(html), start=1):
+    seen = set()
-        if i > limit:
+    for m in LINK.finditer(html):
-            break
+        pkg = m.group(1)
-        rest = m.group("rest")
+        if pkg in seen:
-        rating = _first(RAT, rest)
+            continue
+        win = html[m.start():m.start() + 1600]   # one card's worth of markup
+        nm = NAME.search(win)
+        if not nm:
+            continue   # icon-only anchor with no title nearby; skip
+        seen.add(pkg)
+        rt = RATE_ARIA.search(win) or RATE_STAR.search(win)
         entries.append({
-            "rank": i,
+            "rank": len(entries) + 1,
-            "name": m.group("name").strip(),
+            "name": _html.unescape(nm.group(1).strip()),
-            "developer": _first(DEV, rest),
+            "package": pkg,
-            "category": _first(CAT, rest),
+            "rating": float(rt.group(1)) if rt else None,
-            "package": m.group("package"),
-            "rating": float(rating) if rating else None,
-            "installs": _first(INST, rest),
         })
-    return {"source": "appbrain", "chart": chart, "region": region,
+        if len(entries) >= limit:
+            break
+    return {"source": "google-play", "chart": chart_label, "region": region,
             "count": len(entries), "entries": entries}
 
 def main():
     ap = argparse.ArgumentParser()
-    ap.add_argument("chart", choices=sorted(CHARTS))
+    ap.add_argument("chart", choices=("top", "category"))
-    ap.add_argument("--region", default="us")
+    ap.add_argument("--category")
+    ap.add_argument("--region", default="US")
     ap.add_argument("--limit", type=int, default=25)
     ap.add_argument("--html-file")
     args = ap.parse_args()
 
+    if args.chart == "category" and not args.category:
+        print("ERROR: --category is required for the 'category' chart", file=sys.stderr)
+        sys.exit(2)
+    chart_label = f"category:{args.category}" if args.chart == "category" else "top"
+
     if args.html_file:
         with open(args.html_file, encoding="utf-8") as f:
             html = f.read()
     else:
         try:
-            html = _http_get(CHARTS[args.chart])
+            html = _http_get(_chart_url(args.chart, args.category, args.region))
         except Exception as e:
-            print(f"ERROR: failed to fetch AppBrain chart: {e}", file=sys.stderr)
+            print(f"ERROR: failed to fetch Play chart: {e}", file=sys.stderr)
             sys.exit(1)
 
-    print(json.dumps(parse(html, args.chart, args.region, args.limit), indent=2))
+    print(json.dumps(parse(html, chart_label, args.region, args.limit), indent=2))
 
 if __name__ == "__main__":
     main()

plugins/market-research/tests/fixtures/appbrain-popular.html

@@ -1,18 +0,0 @@
-<!DOCTYPE html><html><body>
-<div class="app-list">
-  <div class="app">
-    <a class="app-icon-name" href="/app/habit-tracker/com.example.habit">Habit Tracker</a>
-    <span class="developer">Focus Labs</span>
-    <span class="category">Productivity</span>
-    <span class="rating">4.6</span>
-    <span class="installs">5,000,000+</span>
-  </div>
-  <div class="app">
-    <a class="app-icon-name" href="/app/budget-buddy/com.example.budget">Budget Buddy</a>
-    <span class="developer">Money Inc</span>
-    <span class="category">Finance</span>
-    <span class="rating">4.2</span>
-    <span class="installs">1,000,000+</span>
-  </div>
-</div>
-</body></html>

plugins/market-research/tests/fixtures/play-chart.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html><html><body>
+<div role="main">
+  <a href="/store/apps/details?id=com.example.habit" jslog="38003; track:click"><div class="TjRVLb"><img src="https://play-lh.googleusercontent.com/x=s256"></div><div class="Epkrse ">Habit Tracker</div></a>
+  <div aria-label="Rated 4.6 stars out of five stars"></div>
+  <a href="/store/apps/details?id=com.example.budget" jslog="38003; track:click"><div class="TjRVLb"><img src="https://play-lh.googleusercontent.com/y=s256"></div><div class="Epkrse ">Budget Buddy</div></a>
+  <div aria-label="Rated 4.2 stars out of five stars"></div>
+  <a href="/store/apps/details?id=com.example.habit" jslog="38003; track:click"><div class="Epkrse ">Habit Tracker</div></a>
+</div>
+</body></html>

plugins/market-research/tests/test-fetch-play-charts.py

@@ -3,11 +3,11 @@ import json, subprocess, sys, os
 
 HERE = os.path.dirname(os.path.abspath(__file__))
 SCRIPT = os.path.join(HERE, "..", "skills", "market-research", "scripts", "fetch-play-charts.py")
-FIXTURE = os.path.join(HERE, "fixtures", "appbrain-popular.html")
+FIXTURE = os.path.join(HERE, "fixtures", "play-chart.html")
 
 def run():
     out = subprocess.check_output(
-        [sys.executable, SCRIPT, "popular", "--region", "us", "--html-file", FIXTURE])
+        [sys.executable, SCRIPT, "top", "--region", "US", "--html-file", FIXTURE])
     return json.loads(out)
 
 def main():
@@ -17,20 +17,19 @@ def main():
         print(f"{'PASS' if cond else 'FAIL'}: {name}")
         if not cond: fails.append(name)
 
-    check("source", d["source"] == "appbrain")
+    check("source", d["source"] == "google-play")
-    check("chart", d["chart"] == "popular")
+    check("chart", d["chart"] == "top")
-    check("count", d["count"] == 2)
+    check("region", d["region"] == "US")
+    check("count (dedup app1 repeat)", d["count"] == 2)
     e0 = d["entries"][0]
     check("rank 1", e0["rank"] == 1)
     check("name", e0["name"] == "Habit Tracker")
     check("package", e0["package"] == "com.example.habit")
-    check("developer", e0["developer"] == "Focus Labs")
-    check("category", e0["category"] == "Productivity")
     check("rating", e0["rating"] == 4.6)
-    check("installs", e0["installs"] == "5,000,000+")
+    for k in ["rank", "name", "package", "rating"]:
-    for k in ["rank", "name", "package", "developer", "category", "rating", "installs"]:
         check(f"key present: {k}", k in e0)
     check("second package", d["entries"][1]["package"] == "com.example.budget")
+    check("second name", d["entries"][1]["name"] == "Budget Buddy")
     sys.exit(1 if fails else 0)
 
 main()