From fbfa3aece7b95c0f2904a86ca27e444ee2d3cc2a Mon Sep 17 00:00:00 2001 From: "fatih.bulut" Date: Tue, 23 Jun 2026 22:42:20 +0300 Subject: [PATCH] feat(market-research): add play.google.com charts scraper Replaces the AppBrain scraper (Cloudflare-blocked 403, confirmed dead). play.google.com server-rendered chart HTML works: live-validated, real Android packages + names + ratings from both /store/apps/top and /store/apps/category/. CLI: [--category CAT] [--region R] [--limit N] [--html-file F]; output source:"google-play", entries {rank,name,package,rating}. Package link is the stable field; title (Epkrse class) and rating are best-effort. Co-Authored-By: Claude Opus 4.8 --- .../scripts/fetch-play-charts.py | 92 ++++++++++--------- .../tests/fixtures/appbrain-popular.html | 18 ---- .../tests/fixtures/play-chart.html | 9 ++ .../tests/test-fetch-play-charts.py | 17 ++-- 4 files changed, 66 insertions(+), 70 deletions(-) delete mode 100644 plugins/market-research/tests/fixtures/appbrain-popular.html create mode 100644 plugins/market-research/tests/fixtures/play-chart.html diff --git a/plugins/market-research/skills/market-research/scripts/fetch-play-charts.py b/plugins/market-research/skills/market-research/scripts/fetch-play-charts.py index 5ca1cc6..80965b1 100755 --- a/plugins/market-research/skills/market-research/scripts/fetch-play-charts.py +++ b/plugins/market-research/skills/market-research/scripts/fetch-play-charts.py @@ -1,20 +1,18 @@ #!/usr/bin/env python3 -"""Fetch an AppBrain Play-store top-chart into a normalized JSON list. +"""Fetch a play.google.com chart page into a normalized JSON list. -Google Play has no public chart feed and renders via obfuscated batchexecute -JS. AppBrain (appbrain.com) publishes server-rendered HTML top-chart pages with -REAL Android package names. This scrapes those rows. Stdlib-only, no pip. +Google Play has no public chart feed, but its server-rendered HTML for +/store/apps/top and /store/apps/category/ carries ranked app cards with +REAL Android package names (unlike Apple's RSS iOS bundle ids). Each card is an + wrapping the icon, then a +
NAME
title, then a star aria-label. Play obfuscates +the title class, so the package LINK (stable) is the load-bearing field; name +and rating are best-effort. Stdlib-only, no pip. -Unlike Apple's RSS (iOS bundle ids), entries here carry Android packages usable -directly for a clone-app handoff. +(AppBrain, the original plan's source, is Cloudflare-blocked 403 — dropped.) """ -import sys, json, re, argparse, urllib.request, ssl, subprocess, shutil +import sys, json, re, html as _html, argparse, urllib.request, ssl, subprocess, shutil -CHARTS = { - "popular": "https://www.appbrain.com/apps/popular/", - "top-grossing": "https://www.appbrain.com/apps/highest-grossing/", - "top-new": "https://www.appbrain.com/apps/new/", -} UA = "Mozilla/5.0" def _http_get(url): @@ -33,58 +31,66 @@ def _http_get(url): raise return out.stdout.decode("utf-8", "replace") -# One app row: a link to /app// followed (within the same row -# block) by developer / category / rating / installs spans. -ROW = re.compile( - r'href="/app/[^"/]+/(?P[\w.]+)"[^>]*>(?P[^<]+)' - r'(?P.*?)(?=href="/app/|\s*|$)', re.DOTALL) -DEV = re.compile(r'class="developer"[^>]*>([^<]+)<') -CAT = re.compile(r'class="category"[^>]*>([^<]+)<') -RAT = re.compile(r'class="rating"[^>]*>\s*([\d.]+)\s*<') -INST = re.compile(r'class="installs"[^>]*>\s*([\d,]+\+)\s*<') +LINK = re.compile(r'href="/store/apps/details\?id=([a-zA-Z0-9._]+)"') +NAME = re.compile(r'class="Epkrse[^"]*"[^>]*>([^<]+)<') +# rating is best-effort: search-style aria-label OR a star-icon followed by a number +RATE_ARIA = re.compile(r'aria-label="Rated\s+([\d.]+)\s+star') +RATE_STAR = re.compile(r'>star]+>\s*([\d.]+)') -def _first(rx, text): - m = rx.search(text) - return m.group(1).strip() if m else None +def _chart_url(chart, category, region): + if chart == "category": + return f"https://play.google.com/store/apps/category/{category}?hl=en&gl={region}" + return f"https://play.google.com/store/apps/top?hl=en&gl={region}" -def parse(html, chart, region, limit): +def parse(html, chart_label, region, limit): entries = [] - for i, m in enumerate(ROW.finditer(html), start=1): - if i > limit: - break - rest = m.group("rest") - rating = _first(RAT, rest) + seen = set() + for m in LINK.finditer(html): + pkg = m.group(1) + if pkg in seen: + continue + win = html[m.start():m.start() + 1600] # one card's worth of markup + nm = NAME.search(win) + if not nm: + continue # icon-only anchor with no title nearby; skip + seen.add(pkg) + rt = RATE_ARIA.search(win) or RATE_STAR.search(win) entries.append({ - "rank": i, - "name": m.group("name").strip(), - "developer": _first(DEV, rest), - "category": _first(CAT, rest), - "package": m.group("package"), - "rating": float(rating) if rating else None, - "installs": _first(INST, rest), + "rank": len(entries) + 1, + "name": _html.unescape(nm.group(1).strip()), + "package": pkg, + "rating": float(rt.group(1)) if rt else None, }) - return {"source": "appbrain", "chart": chart, "region": region, + if len(entries) >= limit: + break + return {"source": "google-play", "chart": chart_label, "region": region, "count": len(entries), "entries": entries} def main(): ap = argparse.ArgumentParser() - ap.add_argument("chart", choices=sorted(CHARTS)) - ap.add_argument("--region", default="us") + ap.add_argument("chart", choices=("top", "category")) + ap.add_argument("--category") + ap.add_argument("--region", default="US") ap.add_argument("--limit", type=int, default=25) ap.add_argument("--html-file") args = ap.parse_args() + if args.chart == "category" and not args.category: + print("ERROR: --category is required for the 'category' chart", file=sys.stderr) + sys.exit(2) + chart_label = f"category:{args.category}" if args.chart == "category" else "top" + if args.html_file: with open(args.html_file, encoding="utf-8") as f: html = f.read() else: try: - html = _http_get(CHARTS[args.chart]) + html = _http_get(_chart_url(args.chart, args.category, args.region)) except Exception as e: - print(f"ERROR: failed to fetch AppBrain chart: {e}", file=sys.stderr) + print(f"ERROR: failed to fetch Play chart: {e}", file=sys.stderr) sys.exit(1) - print(json.dumps(parse(html, args.chart, args.region, args.limit), indent=2)) + print(json.dumps(parse(html, chart_label, args.region, args.limit), indent=2)) if __name__ == "__main__": main() diff --git a/plugins/market-research/tests/fixtures/appbrain-popular.html b/plugins/market-research/tests/fixtures/appbrain-popular.html deleted file mode 100644 index 861ef3c..0000000 --- a/plugins/market-research/tests/fixtures/appbrain-popular.html +++ /dev/null @@ -1,18 +0,0 @@ - -
-
- Habit Tracker - Focus Labs - Productivity - 4.6 - 5,000,000+ -
-
- Budget Buddy - Money Inc - Finance - 4.2 - 1,000,000+ -
-
- diff --git a/plugins/market-research/tests/fixtures/play-chart.html b/plugins/market-research/tests/fixtures/play-chart.html new file mode 100644 index 0000000..3ea3393 --- /dev/null +++ b/plugins/market-research/tests/fixtures/play-chart.html @@ -0,0 +1,9 @@ + +
+
Habit Tracker
 +
+
Budget Buddy
 +
+
Habit Tracker
 +
+ diff --git a/plugins/market-research/tests/test-fetch-play-charts.py b/plugins/market-research/tests/test-fetch-play-charts.py index 23d708a..29bdefa 100644 --- a/plugins/market-research/tests/test-fetch-play-charts.py +++ b/plugins/market-research/tests/test-fetch-play-charts.py @@ -3,11 +3,11 @@ import json, subprocess, sys, os HERE = os.path.dirname(os.path.abspath(__file__)) SCRIPT = os.path.join(HERE, "..", "skills", "market-research", "scripts", "fetch-play-charts.py") -FIXTURE = os.path.join(HERE, "fixtures", "appbrain-popular.html") +FIXTURE = os.path.join(HERE, "fixtures", "play-chart.html") def run(): out = subprocess.check_output( - [sys.executable, SCRIPT, "popular", "--region", "us", "--html-file", FIXTURE]) + [sys.executable, SCRIPT, "top", "--region", "US", "--html-file", FIXTURE]) return json.loads(out) def main(): @@ -17,20 +17,19 @@ def main(): print(f"{'PASS' if cond else 'FAIL'}: {name}") if not cond: fails.append(name) - check("source", d["source"] == "appbrain") - check("chart", d["chart"] == "popular") - check("count", d["count"] == 2) + check("source", d["source"] == "google-play") + check("chart", d["chart"] == "top") + check("region", d["region"] == "US") + check("count (dedup app1 repeat)", d["count"] == 2) e0 = d["entries"][0] check("rank 1", e0["rank"] == 1) check("name", e0["name"] == "Habit Tracker") check("package", e0["package"] == "com.example.habit") - check("developer", e0["developer"] == "Focus Labs") - check("category", e0["category"] == "Productivity") check("rating", e0["rating"] == 4.6) - check("installs", e0["installs"] == "5,000,000+") - for k in ["rank", "name", "package", "developer", "category", "rating", "installs"]: + for k in ["rank", "name", "package", "rating"]: check(f"key present: {k}", k in e0) check("second package", d["entries"][1]["package"] == "com.example.budget") + check("second name", d["entries"][1]["name"] == "Budget Buddy") sys.exit(1 if fails else 0) main()