diff --git a/plugins/clone-app/skills/clone-app/scripts/scrape-play-store.py b/plugins/clone-app/skills/clone-app/scripts/scrape-play-store.py new file mode 100644 index 0000000..1646760 --- /dev/null +++ b/plugins/clone-app/skills/clone-app/scripts/scrape-play-store.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Scrape a Google Play store page into a metrics JSON object. + +Primary source is the embedded ld+json SoftwareApplication block (stable). +Falls back to light regex for installs/updated which aren't always in ld+json. +""" +import sys, json, re, argparse, urllib.request + +KEYS = ["package", "title", "rating", "rating_count", "installs", + "category", "developer", "updated", "source"] + +def fetch(package): + url = f"https://play.google.com/store/apps/details?id={package}&hl=en&gl=US" + req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) + with urllib.request.urlopen(req, timeout=30) as r: + return r.read().decode("utf-8", "replace") + +def parse(html, package): + out = {k: None for k in KEYS} + out["package"] = package + out["source"] = "google-play" + + # ld+json SoftwareApplication block + for m in re.finditer(r'', + html, re.DOTALL): + try: + data = json.loads(m.group(1).strip()) + except Exception: + continue + if isinstance(data, dict) and data.get("@type") == "SoftwareApplication": + out["title"] = data.get("name") + auth = data.get("author") + if isinstance(auth, dict): + out["developer"] = auth.get("name") + out["category"] = data.get("applicationCategory") + ar = data.get("aggregateRating") or {} + if ar.get("ratingValue") is not None: + try: out["rating"] = float(ar["ratingValue"]) + except Exception: pass + if ar.get("ratingCount") is not None: + try: out["rating_count"] = int(ar["ratingCount"]) + except Exception: pass + break + + # installs (e.g. "1,000,000+") + m = re.search(r'([\d,]+\+)\s*\s*Downloads', html) + if m: + out["installs"] = m.group(1) + + # updated date (e.g. "Updated onJun 1, 2026") + m = re.search(r'Updated on\s*([^<]+)', html) + if m: + out["updated"] = m.group(1).strip() + + return out + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("package") + ap.add_argument("--html-file") + args = ap.parse_args() + + if args.html_file: + with open(args.html_file, encoding="utf-8") as f: + html = f.read() + else: + try: + html = fetch(args.package) + except Exception as e: + print(f"ERROR: failed to fetch Play page: {e}", file=sys.stderr) + sys.exit(1) + + print(json.dumps(parse(html, args.package), indent=2)) + +if __name__ == "__main__": + main() diff --git a/plugins/clone-app/tests/fixtures/play-sample.html b/plugins/clone-app/tests/fixtures/play-sample.html new file mode 100644 index 0000000..309662c --- /dev/null +++ b/plugins/clone-app/tests/fixtures/play-sample.html @@ -0,0 +1,11 @@ + +Example App - Apps on Google Play + + +
Updated onJun 1, 2026
+
1,000,000+Downloads
+ diff --git a/plugins/clone-app/tests/test-scrape-play-store.py b/plugins/clone-app/tests/test-scrape-play-store.py new file mode 100644 index 0000000..80f37c5 --- /dev/null +++ b/plugins/clone-app/tests/test-scrape-play-store.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +import json, subprocess, sys, os + +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRIPT = os.path.join(HERE, "..", "skills", "clone-app", "scripts", "scrape-play-store.py") +FIXTURE = os.path.join(HERE, "fixtures", "play-sample.html") + +def run(): + out = subprocess.check_output( + [sys.executable, SCRIPT, "com.example.app", "--html-file", FIXTURE]) + return json.loads(out) + +def main(): + d = run() + fails = [] + def check(name, cond): + print(f"{'PASS' if cond else 'FAIL'}: {name}") + if not cond: fails.append(name) + check("package", d["package"] == "com.example.app") + check("title", d["title"] == "Example App") + check("rating", abs((d["rating"] or 0) - 4.3) < 0.001) + check("rating_count", d["rating_count"] == 12873) + check("developer", d["developer"] == "Example Studio") + check("category", d["category"] == "GAME_PUZZLE") + check("installs", d["installs"] == "1,000,000+") + check("source", d["source"] == "google-play") + # all expected keys present even if null + for k in ["package","title","rating","rating_count","installs","category","developer","updated","source"]: + check(f"key present: {k}", k in d) + sys.exit(1 if fails else 0) + +main()