feat(clone-app): add scrape-play-store.py with ld+json parsing and tests
This commit is contained in:
fatih.bulut
parent
052f0e5cad
commit
4b010a665e
|
|
@ -0,0 +1,76 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Scrape a Google Play store page into a metrics JSON object.
|
||||
|
||||
Primary source is the embedded ld+json SoftwareApplication block (stable).
|
||||
Falls back to light regex for installs/updated which aren't always in ld+json.
|
||||
"""
|
||||
import sys, json, re, argparse, urllib.request
|
||||
|
||||
KEYS = ["package", "title", "rating", "rating_count", "installs",
|
||||
"category", "developer", "updated", "source"]
|
||||
|
||||
def fetch(package):
|
||||
url = f"https://play.google.com/store/apps/details?id={package}&hl=en&gl=US"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.read().decode("utf-8", "replace")
|
||||
|
||||
def parse(html, package):
|
||||
out = {k: None for k in KEYS}
|
||||
out["package"] = package
|
||||
out["source"] = "google-play"
|
||||
|
||||
# ld+json SoftwareApplication block
|
||||
for m in re.finditer(r'<script type="application/ld\+json">(.*?)</script>',
|
||||
html, re.DOTALL):
|
||||
try:
|
||||
data = json.loads(m.group(1).strip())
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(data, dict) and data.get("@type") == "SoftwareApplication":
|
||||
out["title"] = data.get("name")
|
||||
auth = data.get("author")
|
||||
if isinstance(auth, dict):
|
||||
out["developer"] = auth.get("name")
|
||||
out["category"] = data.get("applicationCategory")
|
||||
ar = data.get("aggregateRating") or {}
|
||||
if ar.get("ratingValue") is not None:
|
||||
try: out["rating"] = float(ar["ratingValue"])
|
||||
except Exception: pass
|
||||
if ar.get("ratingCount") is not None:
|
||||
try: out["rating_count"] = int(ar["ratingCount"])
|
||||
except Exception: pass
|
||||
break
|
||||
|
||||
# installs (e.g. "1,000,000+")
|
||||
m = re.search(r'([\d,]+\+)\s*<span>\s*Downloads', html)
|
||||
if m:
|
||||
out["installs"] = m.group(1)
|
||||
|
||||
# updated date (e.g. "Updated on<span>Jun 1, 2026</span>")
|
||||
m = re.search(r'Updated on\s*<span>([^<]+)</span>', html)
|
||||
if m:
|
||||
out["updated"] = m.group(1).strip()
|
||||
|
||||
return out
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("package")
|
||||
ap.add_argument("--html-file")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.html_file:
|
||||
with open(args.html_file, encoding="utf-8") as f:
|
||||
html = f.read()
|
||||
else:
|
||||
try:
|
||||
html = fetch(args.package)
|
||||
except Exception as e:
|
||||
print(f"ERROR: failed to fetch Play page: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(json.dumps(parse(html, args.package), indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
<!DOCTYPE html>
|
||||
<html><head><title>Example App - Apps on Google Play</title>
|
||||
<script type="application/ld+json">
|
||||
{"@type":"SoftwareApplication","name":"Example App","author":{"name":"Example Studio"},
|
||||
"aggregateRating":{"ratingValue":"4.3","ratingCount":"12873"},
|
||||
"applicationCategory":"GAME_PUZZLE"}
|
||||
</script>
|
||||
</head><body>
|
||||
<div>Updated on<span>Jun 1, 2026</span></div>
|
||||
<div>1,000,000+<span>Downloads</span></div>
|
||||
</body></html>
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
#!/usr/bin/env python3
|
||||
import json, subprocess, sys, os
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
SCRIPT = os.path.join(HERE, "..", "skills", "clone-app", "scripts", "scrape-play-store.py")
|
||||
FIXTURE = os.path.join(HERE, "fixtures", "play-sample.html")
|
||||
|
||||
def run():
|
||||
out = subprocess.check_output(
|
||||
[sys.executable, SCRIPT, "com.example.app", "--html-file", FIXTURE])
|
||||
return json.loads(out)
|
||||
|
||||
def main():
|
||||
d = run()
|
||||
fails = []
|
||||
def check(name, cond):
|
||||
print(f"{'PASS' if cond else 'FAIL'}: {name}")
|
||||
if not cond: fails.append(name)
|
||||
check("package", d["package"] == "com.example.app")
|
||||
check("title", d["title"] == "Example App")
|
||||
check("rating", abs((d["rating"] or 0) - 4.3) < 0.001)
|
||||
check("rating_count", d["rating_count"] == 12873)
|
||||
check("developer", d["developer"] == "Example Studio")
|
||||
check("category", d["category"] == "GAME_PUZZLE")
|
||||
check("installs", d["installs"] == "1,000,000+")
|
||||
check("source", d["source"] == "google-play")
|
||||
# all expected keys present even if null
|
||||
for k in ["package","title","rating","rating_count","installs","category","developer","updated","source"]:
|
||||
check(f"key present: {k}", k in d)
|
||||
sys.exit(1 if fails else 0)
|
||||
|
||||
main()
|
||||
Loading…
Reference in New Issue