From b34632ef3babede36b0d1ede61da5a2db066ecf6 Mon Sep 17 00:00:00 2001 From: "fatih.bulut" Date: Sun, 21 Jun 2026 19:19:54 +0300 Subject: [PATCH] feat(clone-app): download APKs via apkeep instead of APKCombo scraping Replace the hand-rolled two-step APKCombo curl/grep flow with apkeep (default source apk-pure): no auth, no JavaScript, handles XAPK split bundling and retries itself, and the per-mirror HTML parsing is no longer ours to maintain. The script still prints a stable app.apk/app.xapk path. - download-apk.sh: invoke apkeep, rename artifact to app., clear error when the binary is missing; CLONE_APP_APKEEP (test stub) and CLONE_APP_APKEEP_SOURCE (mirror override) env hooks - test-download-apk.sh: rewritten against an apkeep stub (xapk/apk/fail/ missing-binary cases) - SKILL.md Phase 1: prose updated for apkeep install + source override Co-Authored-By: Claude Opus 4.8 --- plugins/clone-app/skills/clone-app/SKILL.md | 12 +-- .../skills/clone-app/scripts/download-apk.sh | 89 ++++++------------- plugins/clone-app/tests/test-download-apk.sh | 84 ++++++++--------- 3 files changed, 74 insertions(+), 111 deletions(-) diff --git a/plugins/clone-app/skills/clone-app/SKILL.md b/plugins/clone-app/skills/clone-app/SKILL.md index 7980b97..f949bca 100644 --- a/plugins/clone-app/skills/clone-app/SKILL.md +++ b/plugins/clone-app/skills/clone-app/SKILL.md @@ -34,11 +34,13 @@ Create the working dir: `WORK="./work/$PKG"` and `mkdir -p "$WORK"`. ```bash APK="$(bash ${CLAUDE_PLUGIN_ROOT}/skills/clone-app/scripts/download-apk.sh "$PKG" "$WORK")" ``` -The script downloads from APKCombo (the old APKPure direct endpoint is now behind -a Cloudflare bot challenge), retries 3×, and prints the path (`app.apk` or -`app.xapk`). If it exits non-zero, the app may not be on APKCombo or the page -format changed — tell the user the download failed and ask for a local APK/XAPK -path; set `APK` to that path. +The script downloads via `apkeep` (default source `apk-pure`; no auth, no +JavaScript, handles XAPK split bundling) and prints the path (`app.apk` or +`app.xapk`). It needs the `apkeep` binary on PATH — install with +`brew install apkeep` (or `cargo install apkeep`). If it exits non-zero, +`apkeep` may be missing or the app isn't on that source (try another with +`CLONE_APP_APKEEP_SOURCE=apk-combo`) — tell the user the download failed and +ask for a local APK/XAPK path; set `APK` to that path. ## Phase 2: Reverse Engineering (probe → dispatch → consume) diff --git a/plugins/clone-app/skills/clone-app/scripts/download-apk.sh b/plugins/clone-app/skills/clone-app/scripts/download-apk.sh index b4296d0..f40310c 100755 --- a/plugins/clone-app/skills/clone-app/scripts/download-apk.sh +++ b/plugins/clone-app/skills/clone-app/scripts/download-apk.sh @@ -3,86 +3,55 @@ set -euo pipefail # Download an APK/XAPK for a package by its Android package name. # -# Source: APKCombo. The historical APKPure endpoint -# (d.apkpure.com/b/APK/?version=latest) is now behind a Cloudflare -# bot challenge and returns HTTP 403 "Just a moment..." for every package, -# so a plain curl can no longer use it. +# Source: apkeep (https://github.com/EFForg/apkeep), default download source +# apk-pure. apkeep is a maintained Rust CLI that handles the mirror's HTML, +# Cloudflare User-Agent dance, retries, and split-APK (XAPK) bundling for us — +# no auth, no JavaScript, no per-mirror scraping to maintain. Install with +# `brew install apkeep` (or `cargo install apkeep`). # -# APKCombo serves the real artifact in two steps, both reachable with a -# normal browser User-Agent and no JavaScript: -# 1. GET https://apkcombo.com/app//download/apk -> an HTML page that -# embeds the signed download as a relative /r2?u= -# link (the URL slug segment is ignored by the server, so a fixed "app" -# works for any package). -# 2. GET https://apkcombo.com -> the apk/xapk bytes. +# apkeep writes ".apk" or ".xapk" into the output dir; we +# rename it to "app." so the rest of the skill sees a stable path. # -# CLONE_APP_CURL overrides the curl binary (used by the tests to inject a stub). +# CLONE_APP_APKEEP overrides the apkeep binary (used by the tests to inject a +# stub). CLONE_APP_APKEEP_SOURCE overrides the download source (default apk-pure). package="${1:-}" out_dir="${2:-}" -curl_bin="${CLONE_APP_CURL:-curl}" - -UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36" -REFERER="https://apkcombo.com/" -page_url="https://apkcombo.com/app/${package}/download/apk" +apkeep_bin="${CLONE_APP_APKEEP:-apkeep}" +source="${CLONE_APP_APKEEP_SOURCE:-apk-pure}" if [[ -z "$package" || -z "$out_dir" ]]; then echo "ERROR: usage: download-apk.sh " >&2 exit 1 fi +if ! command -v "$apkeep_bin" >/dev/null 2>&1; then + echo "ERROR: apkeep not found. Install it with 'brew install apkeep'" >&2 + echo " (or 'cargo install apkeep'), then re-run." >&2 + exit 1 +fi + mkdir -p "$out_dir" -page_file="$out_dir/download-page.html" -tmp_file="$out_dir/app.download" -# --- Step 1: fetch the download page and extract the signed /r2 link --------- -r2_path="" -for attempt in 1 2 3; do - if "$curl_bin" -sL -A "$UA" -H "Referer: $REFERER" "$page_url" --output "$page_file" 2>/dev/null; then - r2_path="$(grep -oE '/r2\?u=[^"]*' "$page_file" | head -1 || true)" - [[ -n "$r2_path" ]] && break - fi - echo "download-page attempt $attempt failed (or no link yet), retrying..." >&2 - sleep 1 -done -rm -f "$page_file" - -if [[ -z "$r2_path" ]]; then - echo "ERROR: no download link found for '$package' on APKCombo ($page_url)." >&2 - echo " The app may not be available there, or the page format changed." >&2 +# apkeep handles its own retries; run it once and check the result. +if ! "$apkeep_bin" -a "$package" -d "$source" "$out_dir" >&2; then + echo "ERROR: apkeep failed to download '$package' from $source." >&2 + echo " The app may not be available there; try another -d source." >&2 exit 1 fi -# Links in the HTML use raw '&'; decode '&' defensively in case that changes. -r2_path="${r2_path//&/&}" -artifact_url="https://apkcombo.com${r2_path}" - -# --- Step 2: download the actual artifact ------------------------------------ -ok=0 -for attempt in 1 2 3; do - if "$curl_bin" -sL --fail -A "$UA" -H "Referer: $REFERER" "$artifact_url" --output "$tmp_file" 2>/dev/null; then - ok=1; break - fi - echo "artifact download attempt $attempt failed, retrying..." >&2 - sleep 1 +# apkeep names the artifact after the package; find whichever it produced. +artifact="" +for cand in "$out_dir/$package.xapk" "$out_dir/$package.apk"; do + [[ -f "$cand" ]] && { artifact="$cand"; break; } done -if [[ "$ok" -ne 1 ]]; then - echo "ERROR: failed to download $package artifact after 3 attempts." >&2 - rm -f "$tmp_file" +if [[ -z "$artifact" ]]; then + echo "ERROR: apkeep reported success but no artifact found in $out_dir." >&2 exit 1 fi -# Decide extension: both APK and XAPK are ZIPs. XAPK = a zip bundle containing -# manifest.json plus at least one split .apk entry; a plain APK has neither. -ext="apk" -if entries="$(unzip -Z1 "$tmp_file" 2>/dev/null)"; then - apk_count="$(grep -c '\.apk$' <<<"$entries" || true)" - if grep -q '^manifest\.json$' <<<"$entries" && [[ "$apk_count" -ge 1 ]]; then - ext="xapk" - fi -fi - +ext="${artifact##*.}" final="$out_dir/app.$ext" -mv "$tmp_file" "$final" +mv -f "$artifact" "$final" echo "$final" diff --git a/plugins/clone-app/tests/test-download-apk.sh b/plugins/clone-app/tests/test-download-apk.sh index eda5ed6..2ab22b3 100644 --- a/plugins/clone-app/tests/test-download-apk.sh +++ b/plugins/clone-app/tests/test-download-apk.sh @@ -6,62 +6,54 @@ check() { [[ "$2" == "$3" ]] && echo "PASS: $1" || { echo "FAIL: $1 — expected tmp="$(mktemp -d)" -# Find the --output target in a curl arg list, and the request URL (http... arg). -# Shared helper sourced by the stubs below. -cat > "$tmp/argparse.sh" <<'EOF' -out=""; url=""; prev="" +# Stub apkeep modelling the real CLI: +# apkeep -a -d +# parse out the package and the out_dir (last arg), then drop an artifact there. +# CLONE_APP_FAKE_EXT picks which extension the stub produces; "fail" makes it +# exit non-zero without writing anything. +cat > "$tmp/fakeapkeep.sh" <<'EOF' +#!/usr/bin/env bash +pkg=""; outdir=""; prev="" for a in "$@"; do - [[ "$prev" == "--output" || "$prev" == "-o" ]] && out="$a" - [[ "$a" == http*://* ]] && url="$a" + [[ "$prev" == "-a" || "$prev" == "--app" ]] && pkg="$a" + outdir="$a" # last positional wins; apkeep takes OUTPATH last prev="$a" done +case "$CLONE_APP_FAKE_EXT" in + fail) exit 1 ;; + xapk) + workdir="$(mktemp -d)" + echo '{}' > "$workdir/manifest.json" + echo 'a' > "$workdir/base.apk"; echo 'b' > "$workdir/config.apk" + ( cd "$workdir" && zip -q -r "$outdir/$pkg.xapk" . ) ;; + *) echo 'apk-bytes' > "$outdir/$pkg.apk" ;; +esac +exit 0 EOF +chmod +x "$tmp/fakeapkeep.sh" -# Stub curl modelling the real two-step APKCombo flow: -# call 1 (URL contains /download/apk) -> emit HTML page with an /r2?u= link -# call 2 (URL contains /r2?u=) -> emit a zip (manifest.json + 2 apk) => xapk -cat > "$tmp/fakecurl-xapk.sh" <Download' > "\$out" - exit 0 -elif [[ "\$url" == *"/r2?u="* ]]; then - workdir="\$(mktemp -d)" - echo '{}' > "\$workdir/manifest.json" - echo 'a' > "\$workdir/base.apk"; echo 'b' > "\$workdir/config.apk" - ( cd "\$workdir" && zip -q -r "\$out" . ) - exit 0 -fi -exit 1 -EOF -chmod +x "$tmp/fakecurl-xapk.sh" - -path="$(CLONE_APP_CURL="$tmp/fakecurl-xapk.sh" bash "$SCRIPT" com.example.app "$tmp/out" 2>/dev/null)"; rc=$? +# XAPK case +path="$(CLONE_APP_APKEEP="$tmp/fakeapkeep.sh" CLONE_APP_FAKE_EXT=xapk \ + bash "$SCRIPT" com.example.app "$tmp/out" 2>/dev/null)"; rc=$? check "xapk exit 0" "0" "$rc" check "xapk extension" "xapk" "${path##*.}" +check "xapk renamed to app.*" "app.xapk" "$(basename "$path")" -# Stub where the download page loads but contains NO /r2 link -> exit 1 -cat > "$tmp/fakecurl-nolink.sh" <no link here' > "\$out"; exit 0 -fi -exit 1 -EOF -chmod +x "$tmp/fakecurl-nolink.sh" -out2="$(CLONE_APP_CURL="$tmp/fakecurl-nolink.sh" bash "$SCRIPT" com.example.app "$tmp/out2" 2>/dev/null)"; rc2=$? -check "no-link exit 1" "1" "$rc2" +# APK case +path2="$(CLONE_APP_APKEEP="$tmp/fakeapkeep.sh" CLONE_APP_FAKE_EXT=apk \ + bash "$SCRIPT" com.example.app "$tmp/out-apk" 2>/dev/null)"; rc2=$? +check "apk exit 0" "0" "$rc2" +check "apk extension" "apk" "${path2##*.}" -# Stub curl that always fails -> exit 1 after retries -cat > "$tmp/fakecurl-fail.sh" <<'EOF' -#!/usr/bin/env bash -exit 22 -EOF -chmod +x "$tmp/fakecurl-fail.sh" -out3="$(CLONE_APP_CURL="$tmp/fakecurl-fail.sh" bash "$SCRIPT" com.example.app "$tmp/out3" 2>/dev/null)"; rc3=$? +# apkeep failure -> exit 1 +out3="$(CLONE_APP_APKEEP="$tmp/fakeapkeep.sh" CLONE_APP_FAKE_EXT=fail \ + bash "$SCRIPT" com.example.app "$tmp/out3" 2>/dev/null)"; rc3=$? check "fail exit 1" "1" "$rc3" +# apkeep binary missing -> exit 1 +out4="$(CLONE_APP_APKEEP="$tmp/does-not-exist-apkeep" \ + bash "$SCRIPT" com.example.app "$tmp/out4" 2>/dev/null)"; rc4=$? +check "missing-binary exit 1" "1" "$rc4" + rm -rf "$tmp" exit $fail