feat(clone-app): download APKs via apkeep instead of APKCombo scraping
Replace the hand-rolled two-step APKCombo curl/grep flow with apkeep (default source apk-pure): no auth, no JavaScript, handles XAPK split bundling and retries itself, and the per-mirror HTML parsing is no longer ours to maintain. The script still prints a stable app.apk/app.xapk path. - download-apk.sh: invoke apkeep, rename artifact to app.<ext>, clear error when the binary is missing; CLONE_APP_APKEEP (test stub) and CLONE_APP_APKEEP_SOURCE (mirror override) env hooks - test-download-apk.sh: rewritten against an apkeep stub (xapk/apk/fail/ missing-binary cases) - SKILL.md Phase 1: prose updated for apkeep install + source override Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
fatih.bulut
parent
adf93bacd1
commit
b34632ef3b
|
|
@ -34,11 +34,13 @@ Create the working dir: `WORK="./work/$PKG"` and `mkdir -p "$WORK"`.
|
|||
```bash
|
||||
APK="$(bash ${CLAUDE_PLUGIN_ROOT}/skills/clone-app/scripts/download-apk.sh "$PKG" "$WORK")"
|
||||
```
|
||||
The script downloads from APKCombo (the old APKPure direct endpoint is now behind
|
||||
a Cloudflare bot challenge), retries 3×, and prints the path (`app.apk` or
|
||||
`app.xapk`). If it exits non-zero, the app may not be on APKCombo or the page
|
||||
format changed — tell the user the download failed and ask for a local APK/XAPK
|
||||
path; set `APK` to that path.
|
||||
The script downloads via `apkeep` (default source `apk-pure`; no auth, no
|
||||
JavaScript, handles XAPK split bundling) and prints the path (`app.apk` or
|
||||
`app.xapk`). It needs the `apkeep` binary on PATH — install with
|
||||
`brew install apkeep` (or `cargo install apkeep`). If it exits non-zero,
|
||||
`apkeep` may be missing or the app isn't on that source (try another with
|
||||
`CLONE_APP_APKEEP_SOURCE=apk-combo`) — tell the user the download failed and
|
||||
ask for a local APK/XAPK path; set `APK` to that path.
|
||||
|
||||
## Phase 2: Reverse Engineering (probe → dispatch → consume)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,86 +3,55 @@ set -euo pipefail
|
|||
|
||||
# Download an APK/XAPK for a package by its Android package name.
|
||||
#
|
||||
# Source: APKCombo. The historical APKPure endpoint
|
||||
# (d.apkpure.com/b/APK/<pkg>?version=latest) is now behind a Cloudflare
|
||||
# bot challenge and returns HTTP 403 "Just a moment..." for every package,
|
||||
# so a plain curl can no longer use it.
|
||||
# Source: apkeep (https://github.com/EFForg/apkeep), default download source
|
||||
# apk-pure. apkeep is a maintained Rust CLI that handles the mirror's HTML,
|
||||
# Cloudflare User-Agent dance, retries, and split-APK (XAPK) bundling for us —
|
||||
# no auth, no JavaScript, no per-mirror scraping to maintain. Install with
|
||||
# `brew install apkeep` (or `cargo install apkeep`).
|
||||
#
|
||||
# APKCombo serves the real artifact in two steps, both reachable with a
|
||||
# normal browser User-Agent and no JavaScript:
|
||||
# 1. GET https://apkcombo.com/app/<pkg>/download/apk -> an HTML page that
|
||||
# embeds the signed download as a relative /r2?u=<url-encoded-signed-url>
|
||||
# link (the URL slug segment is ignored by the server, so a fixed "app"
|
||||
# works for any package).
|
||||
# 2. GET https://apkcombo.com<that /r2 path> -> the apk/xapk bytes.
|
||||
# apkeep writes "<package>.apk" or "<package>.xapk" into the output dir; we
|
||||
# rename it to "app.<ext>" so the rest of the skill sees a stable path.
|
||||
#
|
||||
# CLONE_APP_CURL overrides the curl binary (used by the tests to inject a stub).
|
||||
# CLONE_APP_APKEEP overrides the apkeep binary (used by the tests to inject a
|
||||
# stub). CLONE_APP_APKEEP_SOURCE overrides the download source (default apk-pure).
|
||||
|
||||
package="${1:-}"
|
||||
out_dir="${2:-}"
|
||||
curl_bin="${CLONE_APP_CURL:-curl}"
|
||||
|
||||
UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36"
|
||||
REFERER="https://apkcombo.com/"
|
||||
page_url="https://apkcombo.com/app/${package}/download/apk"
|
||||
apkeep_bin="${CLONE_APP_APKEEP:-apkeep}"
|
||||
source="${CLONE_APP_APKEEP_SOURCE:-apk-pure}"
|
||||
|
||||
if [[ -z "$package" || -z "$out_dir" ]]; then
|
||||
echo "ERROR: usage: download-apk.sh <package> <out-dir>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v "$apkeep_bin" >/dev/null 2>&1; then
|
||||
echo "ERROR: apkeep not found. Install it with 'brew install apkeep'" >&2
|
||||
echo " (or 'cargo install apkeep'), then re-run." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$out_dir"
|
||||
page_file="$out_dir/download-page.html"
|
||||
tmp_file="$out_dir/app.download"
|
||||
|
||||
# --- Step 1: fetch the download page and extract the signed /r2 link ---------
|
||||
r2_path=""
|
||||
for attempt in 1 2 3; do
|
||||
if "$curl_bin" -sL -A "$UA" -H "Referer: $REFERER" "$page_url" --output "$page_file" 2>/dev/null; then
|
||||
r2_path="$(grep -oE '/r2\?u=[^"]*' "$page_file" | head -1 || true)"
|
||||
[[ -n "$r2_path" ]] && break
|
||||
fi
|
||||
echo "download-page attempt $attempt failed (or no link yet), retrying..." >&2
|
||||
sleep 1
|
||||
done
|
||||
rm -f "$page_file"
|
||||
|
||||
if [[ -z "$r2_path" ]]; then
|
||||
echo "ERROR: no download link found for '$package' on APKCombo ($page_url)." >&2
|
||||
echo " The app may not be available there, or the page format changed." >&2
|
||||
# apkeep handles its own retries; run it once and check the result.
|
||||
if ! "$apkeep_bin" -a "$package" -d "$source" "$out_dir" >&2; then
|
||||
echo "ERROR: apkeep failed to download '$package' from $source." >&2
|
||||
echo " The app may not be available there; try another -d source." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Links in the HTML use raw '&'; decode '&' defensively in case that changes.
|
||||
r2_path="${r2_path//&/&}"
|
||||
artifact_url="https://apkcombo.com${r2_path}"
|
||||
|
||||
# --- Step 2: download the actual artifact ------------------------------------
|
||||
ok=0
|
||||
for attempt in 1 2 3; do
|
||||
if "$curl_bin" -sL --fail -A "$UA" -H "Referer: $REFERER" "$artifact_url" --output "$tmp_file" 2>/dev/null; then
|
||||
ok=1; break
|
||||
fi
|
||||
echo "artifact download attempt $attempt failed, retrying..." >&2
|
||||
sleep 1
|
||||
# apkeep names the artifact after the package; find whichever it produced.
|
||||
artifact=""
|
||||
for cand in "$out_dir/$package.xapk" "$out_dir/$package.apk"; do
|
||||
[[ -f "$cand" ]] && { artifact="$cand"; break; }
|
||||
done
|
||||
|
||||
if [[ "$ok" -ne 1 ]]; then
|
||||
echo "ERROR: failed to download $package artifact after 3 attempts." >&2
|
||||
rm -f "$tmp_file"
|
||||
if [[ -z "$artifact" ]]; then
|
||||
echo "ERROR: apkeep reported success but no artifact found in $out_dir." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Decide extension: both APK and XAPK are ZIPs. XAPK = a zip bundle containing
|
||||
# manifest.json plus at least one split .apk entry; a plain APK has neither.
|
||||
ext="apk"
|
||||
if entries="$(unzip -Z1 "$tmp_file" 2>/dev/null)"; then
|
||||
apk_count="$(grep -c '\.apk$' <<<"$entries" || true)"
|
||||
if grep -q '^manifest\.json$' <<<"$entries" && [[ "$apk_count" -ge 1 ]]; then
|
||||
ext="xapk"
|
||||
fi
|
||||
fi
|
||||
|
||||
ext="${artifact##*.}"
|
||||
final="$out_dir/app.$ext"
|
||||
mv "$tmp_file" "$final"
|
||||
mv -f "$artifact" "$final"
|
||||
echo "$final"
|
||||
|
|
|
|||
|
|
@ -6,62 +6,54 @@ check() { [[ "$2" == "$3" ]] && echo "PASS: $1" || { echo "FAIL: $1 — expected
|
|||
|
||||
tmp="$(mktemp -d)"
|
||||
|
||||
# Find the --output target in a curl arg list, and the request URL (http... arg).
|
||||
# Shared helper sourced by the stubs below.
|
||||
cat > "$tmp/argparse.sh" <<'EOF'
|
||||
out=""; url=""; prev=""
|
||||
# Stub apkeep modelling the real CLI:
|
||||
# apkeep -a <package> -d <source> <out_dir>
|
||||
# parse out the package and the out_dir (last arg), then drop an artifact there.
|
||||
# CLONE_APP_FAKE_EXT picks which extension the stub produces; "fail" makes it
|
||||
# exit non-zero without writing anything.
|
||||
cat > "$tmp/fakeapkeep.sh" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
pkg=""; outdir=""; prev=""
|
||||
for a in "$@"; do
|
||||
[[ "$prev" == "--output" || "$prev" == "-o" ]] && out="$a"
|
||||
[[ "$a" == http*://* ]] && url="$a"
|
||||
[[ "$prev" == "-a" || "$prev" == "--app" ]] && pkg="$a"
|
||||
outdir="$a" # last positional wins; apkeep takes OUTPATH last
|
||||
prev="$a"
|
||||
done
|
||||
case "$CLONE_APP_FAKE_EXT" in
|
||||
fail) exit 1 ;;
|
||||
xapk)
|
||||
workdir="$(mktemp -d)"
|
||||
echo '{}' > "$workdir/manifest.json"
|
||||
echo 'a' > "$workdir/base.apk"; echo 'b' > "$workdir/config.apk"
|
||||
( cd "$workdir" && zip -q -r "$outdir/$pkg.xapk" . ) ;;
|
||||
*) echo 'apk-bytes' > "$outdir/$pkg.apk" ;;
|
||||
esac
|
||||
exit 0
|
||||
EOF
|
||||
chmod +x "$tmp/fakeapkeep.sh"
|
||||
|
||||
# Stub curl modelling the real two-step APKCombo flow:
|
||||
# call 1 (URL contains /download/apk) -> emit HTML page with an /r2?u= link
|
||||
# call 2 (URL contains /r2?u=) -> emit a zip (manifest.json + 2 apk) => xapk
|
||||
cat > "$tmp/fakecurl-xapk.sh" <<EOF
|
||||
#!/usr/bin/env bash
|
||||
source "$tmp/argparse.sh"
|
||||
if [[ "\$url" == *"/download/apk"* ]]; then
|
||||
printf '%s' '<a href="/r2?u=https%3A%2F%2Fexample%2Ftest.xapk">Download</a>' > "\$out"
|
||||
exit 0
|
||||
elif [[ "\$url" == *"/r2?u="* ]]; then
|
||||
workdir="\$(mktemp -d)"
|
||||
echo '{}' > "\$workdir/manifest.json"
|
||||
echo 'a' > "\$workdir/base.apk"; echo 'b' > "\$workdir/config.apk"
|
||||
( cd "\$workdir" && zip -q -r "\$out" . )
|
||||
exit 0
|
||||
fi
|
||||
exit 1
|
||||
EOF
|
||||
chmod +x "$tmp/fakecurl-xapk.sh"
|
||||
|
||||
path="$(CLONE_APP_CURL="$tmp/fakecurl-xapk.sh" bash "$SCRIPT" com.example.app "$tmp/out" 2>/dev/null)"; rc=$?
|
||||
# XAPK case
|
||||
path="$(CLONE_APP_APKEEP="$tmp/fakeapkeep.sh" CLONE_APP_FAKE_EXT=xapk \
|
||||
bash "$SCRIPT" com.example.app "$tmp/out" 2>/dev/null)"; rc=$?
|
||||
check "xapk exit 0" "0" "$rc"
|
||||
check "xapk extension" "xapk" "${path##*.}"
|
||||
check "xapk renamed to app.*" "app.xapk" "$(basename "$path")"
|
||||
|
||||
# Stub where the download page loads but contains NO /r2 link -> exit 1
|
||||
cat > "$tmp/fakecurl-nolink.sh" <<EOF
|
||||
#!/usr/bin/env bash
|
||||
source "$tmp/argparse.sh"
|
||||
if [[ "\$url" == *"/download/apk"* ]]; then
|
||||
printf '%s' '<html>no link here</html>' > "\$out"; exit 0
|
||||
fi
|
||||
exit 1
|
||||
EOF
|
||||
chmod +x "$tmp/fakecurl-nolink.sh"
|
||||
out2="$(CLONE_APP_CURL="$tmp/fakecurl-nolink.sh" bash "$SCRIPT" com.example.app "$tmp/out2" 2>/dev/null)"; rc2=$?
|
||||
check "no-link exit 1" "1" "$rc2"
|
||||
# APK case
|
||||
path2="$(CLONE_APP_APKEEP="$tmp/fakeapkeep.sh" CLONE_APP_FAKE_EXT=apk \
|
||||
bash "$SCRIPT" com.example.app "$tmp/out-apk" 2>/dev/null)"; rc2=$?
|
||||
check "apk exit 0" "0" "$rc2"
|
||||
check "apk extension" "apk" "${path2##*.}"
|
||||
|
||||
# Stub curl that always fails -> exit 1 after retries
|
||||
cat > "$tmp/fakecurl-fail.sh" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
exit 22
|
||||
EOF
|
||||
chmod +x "$tmp/fakecurl-fail.sh"
|
||||
out3="$(CLONE_APP_CURL="$tmp/fakecurl-fail.sh" bash "$SCRIPT" com.example.app "$tmp/out3" 2>/dev/null)"; rc3=$?
|
||||
# apkeep failure -> exit 1
|
||||
out3="$(CLONE_APP_APKEEP="$tmp/fakeapkeep.sh" CLONE_APP_FAKE_EXT=fail \
|
||||
bash "$SCRIPT" com.example.app "$tmp/out3" 2>/dev/null)"; rc3=$?
|
||||
check "fail exit 1" "1" "$rc3"
|
||||
|
||||
# apkeep binary missing -> exit 1
|
||||
out4="$(CLONE_APP_APKEEP="$tmp/does-not-exist-apkeep" \
|
||||
bash "$SCRIPT" com.example.app "$tmp/out4" 2>/dev/null)"; rc4=$?
|
||||
check "missing-binary exit 1" "1" "$rc4"
|
||||
|
||||
rm -rf "$tmp"
|
||||
exit $fail
|
||||
|
|
|
|||
Loading…
Reference in New Issue