65 lines
2.2 KiB
YAML
65 lines
2.2 KiB
YAML
name: Validate Collection URLs
|
|
|
|
on:
|
|
push:
|
|
paths:
|
|
- 'collections/**.json'
|
|
pull_request:
|
|
paths:
|
|
- 'collections/**.json'
|
|
|
|
jobs:
|
|
validate-urls:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
|
|
- name: Extract and validate URLs
|
|
run: |
|
|
FAILED=0
|
|
CHECKED=0
|
|
FAILED_URLS=""
|
|
|
|
# Recursively extract all non-null string URLs from every JSON file in collections/
|
|
URLS=$(jq -r '.. | .url? | select(type == "string")' collections/*.json | sort -u)
|
|
|
|
while IFS= read -r url; do
|
|
[ -z "$url" ] && continue
|
|
CHECKED=$((CHECKED + 1))
|
|
printf "Checking: %s ... " "$url"
|
|
|
|
# Range: bytes=0-0 GET, following redirects, with the response body
|
|
# discarded. download.kiwix.org 301-redirects BOTH valid and dead files
|
|
# to a mirror, so we must follow to the mirror to learn the real status
|
|
# (206/200 = exists, 404 = gone). The previous --max-filesize 1 aborted
|
|
# on the 301 redirect page's own body BEFORE following it, reporting 301
|
|
# for every URL; it is removed. Range-honoring mirrors return 206 after
|
|
# one byte; --max-time bounds the rare mirror that ignores Range.
|
|
# `|| true` keeps the step's `set -e` from aborting on a curl-level
|
|
# error (e.g. transient network failure); those surface as a non-200
|
|
# code below.
|
|
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
|
|
--range 0-0 \
|
|
--max-time 30 \
|
|
--location \
|
|
"$url" || true)
|
|
|
|
if [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "206" ]; then
|
|
echo "OK ($HTTP_CODE)"
|
|
else
|
|
echo "FAILED ($HTTP_CODE)"
|
|
FAILED=$((FAILED + 1))
|
|
FAILED_URLS="$FAILED_URLS\n - $url (HTTP $HTTP_CODE)"
|
|
fi
|
|
done <<< "$URLS"
|
|
|
|
echo ""
|
|
echo "Checked $CHECKED URLs, $FAILED failed."
|
|
|
|
if [ "$FAILED" -gt 0 ]; then
|
|
echo ""
|
|
echo "Broken URLs:"
|
|
printf "%b\n" "$FAILED_URLS"
|
|
exit 1
|
|
fi
|