Add CI validation for translations and pot_tools dev utility
Add translation-check workflow with two jobs: - validate-po: msgfmt --check on changed .po files, .mo sync warning, tr(f-string) anti-pattern grep on changed .py files - validate-pot: verify all tr() strings exist in base.pot when .py files change Workflow only triggers on .py/.po/.pot file changes. Add scripts/pot_tools.py developer utility (stats, list, add_missing) for managing base.pot.
This commit is contained in:
Softer
parent
f0eaf40d5f
commit
1f4ebd9414
|
|
@ -0,0 +1,56 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Check that all tr() strings from code exist in base.pot.
|
||||
|
||||
Used by CI. Fails if any translatable string is missing from the committed pot.
|
||||
|
||||
Usage:
|
||||
xgettext ... -o /tmp/generated.pot
|
||||
python3 check_pot_freshness.py /tmp/generated.pot locales/base.pot
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def extract_msgids(path: str) -> set[str]:
|
||||
content = Path(path).read_text()
|
||||
ids: set[str] = set()
|
||||
current: str | None = None
|
||||
|
||||
for line in content.splitlines():
|
||||
if line.startswith('msgid '):
|
||||
m = re.search(r'"(.*)"', line)
|
||||
current = m.group(1) if m else ''
|
||||
elif current is not None and line.startswith('"'):
|
||||
m = re.search(r'"(.*)"', line)
|
||||
if m:
|
||||
current += m.group(1)
|
||||
else:
|
||||
if current is not None and current:
|
||||
ids.add(current)
|
||||
current = None
|
||||
|
||||
if current:
|
||||
ids.add(current)
|
||||
|
||||
return ids
|
||||
|
||||
|
||||
def main() -> None:
|
||||
generated = extract_msgids(sys.argv[1])
|
||||
committed = extract_msgids(sys.argv[2])
|
||||
|
||||
missing = sorted(generated - committed)
|
||||
|
||||
if missing:
|
||||
print('::error::New tr() strings not in base.pot - run locales_generator.sh:')
|
||||
for s in missing:
|
||||
print(f' {s}')
|
||||
sys.exit(1)
|
||||
|
||||
print('All tr() strings are present in base.pot')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
name: Translation validation
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'archinstall/**/*.py'
|
||||
- 'archinstall/locales/**'
|
||||
- '.github/workflows/translation-check.yaml'
|
||||
- '.github/scripts/check_pot_freshness.py'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'archinstall/**/*.py'
|
||||
- 'archinstall/locales/**'
|
||||
- '.github/workflows/translation-check.yaml'
|
||||
- '.github/scripts/check_pot_freshness.py'
|
||||
jobs:
|
||||
validate-po:
|
||||
name: Validate .po files and translation patterns
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: archlinux/archlinux:latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Find changed files
|
||||
id: changed
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
base="${{ github.event.pull_request.base.sha }}"
|
||||
head="${{ github.event.pull_request.head.sha }}"
|
||||
else
|
||||
base="${{ github.event.before }}"
|
||||
head="${{ github.sha }}"
|
||||
fi
|
||||
changed_po=$(git diff --name-only "$base" "$head" 2>/dev/null | grep '\.po$' || true)
|
||||
changed_py=$(git diff --name-only "$base" "$head" 2>/dev/null | grep '\.py$' || true)
|
||||
if [ -n "$changed_po" ]; then
|
||||
echo "has_po=true" >> "$GITHUB_OUTPUT"
|
||||
echo "$changed_po" > /tmp/changed_po.txt
|
||||
else
|
||||
echo "has_po=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
if [ -n "$changed_py" ]; then
|
||||
echo "has_py=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_py=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
- name: Install gettext
|
||||
if: steps.changed.outputs.has_po == 'true'
|
||||
run: |
|
||||
pacman-key --init
|
||||
pacman --noconfirm -Sy archlinux-keyring
|
||||
pacman --noconfirm -Syyu
|
||||
pacman --noconfirm -S gettext
|
||||
- name: Check changed .po syntax with msgfmt
|
||||
if: steps.changed.outputs.has_po == 'true'
|
||||
run: |
|
||||
failed=0
|
||||
while IFS= read -r po; do
|
||||
if [ -f "$po" ] && ! msgfmt --check --output-file=/dev/null "$po" 2>&1; then
|
||||
echo "FAIL: $po"
|
||||
failed=1
|
||||
fi
|
||||
done < /tmp/changed_po.txt
|
||||
if [ "$failed" -eq 1 ]; then
|
||||
echo "::error::Some .po files have syntax errors"
|
||||
exit 1
|
||||
fi
|
||||
echo "All changed .po files passed syntax check"
|
||||
- name: Warn if changed .mo files are out of sync
|
||||
if: steps.changed.outputs.has_po == 'true'
|
||||
run: |
|
||||
out_of_sync=0
|
||||
while IFS= read -r po; do
|
||||
if [ ! -f "$po" ]; then continue; fi
|
||||
mo="${po%.po}.mo"
|
||||
if [ ! -f "$mo" ]; then
|
||||
echo "::warning::Missing .mo file for $po"
|
||||
out_of_sync=1
|
||||
continue
|
||||
fi
|
||||
tmp_mo=$(mktemp)
|
||||
msgfmt --output-file="$tmp_mo" "$po"
|
||||
if ! cmp -s "$mo" "$tmp_mo"; then
|
||||
echo "::warning::$mo is out of sync with $po - run locales_generator.sh"
|
||||
out_of_sync=1
|
||||
fi
|
||||
rm -f "$tmp_mo"
|
||||
done < /tmp/changed_po.txt
|
||||
if [ "$out_of_sync" -eq 1 ]; then
|
||||
echo "Some .mo files are out of sync (warning only)"
|
||||
fi
|
||||
- name: Check for tr(f-string) anti-pattern
|
||||
if: steps.changed.outputs.has_py == 'true'
|
||||
run: |
|
||||
if grep -rn --include='*.py' "tr(f['\"]" archinstall/; then
|
||||
echo "::error::Found tr(f'...') pattern. Use tr('...{}').format(...) instead"
|
||||
exit 1
|
||||
fi
|
||||
echo "No tr(f-string) anti-pattern found"
|
||||
|
||||
validate-pot:
|
||||
name: Validate base.pot is up to date
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: archlinux/archlinux:latest
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Check for changed Python files
|
||||
id: check_py
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "pull_request" ]; then
|
||||
base="${{ github.event.pull_request.base.sha }}"
|
||||
head="${{ github.event.pull_request.head.sha }}"
|
||||
else
|
||||
base="${{ github.event.before }}"
|
||||
head="${{ github.sha }}"
|
||||
fi
|
||||
if git diff --name-only "$base" "$head" 2>/dev/null | grep -q '\.py$'; then
|
||||
echo "has_py_changes=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_py_changes=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
- name: Install gettext and python
|
||||
if: steps.check_py.outputs.has_py_changes == 'true'
|
||||
run: |
|
||||
pacman-key --init
|
||||
pacman --noconfirm -Sy archlinux-keyring
|
||||
pacman --noconfirm -Syyu
|
||||
pacman --noconfirm -S gettext python
|
||||
- name: Check for missing translatable strings
|
||||
if: steps.check_py.outputs.has_py_changes == 'true'
|
||||
run: |
|
||||
cd archinstall
|
||||
find . -type f -name '*.py' | xargs xgettext \
|
||||
--no-location --omit-header --keyword='tr' \
|
||||
-d base -o /tmp/generated.pot
|
||||
python3 ../.github/scripts/check_pot_freshness.py /tmp/generated.pot locales/base.pot
|
||||
|
|
@ -0,0 +1,145 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Tools for managing base.pot: find and add missing translatable strings.
|
||||
|
||||
Usage (from repo root):
|
||||
scripts/pot_tools.py stats
|
||||
scripts/pot_tools.py list
|
||||
scripts/pot_tools.py add_missing [--dry-run]
|
||||
|
||||
Requires: gettext (xgettext) installed.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
ARCHINSTALL_DIR = REPO_ROOT / 'archinstall'
|
||||
BASE_POT = ARCHINSTALL_DIR / 'locales' / 'base.pot'
|
||||
|
||||
|
||||
def extract_msgids(path: Path) -> set[str]:
|
||||
content = path.read_text()
|
||||
ids: set[str] = set()
|
||||
current: str | None = None
|
||||
|
||||
for line in content.splitlines():
|
||||
if line.startswith('msgid '):
|
||||
m = re.search(r'"(.*)"', line)
|
||||
current = m.group(1) if m else ''
|
||||
elif current is not None and line.startswith('"'):
|
||||
m = re.search(r'"(.*)"', line)
|
||||
if m:
|
||||
current += m.group(1)
|
||||
else:
|
||||
if current is not None and current:
|
||||
ids.add(current)
|
||||
current = None
|
||||
|
||||
if current:
|
||||
ids.add(current)
|
||||
|
||||
return ids
|
||||
|
||||
|
||||
def generate_fresh_pot() -> Path:
|
||||
fd, tmp_path = tempfile.mkstemp(suffix='.pot')
|
||||
os.close(fd)
|
||||
tmp = Path(tmp_path)
|
||||
py_files = sorted(str(p) for p in ARCHINSTALL_DIR.rglob('*.py'))
|
||||
|
||||
subprocess.run(
|
||||
['xgettext', '--no-location', '--omit-header',
|
||||
'--keyword=tr', '-d', 'base', '-o', str(tmp)] + py_files,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
return tmp
|
||||
|
||||
|
||||
def get_missing(fresh_pot: Path) -> set[str]:
|
||||
generated = extract_msgids(fresh_pot)
|
||||
committed = extract_msgids(BASE_POT)
|
||||
return generated - committed
|
||||
|
||||
|
||||
def cmd_stats() -> None:
|
||||
fresh_pot = generate_fresh_pot()
|
||||
try:
|
||||
generated = extract_msgids(fresh_pot)
|
||||
committed = extract_msgids(BASE_POT)
|
||||
missing = generated - committed
|
||||
|
||||
print(f'Code: {len(generated)} translatable strings')
|
||||
print(f'base.pot: {len(committed)} msgids')
|
||||
print(f' Missing: {len(missing)}')
|
||||
finally:
|
||||
fresh_pot.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def cmd_list() -> None:
|
||||
fresh_pot = generate_fresh_pot()
|
||||
try:
|
||||
missing = sorted(get_missing(fresh_pot))
|
||||
|
||||
if missing:
|
||||
print(f'=== MISSING ({len(missing)}): in code but not in base.pot ===')
|
||||
for s in missing:
|
||||
print(f' + {s}')
|
||||
else:
|
||||
print('No missing strings')
|
||||
finally:
|
||||
fresh_pot.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def cmd_add_missing(dry_run: bool = False) -> None:
|
||||
fresh_pot = generate_fresh_pot()
|
||||
try:
|
||||
missing = sorted(get_missing(fresh_pot))
|
||||
|
||||
if not missing:
|
||||
print('No missing strings, base.pot is up to date')
|
||||
return
|
||||
|
||||
print(f'Adding {len(missing)} missing string(s)')
|
||||
for s in missing:
|
||||
print(f' + {s}')
|
||||
|
||||
if dry_run:
|
||||
print('(dry-run, no changes written)')
|
||||
return
|
||||
|
||||
with open(BASE_POT, 'a') as f:
|
||||
for s in missing:
|
||||
if '{' in s:
|
||||
f.write('\n#, python-brace-format')
|
||||
f.write(f'\nmsgid "{s}"\nmsgstr ""\n')
|
||||
|
||||
print(f'Done. Added to {BASE_POT}')
|
||||
finally:
|
||||
fresh_pot.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) < 2 or sys.argv[1] in ('-h', '--help'):
|
||||
print('Usage: pot_tools.py {stats|list|add_missing} [--dry-run]')
|
||||
sys.exit(0)
|
||||
|
||||
cmd = sys.argv[1]
|
||||
if cmd == 'stats':
|
||||
cmd_stats()
|
||||
elif cmd == 'list':
|
||||
cmd_list()
|
||||
elif cmd == 'add_missing':
|
||||
dry_run = '--dry-run' in sys.argv
|
||||
cmd_add_missing(dry_run)
|
||||
else:
|
||||
print(f'Unknown command: {cmd}')
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue