diff --git a/extras/test-scrapyd.sh b/extras/test-scrapyd.sh new file mode 100755 index 000000000..92a1df01b --- /dev/null +++ b/extras/test-scrapyd.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# +# This script is a quick system test for Scrapyd that: +# +# 1. runs scrapyd +# 2. creates a new project and deploys it on scrapyd +# 3. schedules a spider on scrapyd and waits for it to finish +# 4. check the spider scraped the expected data +# + +set -e + +export PATH=$PATH:$(pwd)/bin +export PYTHONPATH=$PYTHONPATH:$(pwd) + +scrapyd_dir=$(mktemp /tmp/test-scrapyd.XXXXXXX -d) +scrapyd_log=$(mktemp /tmp/test-scrapyd.XXXXXXX.log) +scrapy_dir=$(mktemp /tmp/test-scrapyd.XXXXXXX -d) +feed_path=$(mktemp /tmp/test-scrapyd.XXXXXXX.jl) + +twistd -ny extras/scrapyd.tac -d $scrapyd_dir -l $scrapyd_log & + +cd $scrapy_dir +scrapy startproject testproj +cd testproj +cat > testproj/spiders/insophia.py < scrapy.cfg <