fixed bug in FeedExports extension, introduced in previous commit

extras/test-scrapyd.sh

@@ -14,9 +14,11 @@ export PATH=$PATH:$(pwd)/bin
 export PYTHONPATH=$PYTHONPATH:$(pwd)
 
 scrapyd_dir=$(mktemp /tmp/test-scrapyd.XXXXXXX -d)
-scrapyd_log=$(mktemp /tmp/test-scrapyd.XXXXXXX)
-scrapy_dir=$(mktemp /tmp/test-scrapyd.XXXXXXX -d)
-feed_path=$(mktemp /tmp/test-scrapyd.XXXXXXX)
+scrapyd_log=$scrapyd_dir/scrapyd.log
+scrapy_dir=$(mktemp /tmp/test-scrapy.XXXXXXX -d)
+
+echo "scrapyd dir: $scrapyd_dir"
+echo "scrapy dir : $scrapy_dir"
 
 twistd -ny extras/scrapyd.tac -d $scrapyd_dir -l $scrapyd_log &
 
@@ -68,7 +70,7 @@ project = testproj
 
 scrapy deploy
 
-curl -s http://localhost:6800/schedule.json -d project=testproj -d spider=insophia -d setting=FEED_URI=$feed_path -d arg=SOME_ARGUMENT
+curl -s http://localhost:6800/schedule.json -d project=testproj -d spider=insophia -d arg=SOME_ARGUMENT
 
 echo "waiting 20 seconds for spider to run and finish..."
 sleep 20
@@ -81,12 +83,30 @@ if ! grep -q "Process finished" $scrapyd_log; then
     exit 1
 fi
 
+feed_path=$(find $scrapyd_dir/items -name '*.jl')
+if [ ! -f "$feed_path" ]; then
+    echo "items feed not generated: $feed_path"
+    exit 1
+fi
+
+log_path=$(find $scrapyd_dir/logs -name '*.log')
+if [ ! -f "$log_path" ]; then
+    echo "log file not generated: $log_path"
+    exit 1
+fi
+
 numitems="$(cat $feed_path | wc -l)"
 if [ "$numitems" != "7" ]; then
     echo "error: wrong number of items scraped: $numitems"
     exit 1
 fi
 
+numscraped="$(cat $log_path | grep Scraped | wc -l)"
+if [ "$numscraped" != "7" ]; then
+    echo "error: wrong number of 'Scraped' lines in log: $numscraped"
+    exit 1
+fi
+
 if ! grep -q "About Us" $feed_path; then
     echo "error: About Us page not scraped"
     exit 1
@@ -97,6 +117,6 @@ if ! grep -q "SOME_ARGUMENT" $feed_path; then
     exit 1
 fi
 
-rm -rf /tmp/test-scrapyd.*
+rm -rf /tmp/test-scrapyd.* /tmp/test-scrapy.*
 
 echo "All tests OK"

scrapy/contrib/feedexport.py

@@ -151,16 +151,12 @@ class FeedExporter(object):
 
     @classmethod
     def from_crawler(cls, crawler):
-        o = cls()
+        o = cls(crawler.settings)
         crawler.signals.connect(o.open_spider, signals.spider_opened)
         crawler.signals.connect(o.close_spider, signals.spider_closed)
         crawler.signals.connect(o.item_scraped, signals.item_scraped)
         return o
 
-    @classmethod
-    def from_crawler(cls, crawler):
-        return FeedExporter(crawler.settings)
-
     def open_spider(self, spider):
         uri = self.urifmt % self._get_uri_params(spider)
         storage = self._get_storage(uri)