Feed exporter: start exporting only on first item

Fixes GH-872

scrapy/extensions/feedexport.py

@@ -170,6 +170,7 @@ class FeedExporter(object):
         if not self._exporter_supported(self.format):
             raise NotConfigured
         self.store_empty = settings.getbool('FEED_STORE_EMPTY')
+        self._exporting = False
         self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
         uripar = settings['FEED_URI_PARAMS']
         self._uripar = load_object(uripar) if uripar else lambda x, y: None
@@ -188,14 +189,18 @@ class FeedExporter(object):
         file = storage.open(spider)
         exporter = self._get_exporter(file, fields_to_export=self.export_fields,
             encoding=self.export_encoding)
-        exporter.start_exporting()
+        if self.store_empty:
+            exporter.start_exporting()
+            self._exporting = True
         self.slot = SpiderSlot(file, exporter, storage, uri)
 
     def close_spider(self, spider):
         slot = self.slot
         if not slot.itemcount and not self.store_empty:
             return
-        slot.exporter.finish_exporting()
+        if self._exporting:
+            slot.exporter.finish_exporting()
+            self._exporting = False
         logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
         log_args = {'format': self.format,
                     'itemcount': slot.itemcount,
@@ -210,6 +215,9 @@ class FeedExporter(object):
 
     def item_scraped(self, item, spider):
         slot = self.slot
+        if not self._exporting:
+            slot.exporter.start_exporting()
+            self._exporting = True
         slot.exporter.export_item(item)
         slot.itemcount += 1
         return item

tests/test_feedexport.py

@@ -197,6 +197,21 @@ class FeedExportTest(unittest.TestCase):
         data = yield self.run_and_export(TestSpider, settings)
         defer.returnValue(data)
 
+    @defer.inlineCallbacks
+    def exported_no_data(self, settings):
+        """
+        Return exported data which a spider yielding no ``items`` would return.
+        """
+        class TestSpider(scrapy.Spider):
+            name = 'testspider'
+            start_urls = ['http://localhost:8998/']
+
+            def parse(self, response):
+                pass
+
+        data = yield self.run_and_export(TestSpider, settings)
+        defer.returnValue(data)
+
     @defer.inlineCallbacks
     def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
         settings = settings or {}
@@ -283,6 +298,32 @@ class FeedExportTest(unittest.TestCase):
         header = self.MyItem.fields.keys()
         yield self.assertExported(items, header, rows, ordered=False)
 
+    @defer.inlineCallbacks
+    def test_export_no_items_not_store_empty(self):
+        formats = ('json',
+                   'jsonlines',
+                   'xml',
+                   'csv',)
+
+        for fmt in formats:
+            settings = {'FEED_FORMAT': fmt}
+            data = yield self.exported_no_data(settings)
+            self.assertEqual(data, b'')
+
+    @defer.inlineCallbacks
+    def test_export_no_items_store_empty(self):
+        formats = (
+            ('json', b'[\n\n]'),
+            ('jsonlines', b''),
+            ('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),
+            ('csv', b''),
+        )
+
+        for fmt, expctd in formats:
+            settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True}
+            data = yield self.exported_no_data(settings)
+            self.assertEqual(data, expctd)
+
     @defer.inlineCallbacks
     def test_export_multiple_item_classes(self):
 
@@ -376,26 +417,26 @@ class FeedExportTest(unittest.TestCase):
     def test_export_encoding(self):
         items = [dict({'foo': u'Test\xd6'})]
         header = ['foo']
-        
+
         formats = {
             'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'),
             'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),
             'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),
         }
-        
+
         for format in formats:
             settings = {'FEED_FORMAT': format}
             data = yield self.exported_data(items, settings)
             self.assertEqual(formats[format], data)
-            
+
         formats = {
             'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'),
             'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),
             'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),
             'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),
         }
-        
+
         for format in formats:
             settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'}
             data = yield self.exported_data(items, settings)