RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Change default value of FEED_STORAGE_BATCH_ITEM_COUNT to 0

This commit is contained in:
BroodingKangaroo 2020-06-26 18:45:21 +03:00
parent 77333666f1
commit c3cee74fd4
4 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/topics/feed-exports.rst
View File

@ -435,7 +435,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
FEED_STORAGE_BATCH_ITEM_COUNT
-----------------------------
Default: ``None``
Default: ``0``
If assigned an integer number higher than ``0``, Scrapy generates multiple output files
storing up to the specified number of items in each output file.

2
scrapy/extensions/feedexport.py
View File

@ -243,7 +243,7 @@ class FeedExporter:
self.storages = self._load_components('FEED_STORAGES')
self.exporters = self._load_components('FEED_EXPORTERS')
self.storage_batch_item_count = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)
self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
for uri, feed in self.feeds.items():
if not self._storage_supported(uri):
raise NotConfigured

2
scrapy/settings/default_settings.py
View File

@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {
's3': 'scrapy.extensions.feedexport.S3FeedStorage',
'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
}
FEED_STORAGE_BATCH_ITEM_COUNT = None
FEED_STORAGE_BATCH_ITEM_COUNT = 0
FEED_EXPORTERS = {}
FEED_EXPORTERS_BASE = {
'json': 'scrapy.exporters.JsonItemExporter',

20
tests/test_feedexport.py
View File

@ -1144,7 +1144,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
},
})
batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
for batch in data['jl']:
@ -1160,7 +1160,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
},
})
batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
data = yield self.exported_data(items, settings)
for batch in data['csv']:
got_batch = csv.DictReader(to_unicode(batch).splitlines())
@ -1176,7 +1176,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
},
})
batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
for batch in data['xml']:
@ -1194,7 +1194,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
},
})
batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
# XML
@ -1219,7 +1219,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
},
})
batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
import pickle
@ -1236,7 +1236,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
},
})
batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
rows = [{k: v for k, v in row.items() if v} for row in rows]
data = yield self.exported_data(items, settings)
import marshal
@ -1262,7 +1262,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
'FEED_STORAGE_BATCH_ITEM_COUNT': 2
}
header = self.MyItem.fields.keys()
yield self.assertExported(items, header, rows, settings=settings)
yield self.assertExported(items, header, rows, settings=Settings(settings))
def test_wrong_path(self):
""" If path is without %(batch_time)s or %(batch_id)s an exception must be raised """
@ -1412,14 +1412,14 @@ class BatchDeliveriesTest(FeedExportTestBase):
bucket_name=s3_test_bucket_name, prefix=prefix
)
storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)
settings = {
settings = Settings({
'FEEDS': {
s3_test_file_uri: {
'format': 'json',
},
},
'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
}
})
items = [
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
@ -1436,7 +1436,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
s3 = boto3.resource('s3')
my_bucket = s3.Bucket(s3_test_bucket_name)
batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']
batch_size = settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
with MockServer() as s:
runner = CrawlerRunner(Settings(settings))