diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 26c247cdd..7a4b054e9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -268,6 +268,45 @@ in multiple files, with the specified maximum item count per file. That way, as soon as a file reaches the maximum item count, that file is delivered to the feed URI, allowing item delivery to start way before the end of the crawl. +.. _item-filter: + +Item filtering +============== + +.. versionadded:: VERSION + +You can filter items that you want to allow for a particular feed by using the +``item_classes`` option in :ref:`feeds options `. Only items of +the specified types will be added to the feed. + +The ``item_classes`` option is implemented by the :class:`~scrapy.extensions.feedexport.ItemFilter` +class, which is the default value of the ``item_filter`` :ref:`feed option `. + +You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s +method ``accepts`` and taking ``feed_options`` as an argument. + +For instance:: + + class MyCustomFilter: + + def __init__(self, feed_options): + self.feed_options = feed_options + + def accepts(self, item): + if "field1" in item and item["field1"] == "expected_data": + return True + return False + + +You can assign your custom filtering class to the ``item_filter`` :ref:`option of a feed `. +See :setting:`FEEDS` for examples. + +ItemFilter +---------- + +.. autoclass:: scrapy.extensions.feedexport.ItemFilter + :members: + Settings ======== @@ -311,6 +350,7 @@ For instance:: 'format': 'json', 'encoding': 'utf8', 'store_empty': False, + 'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'], 'fields': None, 'indent': 4, 'item_export_kwargs': { @@ -320,12 +360,14 @@ For instance:: '/home/user/documents/items.xml': { 'format': 'xml', 'fields': ['name', 'price'], + 'item_filter': MyCustomFilter1, 'encoding': 'latin1', 'indent': 8, }, pathlib.Path('items.csv'): { 'format': 'csv', 'fields': ['price', 'name'], + 'item_filter': 'myproject.filters.MyCustomFilter2', }, } @@ -347,6 +389,18 @@ as a fallback value if that key is not provided for a specific feed definition: - ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`. +- ``item_classes``: list of :ref:`item classes ` to export. + + If undefined or empty, all items are exported. + + .. versionadded:: VERSION + +- ``item_filter``: a :ref:`filter class ` to filter items to export. + + :class:`~scrapy.extensions.feedexport.ItemFilter` is used be default. + + .. versionadded:: VERSION + - ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`. - ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class `. diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index bec114707..89dca12f4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -46,6 +46,38 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): return builder(*preargs, uri, *args, **kwargs) +class ItemFilter: + """ + This will be used by FeedExporter to decide if an item should be allowed + to be exported to a particular feed. + + :param feed_options: feed specific options passed from FeedExporter + :type feed_options: dict + """ + + def __init__(self, feed_options): + self.feed_options = feed_options + self.item_classes = set() + + if 'item_classes' in self.feed_options: + for item_class in self.feed_options['item_classes']: + self.item_classes.add(load_object(item_class)) + + def accepts(self, item): + """ + Return ``True`` if `item` should be exported or ``False`` otherwise. + + :param item: scraped item which user wants to check if is acceptable + :type item: :ref:`Scrapy items ` + :return: `True` if accepted, `False` otherwise + :rtype: bool + """ + if self.item_classes: + return isinstance(item, tuple(self.item_classes)) + + return True # accept all items if none declared in item_classes + + class IFeedStorage(Interface): """Interface that all Feed Storages must implement""" @@ -215,7 +247,7 @@ class FTPFeedStorage(BlockingFeedStorage): class _FeedSlot: - def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template): + def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template, filter): self.file = file self.exporter = exporter self.storage = storage @@ -225,6 +257,7 @@ class _FeedSlot: self.store_empty = store_empty self.uri_template = uri_template self.uri = uri + self.filter = filter # flags self.itemcount = 0 self._exporting = False @@ -255,6 +288,7 @@ class FeedExporter: self.settings = crawler.settings self.feeds = {} self.slots = [] + self.filters = {} if not self.settings['FEEDS'] and not self.settings['FEED_URI']: raise NotConfigured @@ -269,12 +303,14 @@ class FeedExporter: uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) + self.filters[uri] = self._load_filter(feed_options) # End: Backward compatibility for FEED_URI and FEED_FORMAT settings # 'FEEDS' setting takes precedence over 'FEED_URI' for uri, feed_options in self.settings.getdict('FEEDS').items(): uri = str(uri) # handle pathlib.Path objects self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings) + self.filters[uri] = self._load_filter(feed_options) self.storages = self._load_components('FEED_STORAGES') self.exporters = self._load_components('FEED_EXPORTERS') @@ -368,6 +404,7 @@ class FeedExporter: store_empty=feed_options['store_empty'], batch_id=batch_id, uri_template=uri_template, + filter=self.filters[uri_template] ) if slot.store_empty: slot.start_exporting() @@ -376,6 +413,10 @@ class FeedExporter: def item_scraped(self, item, spider): slots = [] for slot in self.slots: + if not slot.filter.accepts(item): + slots.append(slot) # if slot doesn't accept item, continue with next slot + continue + slot.start_exporting() slot.exporter.export_item(item) slot.itemcount += 1 @@ -486,3 +527,8 @@ class FeedExporter: uripar_function = load_object(uri_params) if uri_params else lambda x, y: None uripar_function(params, spider) return params + + def _load_filter(self, feed_options): + # load the item filter if declared else load the default filter class + item_filter_class = load_object(feed_options.get("item_filter", ItemFilter)) + return item_filter_class(feed_options) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index df7ec4461..81437b011 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -561,6 +561,10 @@ class FeedExportTestBase(ABC, unittest.TestCase): egg = scrapy.Field() baz = scrapy.Field() + class MyItem2(scrapy.Item): + foo = scrapy.Field() + hello = scrapy.Field() + def _random_temp_filename(self, inter_dir=''): chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = ''.join(chars) @@ -888,13 +892,9 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_export_multiple_item_classes(self): - class MyItem2(scrapy.Item): - foo = scrapy.Field() - hello = scrapy.Field() - items = [ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), - MyItem2({'hello': 'world2', 'foo': 'bar2'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), self.MyItem({'foo': 'bar3', 'egg': 'spam3', 'baz': 'quux3'}), {'hello': 'world4', 'egg': 'spam4'}, ] @@ -929,6 +929,114 @@ class FeedExportTest(FeedExportTestBase): yield self.assertExported(items, header, rows, settings=settings, ordered=True) + @defer.inlineCallbacks + def test_export_based_on_item_classes(self): + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), + {'hello': 'world3', 'egg': 'spam3'}, + ] + + formats = { + 'csv': b'baz,egg,foo\r\n,spam1,bar1\r\n', + 'json': b'[\n{"hello": "world2", "foo": "bar2"}\n]', + 'jsonlines': ( + b'{"foo": "bar1", "egg": "spam1"}\n' + b'{"hello": "world2", "foo": "bar2"}\n' + ), + 'xml': ( + b'\n\n' + b'bar1spam1\n' + b'world2bar2\nworld3' + b'spam3\n' + ), + } + + settings = { + 'FEEDS': { + self._random_temp_filename(): { + 'format': 'csv', + 'item_classes': [self.MyItem], + }, + self._random_temp_filename(): { + 'format': 'json', + 'item_classes': [self.MyItem2], + }, + self._random_temp_filename(): { + 'format': 'jsonlines', + 'item_classes': [self.MyItem, self.MyItem2], + }, + self._random_temp_filename(): { + 'format': 'xml', + }, + }, + } + + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + self.assertEqual(expected, data[fmt]) + + @defer.inlineCallbacks + def test_export_based_on_custom_filters(self): + items = [ + self.MyItem({'foo': 'bar1', 'egg': 'spam1'}), + self.MyItem2({'hello': 'world2', 'foo': 'bar2'}), + {'hello': 'world3', 'egg': 'spam3'}, + ] + + MyItem = self.MyItem + + class CustomFilter1: + def __init__(self, feed_options): + pass + + def accepts(self, item): + return isinstance(item, MyItem) + + class CustomFilter2(scrapy.extensions.feedexport.ItemFilter): + def accepts(self, item): + if 'foo' not in item.fields: + return False + return True + + class CustomFilter3(scrapy.extensions.feedexport.ItemFilter): + def accepts(self, item): + if isinstance(item, tuple(self.item_classes)) and item['foo'] == "bar1": + return True + return False + + formats = { + 'json': b'[\n{"foo": "bar1", "egg": "spam1"}\n]', + 'xml': ( + b'\n\n' + b'bar1spam1\n' + b'world2bar2\n' + ), + 'jsonlines': b'{"foo": "bar1", "egg": "spam1"}\n', + } + + settings = { + 'FEEDS': { + self._random_temp_filename(): { + 'format': 'json', + 'item_filter': CustomFilter1, + }, + self._random_temp_filename(): { + 'format': 'xml', + 'item_filter': CustomFilter2, + }, + self._random_temp_filename(): { + 'format': 'jsonlines', + 'item_classes': [self.MyItem, self.MyItem2], + 'item_filter': CustomFilter3, + }, + }, + } + + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + self.assertEqual(expected, data[fmt]) + @defer.inlineCallbacks def test_export_dicts(self): # When dicts are used, only keys from the first row are used as