mirror of https://github.com/scrapy/scrapy.git
Feeds: Item Filters (#5178)
This commit is contained in:
D R Siddhartha
GitHub
parent
c062ed017a
commit
4ddc9d6b55
|
|
@ -268,6 +268,45 @@ in multiple files, with the specified maximum item count per file. That way, as
|
|||
soon as a file reaches the maximum item count, that file is delivered to the
|
||||
feed URI, allowing item delivery to start way before the end of the crawl.
|
||||
|
||||
.. _item-filter:
|
||||
|
||||
Item filtering
|
||||
==============
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
You can filter items that you want to allow for a particular feed by using the
|
||||
``item_classes`` option in :ref:`feeds options <feed-options>`. Only items of
|
||||
the specified types will be added to the feed.
|
||||
|
||||
The ``item_classes`` option is implemented by the :class:`~scrapy.extensions.feedexport.ItemFilter`
|
||||
class, which is the default value of the ``item_filter`` :ref:`feed option <feed-options>`.
|
||||
|
||||
You can create your own custom filtering class by implementing :class:`~scrapy.extensions.feedexport.ItemFilter`'s
|
||||
method ``accepts`` and taking ``feed_options`` as an argument.
|
||||
|
||||
For instance::
|
||||
|
||||
class MyCustomFilter:
|
||||
|
||||
def __init__(self, feed_options):
|
||||
self.feed_options = feed_options
|
||||
|
||||
def accepts(self, item):
|
||||
if "field1" in item and item["field1"] == "expected_data":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
You can assign your custom filtering class to the ``item_filter`` :ref:`option of a feed <feed-options>`.
|
||||
See :setting:`FEEDS` for examples.
|
||||
|
||||
ItemFilter
|
||||
----------
|
||||
|
||||
.. autoclass:: scrapy.extensions.feedexport.ItemFilter
|
||||
:members:
|
||||
|
||||
|
||||
Settings
|
||||
========
|
||||
|
|
@ -311,6 +350,7 @@ For instance::
|
|||
'format': 'json',
|
||||
'encoding': 'utf8',
|
||||
'store_empty': False,
|
||||
'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'],
|
||||
'fields': None,
|
||||
'indent': 4,
|
||||
'item_export_kwargs': {
|
||||
|
|
@ -320,12 +360,14 @@ For instance::
|
|||
'/home/user/documents/items.xml': {
|
||||
'format': 'xml',
|
||||
'fields': ['name', 'price'],
|
||||
'item_filter': MyCustomFilter1,
|
||||
'encoding': 'latin1',
|
||||
'indent': 8,
|
||||
},
|
||||
pathlib.Path('items.csv'): {
|
||||
'format': 'csv',
|
||||
'fields': ['price', 'name'],
|
||||
'item_filter': 'myproject.filters.MyCustomFilter2',
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -347,6 +389,18 @@ as a fallback value if that key is not provided for a specific feed definition:
|
|||
|
||||
- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
|
||||
|
||||
- ``item_classes``: list of :ref:`item classes <topics-items>` to export.
|
||||
|
||||
If undefined or empty, all items are exported.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
- ``item_filter``: a :ref:`filter class <item-filter>` to filter items to export.
|
||||
|
||||
:class:`~scrapy.extensions.feedexport.ItemFilter` is used be default.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
|
||||
|
||||
- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
|
||||
|
|
|
|||
|
|
@ -46,6 +46,38 @@ def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
|
|||
return builder(*preargs, uri, *args, **kwargs)
|
||||
|
||||
|
||||
class ItemFilter:
|
||||
"""
|
||||
This will be used by FeedExporter to decide if an item should be allowed
|
||||
to be exported to a particular feed.
|
||||
|
||||
:param feed_options: feed specific options passed from FeedExporter
|
||||
:type feed_options: dict
|
||||
"""
|
||||
|
||||
def __init__(self, feed_options):
|
||||
self.feed_options = feed_options
|
||||
self.item_classes = set()
|
||||
|
||||
if 'item_classes' in self.feed_options:
|
||||
for item_class in self.feed_options['item_classes']:
|
||||
self.item_classes.add(load_object(item_class))
|
||||
|
||||
def accepts(self, item):
|
||||
"""
|
||||
Return ``True`` if `item` should be exported or ``False`` otherwise.
|
||||
|
||||
:param item: scraped item which user wants to check if is acceptable
|
||||
:type item: :ref:`Scrapy items <topics-items>`
|
||||
:return: `True` if accepted, `False` otherwise
|
||||
:rtype: bool
|
||||
"""
|
||||
if self.item_classes:
|
||||
return isinstance(item, tuple(self.item_classes))
|
||||
|
||||
return True # accept all items if none declared in item_classes
|
||||
|
||||
|
||||
class IFeedStorage(Interface):
|
||||
"""Interface that all Feed Storages must implement"""
|
||||
|
||||
|
|
@ -215,7 +247,7 @@ class FTPFeedStorage(BlockingFeedStorage):
|
|||
|
||||
|
||||
class _FeedSlot:
|
||||
def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):
|
||||
def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template, filter):
|
||||
self.file = file
|
||||
self.exporter = exporter
|
||||
self.storage = storage
|
||||
|
|
@ -225,6 +257,7 @@ class _FeedSlot:
|
|||
self.store_empty = store_empty
|
||||
self.uri_template = uri_template
|
||||
self.uri = uri
|
||||
self.filter = filter
|
||||
# flags
|
||||
self.itemcount = 0
|
||||
self._exporting = False
|
||||
|
|
@ -255,6 +288,7 @@ class FeedExporter:
|
|||
self.settings = crawler.settings
|
||||
self.feeds = {}
|
||||
self.slots = []
|
||||
self.filters = {}
|
||||
|
||||
if not self.settings['FEEDS'] and not self.settings['FEED_URI']:
|
||||
raise NotConfigured
|
||||
|
|
@ -269,12 +303,14 @@ class FeedExporter:
|
|||
uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects
|
||||
feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
|
||||
self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
|
||||
self.filters[uri] = self._load_filter(feed_options)
|
||||
# End: Backward compatibility for FEED_URI and FEED_FORMAT settings
|
||||
|
||||
# 'FEEDS' setting takes precedence over 'FEED_URI'
|
||||
for uri, feed_options in self.settings.getdict('FEEDS').items():
|
||||
uri = str(uri) # handle pathlib.Path objects
|
||||
self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
|
||||
self.filters[uri] = self._load_filter(feed_options)
|
||||
|
||||
self.storages = self._load_components('FEED_STORAGES')
|
||||
self.exporters = self._load_components('FEED_EXPORTERS')
|
||||
|
|
@ -368,6 +404,7 @@ class FeedExporter:
|
|||
store_empty=feed_options['store_empty'],
|
||||
batch_id=batch_id,
|
||||
uri_template=uri_template,
|
||||
filter=self.filters[uri_template]
|
||||
)
|
||||
if slot.store_empty:
|
||||
slot.start_exporting()
|
||||
|
|
@ -376,6 +413,10 @@ class FeedExporter:
|
|||
def item_scraped(self, item, spider):
|
||||
slots = []
|
||||
for slot in self.slots:
|
||||
if not slot.filter.accepts(item):
|
||||
slots.append(slot) # if slot doesn't accept item, continue with next slot
|
||||
continue
|
||||
|
||||
slot.start_exporting()
|
||||
slot.exporter.export_item(item)
|
||||
slot.itemcount += 1
|
||||
|
|
@ -486,3 +527,8 @@ class FeedExporter:
|
|||
uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
|
||||
uripar_function(params, spider)
|
||||
return params
|
||||
|
||||
def _load_filter(self, feed_options):
|
||||
# load the item filter if declared else load the default filter class
|
||||
item_filter_class = load_object(feed_options.get("item_filter", ItemFilter))
|
||||
return item_filter_class(feed_options)
|
||||
|
|
|
|||
|
|
@ -561,6 +561,10 @@ class FeedExportTestBase(ABC, unittest.TestCase):
|
|||
egg = scrapy.Field()
|
||||
baz = scrapy.Field()
|
||||
|
||||
class MyItem2(scrapy.Item):
|
||||
foo = scrapy.Field()
|
||||
hello = scrapy.Field()
|
||||
|
||||
def _random_temp_filename(self, inter_dir=''):
|
||||
chars = [random.choice(ascii_letters + digits) for _ in range(15)]
|
||||
filename = ''.join(chars)
|
||||
|
|
@ -888,13 +892,9 @@ class FeedExportTest(FeedExportTestBase):
|
|||
@defer.inlineCallbacks
|
||||
def test_export_multiple_item_classes(self):
|
||||
|
||||
class MyItem2(scrapy.Item):
|
||||
foo = scrapy.Field()
|
||||
hello = scrapy.Field()
|
||||
|
||||
items = [
|
||||
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
|
||||
MyItem2({'hello': 'world2', 'foo': 'bar2'}),
|
||||
self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),
|
||||
self.MyItem({'foo': 'bar3', 'egg': 'spam3', 'baz': 'quux3'}),
|
||||
{'hello': 'world4', 'egg': 'spam4'},
|
||||
]
|
||||
|
|
@ -929,6 +929,114 @@ class FeedExportTest(FeedExportTestBase):
|
|||
yield self.assertExported(items, header, rows,
|
||||
settings=settings, ordered=True)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_based_on_item_classes(self):
|
||||
items = [
|
||||
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
|
||||
self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),
|
||||
{'hello': 'world3', 'egg': 'spam3'},
|
||||
]
|
||||
|
||||
formats = {
|
||||
'csv': b'baz,egg,foo\r\n,spam1,bar1\r\n',
|
||||
'json': b'[\n{"hello": "world2", "foo": "bar2"}\n]',
|
||||
'jsonlines': (
|
||||
b'{"foo": "bar1", "egg": "spam1"}\n'
|
||||
b'{"hello": "world2", "foo": "bar2"}\n'
|
||||
),
|
||||
'xml': (
|
||||
b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
|
||||
b'<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>'
|
||||
b'world2</hello><foo>bar2</foo></item>\n<item><hello>world3'
|
||||
b'</hello><egg>spam3</egg></item>\n</items>'
|
||||
),
|
||||
}
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
self._random_temp_filename(): {
|
||||
'format': 'csv',
|
||||
'item_classes': [self.MyItem],
|
||||
},
|
||||
self._random_temp_filename(): {
|
||||
'format': 'json',
|
||||
'item_classes': [self.MyItem2],
|
||||
},
|
||||
self._random_temp_filename(): {
|
||||
'format': 'jsonlines',
|
||||
'item_classes': [self.MyItem, self.MyItem2],
|
||||
},
|
||||
self._random_temp_filename(): {
|
||||
'format': 'xml',
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(items, settings)
|
||||
for fmt, expected in formats.items():
|
||||
self.assertEqual(expected, data[fmt])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_based_on_custom_filters(self):
|
||||
items = [
|
||||
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
|
||||
self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),
|
||||
{'hello': 'world3', 'egg': 'spam3'},
|
||||
]
|
||||
|
||||
MyItem = self.MyItem
|
||||
|
||||
class CustomFilter1:
|
||||
def __init__(self, feed_options):
|
||||
pass
|
||||
|
||||
def accepts(self, item):
|
||||
return isinstance(item, MyItem)
|
||||
|
||||
class CustomFilter2(scrapy.extensions.feedexport.ItemFilter):
|
||||
def accepts(self, item):
|
||||
if 'foo' not in item.fields:
|
||||
return False
|
||||
return True
|
||||
|
||||
class CustomFilter3(scrapy.extensions.feedexport.ItemFilter):
|
||||
def accepts(self, item):
|
||||
if isinstance(item, tuple(self.item_classes)) and item['foo'] == "bar1":
|
||||
return True
|
||||
return False
|
||||
|
||||
formats = {
|
||||
'json': b'[\n{"foo": "bar1", "egg": "spam1"}\n]',
|
||||
'xml': (
|
||||
b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
|
||||
b'<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>'
|
||||
b'world2</hello><foo>bar2</foo></item>\n</items>'
|
||||
),
|
||||
'jsonlines': b'{"foo": "bar1", "egg": "spam1"}\n',
|
||||
}
|
||||
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
self._random_temp_filename(): {
|
||||
'format': 'json',
|
||||
'item_filter': CustomFilter1,
|
||||
},
|
||||
self._random_temp_filename(): {
|
||||
'format': 'xml',
|
||||
'item_filter': CustomFilter2,
|
||||
},
|
||||
self._random_temp_filename(): {
|
||||
'format': 'jsonlines',
|
||||
'item_classes': [self.MyItem, self.MyItem2],
|
||||
'item_filter': CustomFilter3,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
data = yield self.exported_data(items, settings)
|
||||
for fmt, expected in formats.items():
|
||||
self.assertEqual(expected, data[fmt])
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_dicts(self):
|
||||
# When dicts are used, only keys from the first row are used as
|
||||
|
|
|
|||
Loading…
Reference in New Issue