mirror of https://github.com/scrapy/scrapy.git
additional simplification of OffsiteMiddleware
This commit is contained in:
parent
eca05c9e12
commit
a74b0b1764
|
|
@ -3,13 +3,10 @@ OffsiteMiddleware: Filters out Requests for URLs outside the domains covered by
|
|||
the spider.
|
||||
"""
|
||||
|
||||
from itertools import ifilter
|
||||
|
||||
from scrapy.item import ScrapedItem
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.url import url_is_from_spider
|
||||
|
||||
class OffsiteMiddleware(object):
|
||||
|
||||
def process_spider_output(self, response, result, spider):
|
||||
filter = lambda x: isinstance(x, ScrapedItem) or url_is_from_spider(x.url, spider)
|
||||
return ifilter(filter, result or ())
|
||||
return (x for x in result if not isinstance(x, Request) or url_is_from_spider(x.url, spider))
|
||||
|
|
|
|||
Loading…
Reference in New Issue