diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index 312c61ae0..e6b233aa2 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -3,13 +3,10 @@ OffsiteMiddleware: Filters out Requests for URLs outside the domains covered by the spider. """ -from itertools import ifilter - -from scrapy.item import ScrapedItem +from scrapy.http import Request from scrapy.utils.url import url_is_from_spider class OffsiteMiddleware(object): def process_spider_output(self, response, result, spider): - filter = lambda x: isinstance(x, ScrapedItem) or url_is_from_spider(x.url, spider) - return ifilter(filter, result or ()) + return (x for x in result if not isinstance(x, Request) or url_is_from_spider(x.url, spider))