additional simplification of OffsiteMiddleware

This commit is contained in:
Pablo Hoffman 2009-06-09 13:09:35 -03:00
parent eca05c9e12
commit a74b0b1764
1 changed files with 2 additions and 5 deletions

View File

@ -3,13 +3,10 @@ OffsiteMiddleware: Filters out Requests for URLs outside the domains covered by
the spider.
"""
from itertools import ifilter
from scrapy.item import ScrapedItem
from scrapy.http import Request
from scrapy.utils.url import url_is_from_spider
class OffsiteMiddleware(object):
def process_spider_output(self, response, result, spider):
filter = lambda x: isinstance(x, ScrapedItem) or url_is_from_spider(x.url, spider)
return ifilter(filter, result or ())
return (x for x in result if not isinstance(x, Request) or url_is_from_spider(x.url, spider))