From a74b0b1764ef72e0b7d4f618dbbde0f9e0adf278 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 9 Jun 2009 13:09:35 -0300 Subject: [PATCH] additional simplification of OffsiteMiddleware --- scrapy/contrib/spidermiddleware/offsite.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index 312c61ae0..e6b233aa2 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -3,13 +3,10 @@ OffsiteMiddleware: Filters out Requests for URLs outside the domains covered by the spider. """ -from itertools import ifilter - -from scrapy.item import ScrapedItem +from scrapy.http import Request from scrapy.utils.url import url_is_from_spider class OffsiteMiddleware(object): def process_spider_output(self, response, result, spider): - filter = lambda x: isinstance(x, ScrapedItem) or url_is_from_spider(x.url, spider) - return ifilter(filter, result or ()) + return (x for x in result if not isinstance(x, Request) or url_is_from_spider(x.url, spider))