From eca05c9e12bf7184f309a2d407eac10206ef762c Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 9 Jun 2009 12:37:15 -0300 Subject: [PATCH] OffsiteMiddleware: removed logging and simplified implementation --- scrapy/contrib/spidermiddleware/offsite.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index cea2cdbbd..312c61ae0 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -3,16 +3,13 @@ OffsiteMiddleware: Filters out Requests for URLs outside the domains covered by the spider. """ -from scrapy import log -from scrapy.http import Request +from itertools import ifilter + +from scrapy.item import ScrapedItem from scrapy.utils.url import url_is_from_spider class OffsiteMiddleware(object): - def process_spider_output(self, response, result, spider): - def _filter(r): - if isinstance(r, Request) and not url_is_from_spider(r.url, spider): - log.msg("Ignoring link (offsite): %s " % r.url, level=log.DEBUG, domain=spider.domain_name) - return False - return True - return (r for r in result or () if _filter(r)) + def process_spider_output(self, response, result, spider): + filter = lambda x: isinstance(x, ScrapedItem) or url_is_from_spider(x.url, spider) + return ifilter(filter, result or ())