From 08424ee093f82922e5f706998e3b4717e30e2b87 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sat, 15 Aug 2009 20:44:11 -0300 Subject: [PATCH] OffsiteMiddleware: isolate policy of urls belonging to spiders into a separate method --- scrapy/contrib/spidermiddleware/offsite.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index 14375d9fc..c406e7d55 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -10,4 +10,9 @@ from scrapy.utils.url import url_is_from_spider class OffsiteMiddleware(object): def process_spider_output(self, response, result, spider): - return (x for x in result if not isinstance(x, Request) or url_is_from_spider(x.url, spider)) + return (x for x in result if not isinstance(x, Request) or \ + self.is_url_from_spider(x.url, spider)) + + def is_url_from_spider(self, url, spider): + # XXX: should we ask this to the spider manager? + return url_is_from_spider(url, spider)