From e82d9d885ef310b2103bac889b6a014170e41e0c Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 18 Aug 2009 12:43:25 -0300 Subject: [PATCH] some speedups to offsite spider middleware using regexes and urlparse_cached --- scrapy/contrib/spidermiddleware/offsite.py | 34 ++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index c406e7d55..737262772 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -4,15 +4,39 @@ Offsite Spider Middleware See documentation in docs/ref/spider-middleware.rst """ +import re + +from scrapy.xlib.pydispatch import dispatcher +from scrapy.core import signals from scrapy.http import Request -from scrapy.utils.url import url_is_from_spider +from scrapy.utils.httpobj import urlparse_cached class OffsiteMiddleware(object): + def __init__(self): + self.host_regexes = {} + dispatcher.connect(self.domain_opened, signal=signals.domain_opened) + dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + def process_spider_output(self, response, result, spider): return (x for x in result if not isinstance(x, Request) or \ - self.is_url_from_spider(x.url, spider)) + self.should_follow(x, spider)) - def is_url_from_spider(self, url, spider): - # XXX: should we ask this to the spider manager? - return url_is_from_spider(url, spider) + def should_follow(self, request, spider): + regex = self.host_regexes[spider] + # hostanme can be None for wrong urls (like javascript links) + host = urlparse_cached(request).hostname or '' + return bool(regex.search(host)) + + def get_host_regex(self, domains): + """Override this method to implement a different offsite policy""" + domains = [d.replace('.', r'\.') for d in domains] + regex = r'^(|.*\.)(%s)$' % '|'.join(domains) + return re.compile(regex) + + def domain_opened(self, spider): + domains = [spider.domain_name] + spider.extra_domain_names + self.host_regexes[spider] = self.get_host_regex(domains) + + def domain_closed(self, spider): + del self.host_regexes[spider]