diff --git a/scrapy/trunk/scrapy/contrib/itemsampler.py b/scrapy/trunk/scrapy/contrib/itemsampler.py index 0ae3e892b..f31d87e02 100644 --- a/scrapy/trunk/scrapy/contrib/itemsampler.py +++ b/scrapy/trunk/scrapy/contrib/itemsampler.py @@ -83,13 +83,13 @@ class ItemSamplerMiddleware(object): if not settings['ITEMSAMPLER_FILE']: raise NotConfigured - def process_scrape(self, response, spider): + def process_spider_input(self, response, spider): if stats.getpath("%s/items_sampled" % spider.domain_name) >= items_per_domain: return [] elif max_response_size and max_response_size > len(response): return [] - def process_result(self, response, result, spider): + def process_spider_output(self, response, result, spider): requests, items = [], [] for r in result: if isinstance(r, Request): diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py index 1727052c2..81e5e4cd4 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py @@ -17,7 +17,7 @@ class DepthMiddleware(object): if self.stats and self.maxdepth: stats.setpath('_envinfo/request_depth_limit', self.maxdepth) - def process_result(self, response, result, spider): + def process_spider_output(self, response, result, spider): def _filter(request): if isinstance(request, Request): request.depth = response.request.depth + 1 diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py index 0f6311959..7f5dd7548 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py @@ -21,7 +21,7 @@ class RequestLimitMiddleware(object): if not self.max_queue_size: raise NotConfigured - def process_result(self, response, result, spider): + def process_spider_output(self, response, result, spider): requests = [] items = [] for r in result: diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py index 7a0644d04..cea2cdbbd 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py @@ -8,7 +8,7 @@ from scrapy.http import Request from scrapy.utils.url import url_is_from_spider class OffsiteMiddleware(object): - def process_result(self, response, result, spider): + def process_spider_output(self, response, result, spider): def _filter(r): if isinstance(r, Request) and not url_is_from_spider(r.url, spider): log.msg("Ignoring link (offsite): %s " % r.url, level=log.DEBUG, domain=spider.domain_name) diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py index ee05f6918..e985e1129 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py @@ -6,7 +6,7 @@ originated it. from scrapy.http import Request class RefererMiddleware(object): - def process_result(self, response, result, spider): + def process_spider_output(self, response, result, spider): def _set_referer(r): if isinstance(r, Request): r.headers.setdefault('Referer', response.url) diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py index b4eceef31..5b01d7be7 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py @@ -12,7 +12,7 @@ class RestrictMiddleware(object): if not self.allowed_urls: raise NotConfigured - def process_result(self, response, result, spider): + def process_spider_output(self, response, result, spider): def _filter(r): if isinstance(r, Request) and r.url not in self.allowed_urls: return False diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/urlfilter.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/urlfilter.py index 1034f747d..e94c0c52e 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/urlfilter.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/urlfilter.py @@ -6,7 +6,7 @@ from scrapy.http import Request from scrapy.utils.url import canonicalize_url class UrlFilterMiddleware(object): - def process_result(self, response, result, spider): + def process_spider_output(self, response, result, spider): disabled = getattr(spider, 'urlfilter_disabled', False) for r in result: if isinstance(r, Request) and not disabled: diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py index cfca458ef..600904ce0 100644 --- a/scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py @@ -13,7 +13,7 @@ class UrlLengthMiddleware(object): if not self.maxlength: raise NotConfigured - def process_result(self, response, result, spider): + def process_spider_output(self, response, result, spider): def _filter(request): if isinstance(request, Request) and len(request.url) > self.maxlength: log.msg("Ignoring link (url length > %d): %s " % (self.maxlength, request.url), level=log.DEBUG, domain=spider.domain_name) diff --git a/scrapy/trunk/scrapy/contrib/spiders/crawl.py b/scrapy/trunk/scrapy/contrib/spiders/crawl.py index 7a2626be6..885072e32 100644 --- a/scrapy/trunk/scrapy/contrib/spiders/crawl.py +++ b/scrapy/trunk/scrapy/contrib/spiders/crawl.py @@ -67,7 +67,7 @@ class CrawlSpider(BaseSpider): return a list of ScrapedItems and/or Requests""" return [] - def process_results(self, results, response): + def process_spider_output(self, results, response): """This overridable method is called for each result (item or request) returned by the spider, and it's intended to perform any last time processing required before returning the results to the framework core, diff --git a/scrapy/trunk/scrapy/spider/middleware.py b/scrapy/trunk/scrapy/spider/middleware.py index 799e177df..19d0c4312 100644 --- a/scrapy/trunk/scrapy/spider/middleware.py +++ b/scrapy/trunk/scrapy/spider/middleware.py @@ -1,5 +1,9 @@ """ -Spider middleware manager +This module implements the Spider Middleware manager. For more information see +the Spider Middleware doc in: + +docs/topics/spider-middleware.rst + """ from scrapy import log @@ -23,12 +27,12 @@ class SpiderMiddlewareManager(object): self.load() def _add_middleware(self, mw): - if hasattr(mw, 'process_scrape'): - self.spider_middleware.append(mw.process_scrape) - if hasattr(mw, 'process_result'): - self.result_middleware.insert(0, mw.process_result) - if hasattr(mw, 'process_exception'): - self.exception_middleware.insert(0, mw.process_exception) + if hasattr(mw, 'process_spider_input'): + self.spider_middleware.append(mw.process_spider_input) + if hasattr(mw, 'process_spider_output'): + self.result_middleware.insert(0, mw.process_spider_output) + if hasattr(mw, 'process_spider_exception'): + self.exception_middleware.insert(0, mw.process_spider_exception) def load(self): """Load middleware defined in settings module""" @@ -48,7 +52,7 @@ class SpiderMiddlewareManager(object): def scrape(self, request, response, spider): fname = lambda f:'%s.%s' % (f.im_self.__class__.__name__, f.im_func.__name__) - def process_scrape(response): + def process_spider_input(response): for method in self.spider_middleware: result = method(response=response, spider=spider) assert result is None or _isiterable(result), \ @@ -59,7 +63,7 @@ class SpiderMiddlewareManager(object): return self.call(request=request, response=response, spider=spider) - def process_result(result): + def process_spider_output(result): for method in self.result_middleware: result = method(response=response, result=result, spider=spider) assert _isiterable(result), \ @@ -67,7 +71,7 @@ class SpiderMiddlewareManager(object): (fname(method), type(result)) return result - def process_exception(_failure): + def process_spider_exception(_failure): exception = _failure.value for method in self.exception_middleware: result = method(response=response, exception=exception, spider=spider) @@ -78,9 +82,9 @@ class SpiderMiddlewareManager(object): return result return _failure - dfd = mustbe_deferred(process_scrape, response) - dfd.addErrback(process_exception) - dfd.addCallback(process_result) + dfd = mustbe_deferred(process_spider_input, response) + dfd.addErrback(process_spider_exception) + dfd.addCallback(process_spider_output) return dfd def call(self, request, response, spider):