renamed spider middleware methods to more consistent ones: process_spider_input, process_spider_output, process_spider_exception

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40661

scrapy/trunk/scrapy/contrib/itemsampler.py

@@ -83,13 +83,13 @@ class ItemSamplerMiddleware(object):
         if not settings['ITEMSAMPLER_FILE']:
             raise NotConfigured
 
-    def process_scrape(self, response, spider):
+    def process_spider_input(self, response, spider):
         if stats.getpath("%s/items_sampled" % spider.domain_name) >= items_per_domain:
             return []
         elif max_response_size and max_response_size > len(response):  
             return []
 
-    def process_result(self, response, result, spider):
+    def process_spider_output(self, response, result, spider):
         requests, items = [], []
         for r in result:
             if isinstance(r, Request):

scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py

@@ -17,7 +17,7 @@ class DepthMiddleware(object):
         if self.stats and self.maxdepth:
             stats.setpath('_envinfo/request_depth_limit', self.maxdepth)
 
-    def process_result(self, response, result, spider):
+    def process_spider_output(self, response, result, spider):
         def _filter(request):
             if isinstance(request, Request):
                 request.depth = response.request.depth + 1

scrapy/trunk/scrapy/contrib/spidermiddleware/limit.py

@@ -21,7 +21,7 @@ class RequestLimitMiddleware(object):
         if not self.max_queue_size:
             raise NotConfigured
 
-    def process_result(self, response, result, spider):
+    def process_spider_output(self, response, result, spider):
         requests = []
         items = []
         for r in result:

scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py

@@ -8,7 +8,7 @@ from scrapy.http import Request
 from scrapy.utils.url import url_is_from_spider
 
 class OffsiteMiddleware(object):
-    def process_result(self, response, result, spider):
+    def process_spider_output(self, response, result, spider):
         def _filter(r):
             if isinstance(r, Request) and not url_is_from_spider(r.url, spider):
                 log.msg("Ignoring link (offsite): %s " % r.url, level=log.DEBUG, domain=spider.domain_name)

scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py

@@ -6,7 +6,7 @@ originated it.
 from scrapy.http import Request
 
 class RefererMiddleware(object):
-    def process_result(self, response, result, spider):
+    def process_spider_output(self, response, result, spider):
         def _set_referer(r):
             if isinstance(r, Request):
                 r.headers.setdefault('Referer', response.url)

scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py

@@ -12,7 +12,7 @@ class RestrictMiddleware(object):
         if not self.allowed_urls:
             raise NotConfigured
 
-    def process_result(self, response, result, spider):
+    def process_spider_output(self, response, result, spider):
         def _filter(r):
             if isinstance(r, Request) and r.url not in self.allowed_urls:
                 return False

scrapy/trunk/scrapy/contrib/spidermiddleware/urlfilter.py

@@ -6,7 +6,7 @@ from scrapy.http import Request
 from scrapy.utils.url import canonicalize_url
 
 class UrlFilterMiddleware(object):
-    def process_result(self, response, result, spider):
+    def process_spider_output(self, response, result, spider):
         disabled = getattr(spider, 'urlfilter_disabled', False)
         for r in result:
             if isinstance(r, Request) and not disabled:

scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py

@@ -13,7 +13,7 @@ class UrlLengthMiddleware(object):
         if not self.maxlength:
             raise NotConfigured
 
-    def process_result(self, response, result, spider):
+    def process_spider_output(self, response, result, spider):
         def _filter(request):
             if isinstance(request, Request) and len(request.url) > self.maxlength:
                 log.msg("Ignoring link (url length > %d): %s " % (self.maxlength, request.url), level=log.DEBUG, domain=spider.domain_name)

scrapy/trunk/scrapy/contrib/spiders/crawl.py

@@ -67,7 +67,7 @@ class CrawlSpider(BaseSpider):
         return a list of ScrapedItems and/or Requests"""
         return []
 
-    def process_results(self, results, response):
+    def process_spider_output(self, results, response):
         """This overridable method is called for each result (item or request)
         returned by the spider, and it's intended to perform any last time
         processing required before returning the results to the framework core,

scrapy/trunk/scrapy/spider/middleware.py

@@ -1,5 +1,9 @@
 """
-Spider middleware manager
+This module implements the Spider Middleware manager. For more information see
+the Spider Middleware doc in:
+
+docs/topics/spider-middleware.rst
+
 """
 
 from scrapy import log
@@ -23,12 +27,12 @@ class SpiderMiddlewareManager(object):
         self.load()
 
     def _add_middleware(self, mw):
-        if hasattr(mw, 'process_scrape'):
-            self.spider_middleware.append(mw.process_scrape)
-        if hasattr(mw, 'process_result'):
-            self.result_middleware.insert(0, mw.process_result)
-        if hasattr(mw, 'process_exception'):
-            self.exception_middleware.insert(0, mw.process_exception)
+        if hasattr(mw, 'process_spider_input'):
+            self.spider_middleware.append(mw.process_spider_input)
+        if hasattr(mw, 'process_spider_output'):
+            self.result_middleware.insert(0, mw.process_spider_output)
+        if hasattr(mw, 'process_spider_exception'):
+            self.exception_middleware.insert(0, mw.process_spider_exception)
 
     def load(self):
         """Load middleware defined in settings module"""
@@ -48,7 +52,7 @@ class SpiderMiddlewareManager(object):
     def scrape(self, request, response, spider):
         fname = lambda f:'%s.%s' % (f.im_self.__class__.__name__, f.im_func.__name__)
 
-        def process_scrape(response):
+        def process_spider_input(response):
             for method in self.spider_middleware:
                 result = method(response=response, spider=spider)
                 assert result is None or _isiterable(result), \
@@ -59,7 +63,7 @@ class SpiderMiddlewareManager(object):
             return self.call(request=request, response=response, spider=spider)
 
 
-        def process_result(result):
+        def process_spider_output(result):
             for method in self.result_middleware:
                 result = method(response=response, result=result, spider=spider)
                 assert _isiterable(result), \
@@ -67,7 +71,7 @@ class SpiderMiddlewareManager(object):
                     (fname(method), type(result))
             return result
 
-        def process_exception(_failure):
+        def process_spider_exception(_failure):
             exception = _failure.value
             for method in self.exception_middleware:
                 result = method(response=response, exception=exception, spider=spider)
@@ -78,9 +82,9 @@ class SpiderMiddlewareManager(object):
                     return result
             return _failure
 
-        dfd = mustbe_deferred(process_scrape, response)
-        dfd.addErrback(process_exception)
-        dfd.addCallback(process_result)
+        dfd = mustbe_deferred(process_spider_input, response)
+        dfd.addErrback(process_spider_exception)
+        dfd.addCallback(process_spider_output)
         return dfd
 
     def call(self, request, response, spider):