finally remove HttpException

in this changeset:
* remove HttpException from engine and core exceptions
* replace dwmw ErrorPages with spidermw HttpError
* bugfix image pipeline media_to_download method when stat_key returns None

scrapy/conf/default_settings.py

@@ -59,7 +59,6 @@ DOWNLOADER_MIDDLEWARES = {}
 DOWNLOADER_MIDDLEWARES_BASE = {
     # Engine side
     'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100,
-    'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware': 200,
     'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300,
     'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400,
     'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500,
@@ -170,6 +169,7 @@ SPIDER_MIDDLEWARES = {}
 
 SPIDER_MIDDLEWARES_BASE = {
     # Engine side
+    'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50,
     'scrapy.contrib.itemsampler.ItemSamplerMiddleware': 100,
     'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware': 200,
     'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware': 300,

scrapy/contrib/downloadermiddleware/errorpages.py

@@ -1,4 +1,4 @@
-from scrapy.core.exceptions import HttpException
+from scrapy.core.exceptions import IgnoreRequest
 from scrapy.utils.response import response_status_message
 
 class ErrorPagesMiddleware(object):
@@ -13,5 +13,5 @@ class ErrorPagesMiddleware(object):
         if 200 <= status < 300 or status in getattr(spider, 'handle_httpstatus_list', []):
             return response
         else:
-            raise HttpException(status, None, response)
+            raise IgnoreRequest(response_status_message(status))
 

scrapy/contrib/pipeline/images.py

@@ -11,7 +11,6 @@ from scrapy import log
 from scrapy.stats import stats
 from scrapy.utils.misc import md5sum
 from scrapy.core.exceptions import DropItem, NotConfigured
-from scrapy.core.exceptions import HttpException, IgnoreRequest
 from scrapy.conf import settings
 from scrapy.contrib.pipeline.media import MediaPipeline
 
@@ -105,6 +104,9 @@ class BaseImagesPipeline(MediaPipeline):
 
     def media_to_download(self, request, info):
         def _onsuccess(result):
+            if not result:
+                return # returning None force download
+
             last_modified = result.get('last_modified', None)
             if not last_modified:
                 return # returning None force download

scrapy/contrib/spidermiddleware/httperror.py

@@ -0,0 +1,13 @@
+
+
+class HttpErrorMiddleware(object):
+    """Filter out response outside of a range of valid status codes
+
+    This middleware filters out every response with status outside of the range 200<=status<300
+    Spiders can add more exceptions using `handle_httpstatus_list` spider attribute.
+    """
+
+    def process_spider_input(self, response, spider):
+        if not (200 <= response.status < 300 or \
+                response.status in getattr(spider, 'handle_httpstatus_list', [])):
+            return [] # skip response

scrapy/core/engine.py

@@ -16,7 +16,7 @@ from scrapy.conf import settings
 from scrapy.core import signals
 from scrapy.core.scheduler import Scheduler, SchedulerMiddlewareManager
 from scrapy.core.downloader import Downloader
-from scrapy.core.exceptions import IgnoreRequest, HttpException, DontCloseDomain
+from scrapy.core.exceptions import IgnoreRequest, DontCloseDomain
 from scrapy.http import Response, Request
 from scrapy.item import ScrapedItem
 from scrapy.item.pipeline import ItemPipelineManager
@@ -330,6 +330,7 @@ class ExecutionEngine(object):
         if self.debug_mode:
             log.msg('Downloading %s' % request_info(request), log.DEBUG)
         domain = spider.domain_name
+        referer = request.headers.get('Referer', None)
 
         def _on_success(response):
             """handle the result of a page download"""
@@ -338,7 +339,7 @@ class ExecutionEngine(object):
                 log.msg("Requested %s" % request_info(request), level=log.DEBUG, domain=domain)
             if isinstance(response, Response):
                 response.request = request # tie request to obtained response
-                log.msg("Crawled %s from <%s>" % (response, request.headers.get('referer')), level=log.DEBUG, domain=domain)
+                log.msg("Crawled %s from <%s>" % (response, referer), level=log.DEBUG, domain=domain)
                 return response
             elif isinstance(response, Request):
                 redirected = response # proper alias
@@ -349,11 +350,7 @@ class ExecutionEngine(object):
         def _on_error(_failure):
             """handle an error processing a page"""
             ex = _failure.value
-            if isinstance(ex, IgnoreRequest):
-                log.msg(_failure.getErrorMessage(), level=log.DEBUG, domain=domain)
-                return _failure
-            referer = request.headers.get('Referer', None)
-            errmsg = str(ex) if isinstance(ex, HttpException) else str(_failure)
+            errmsg = str(_failure) if not isinstance(ex, IgnoreRequest) else _failure.getErrorMessage()
             log.msg("Downloading <%s> from <%s>: %s" % (request.url, referer, errmsg), log.ERROR, domain=domain)
             return Failure(IgnoreRequest(str(ex)))
 

scrapy/core/exceptions.py

@@ -21,20 +21,6 @@ class DontCloseDomain(Exception):
     """Request the domain not to be closed yet"""
     pass
 
-class HttpException(Exception):
-    def __init__(self, status, message, response):
-        if not message:
-            from twisted.web import http
-            message = http.responses.get(int(status))
-
-        self.status = int(status)
-        self.message = message
-        self.response = response
-        Exception.__init__(self, status, message, response)
-
-    def __str__(self):
-        return '%s %s' % (self.status, self.message)
-
 # Items
 
 class DropItem(Exception):