mirror of https://github.com/scrapy/scrapy.git
merged with Daniel's HttpException-removal branch
This commit is contained in:
Pablo Hoffman
commit
04e7f8f5f6
|
|
@ -4,11 +4,14 @@
|
|||
Built-in downloader middleware reference
|
||||
========================================
|
||||
|
||||
This document explains all downloader middleware components that come with
|
||||
This page describes all downloader middleware components that come with
|
||||
Scrapy. For information on how to use them and how to write your own downloader
|
||||
middleware, see the :ref:`downloader middleware usage guide
|
||||
<topics-downloader-middleware>`.
|
||||
|
||||
For a list of the components enabled by default (and their orders) see the
|
||||
:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting.
|
||||
|
||||
Available downloader middlewares
|
||||
================================
|
||||
|
||||
|
|
|
|||
|
|
@ -265,7 +265,7 @@ Default::
|
|||
}
|
||||
|
||||
The default headers used for Scrapy HTTP Requests. They're populated in the
|
||||
DefaultHeadersMiddleware.
|
||||
:class:`~scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware`.
|
||||
|
||||
.. setting:: DEFAULT_SPIDER
|
||||
|
||||
|
|
@ -669,6 +669,27 @@ Default: ``Not Defined``
|
|||
The name of the current project. It matches the project module name as created
|
||||
by ``startproject`` command, and is only defined by project settings file.
|
||||
|
||||
.. setting:: REDIRECT_MAX_TIMES
|
||||
|
||||
REDIRECT_MAX_TIMES
|
||||
------------------
|
||||
|
||||
Default: ``20``
|
||||
|
||||
Defines the maximun times a request can be redirected. After this maximun the
|
||||
request's response is returned as is. We used Firefox default value for the
|
||||
same task.
|
||||
|
||||
.. setting:: REDIRECT_MAX_METAREFRESH_DELAY
|
||||
|
||||
REDIRECT_MAX_METAREFRESH_DELAY
|
||||
------------------------------
|
||||
|
||||
Default: ``100``
|
||||
|
||||
Some sites use meta-refresh for redirecting to a session expired page, so we
|
||||
restrict automatic redirection to a maximum delay (in seconds)
|
||||
|
||||
.. setting:: REQUESTS_QUEUE_SIZE
|
||||
|
||||
REQUESTS_PER_DOMAIN
|
||||
|
|
@ -901,7 +922,7 @@ The default User-Agent to use when crawling, unless overrided.
|
|||
WEBCONSOLE_ENABLED
|
||||
------------------
|
||||
|
||||
Default: ``"%s/%s" % (BOT_NAME, BOT_VERSION)``
|
||||
Default: True
|
||||
|
||||
A boolean which specifies if the web management console will be enabled
|
||||
(provided its extension is also enabled).
|
||||
|
|
|
|||
|
|
@ -64,7 +64,6 @@ DOWNLOADER_MIDDLEWARES = {}
|
|||
DOWNLOADER_MIDDLEWARES_BASE = {
|
||||
# Engine side
|
||||
'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100,
|
||||
'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware': 200,
|
||||
'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300,
|
||||
'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400,
|
||||
'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500,
|
||||
|
|
@ -140,14 +139,15 @@ NEWSPIDER_MODULE = ''
|
|||
|
||||
PRIORITIZER = 'scrapy.core.prioritizers.RandomPrioritizer'
|
||||
|
||||
REDIRECTMIDDLEWARE_MAX_TIMES = 20 # uses Firefox default setting
|
||||
REDIRECT_MAX_METAREFRESH_DELAY = 100
|
||||
REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
|
||||
|
||||
REQUESTS_QUEUE_SIZE = 0
|
||||
REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain
|
||||
|
||||
# contrib.middleware.retry.RetryMiddleware default settings
|
||||
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
|
||||
RETRY_HTTP_CODES = ['500', '503', '504', '400', '408', '200']
|
||||
RETRY_HTTP_CODES = ['500', '503', '504', '400', '408']
|
||||
|
||||
|
||||
ROBOTSTXT_OBEY = False
|
||||
|
|
@ -170,6 +170,7 @@ SPIDER_MIDDLEWARES = {}
|
|||
|
||||
SPIDER_MIDDLEWARES_BASE = {
|
||||
# Engine side
|
||||
'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50,
|
||||
'scrapy.contrib.itemsampler.ItemSamplerMiddleware': 100,
|
||||
'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware': 200,
|
||||
'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware': 300,
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ from pydispatch import dispatcher
|
|||
from scrapy.core import signals
|
||||
from scrapy.http import Response
|
||||
from scrapy.http.cookies import CookieJar
|
||||
from scrapy.core.exceptions import HttpException
|
||||
from scrapy.conf import settings
|
||||
from scrapy import log
|
||||
|
||||
|
|
@ -44,11 +43,6 @@ class CookiesMiddleware(object):
|
|||
|
||||
return response
|
||||
|
||||
# cookies should be set on non-200 responses too
|
||||
def process_exception(self, request, exception, spider):
|
||||
if isinstance(exception, HttpException):
|
||||
self.process_response(request, exception.response, spider)
|
||||
|
||||
def domain_closed(self, domain):
|
||||
self.jars.pop(domain, None)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,17 @@
|
|||
from scrapy.core.exceptions import HttpException
|
||||
from scrapy.core.exceptions import IgnoreRequest
|
||||
from scrapy.utils.response import response_status_message
|
||||
|
||||
class ErrorPagesMiddleware(object):
|
||||
"""This middleware allows the spiders to receive error (non 200) responses,
|
||||
the same way the receive normal responses"""
|
||||
"""This middleware filters out responses with status code others than 2XX
|
||||
or defined in spider handle_httpstatus_list attribute.
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
if isinstance(exception, HttpException):
|
||||
statuses = getattr(spider, 'handle_httpstatus_list', None)
|
||||
httpstatus = exception.response.status
|
||||
if statuses and httpstatus in statuses:
|
||||
return exception.response
|
||||
TODO: move this mw to spidermiddleware and remove me
|
||||
"""
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
status = response.status
|
||||
if 200 <= status < 300 or status in getattr(spider, 'handle_httpstatus_list', []):
|
||||
return response
|
||||
else:
|
||||
raise IgnoreRequest(response_status_message(status))
|
||||
|
||||
|
|
|
|||
|
|
@ -10,12 +10,13 @@ from pydispatch import dispatcher
|
|||
from scrapy.core import signals
|
||||
from scrapy import log
|
||||
from scrapy.http import Response, Headers
|
||||
from scrapy.core.exceptions import NotConfigured, HttpException, IgnoreRequest
|
||||
from scrapy.core.exceptions import NotConfigured, IgnoreRequest
|
||||
from scrapy.core.downloader.responsetypes import responsetypes
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.http import headers_dict_to_raw, headers_raw_to_dict
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
||||
class HttpCacheMiddleware(object):
|
||||
def __init__(self):
|
||||
if not settings['HTTPCACHE_DIR']:
|
||||
|
|
@ -33,37 +34,25 @@ class HttpCacheMiddleware(object):
|
|||
|
||||
key = request_fingerprint(request)
|
||||
domain = spider.domain_name
|
||||
|
||||
try:
|
||||
response = self.cache.retrieve_response(domain, key)
|
||||
except:
|
||||
log.msg("Corrupt cache for %s" % request.url, log.WARNING)
|
||||
response = False
|
||||
|
||||
if response:
|
||||
if not 200 <= int(response.status) < 300:
|
||||
raise HttpException(response.status, None, response)
|
||||
return response
|
||||
elif self.ignore_missing:
|
||||
raise IgnoreRequest("Ignored request not in cache: %s" % request)
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if not is_cacheable(request):
|
||||
return response
|
||||
|
||||
if isinstance(response, Response) and not response.meta.get('cached'):
|
||||
if is_cacheable(request):
|
||||
key = request_fingerprint(request)
|
||||
domain = spider.domain_name
|
||||
self.cache.store(domain, key, request, response)
|
||||
self.cache.store(spider.domain_name, key, request, response)
|
||||
|
||||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
if not is_cacheable(request):
|
||||
return
|
||||
|
||||
if isinstance(exception, HttpException) and isinstance(exception.response, Response):
|
||||
key = request_fingerprint(request)
|
||||
domain = spider.domain_name
|
||||
self.cache.store(domain, key, request, exception.response)
|
||||
|
||||
def is_cacheable(request):
|
||||
return request.url.scheme in ['http', 'https']
|
||||
|
|
|
|||
|
|
@ -1,41 +1,33 @@
|
|||
from scrapy import log
|
||||
from scrapy.core.exceptions import HttpException
|
||||
from scrapy.utils.url import urljoin_rfc as urljoin
|
||||
from scrapy.utils.response import get_meta_refresh
|
||||
from scrapy.conf import settings
|
||||
|
||||
# some sites use meta-refresh for redirecting to a session expired page, so we
|
||||
# restrict automatic redirection to a maximum delay (in number of seconds)
|
||||
META_REFRESH_MAXSEC = 100
|
||||
MAX_REDIRECT_LOOP = 10
|
||||
|
||||
class RedirectMiddleware(object):
|
||||
"""Handle redirection of requests based on response status and meta-refresh html tag"""
|
||||
|
||||
def __init__(self):
|
||||
self.max_redirect_times = settings.getint('REDIRECTMIDDLEWARE_MAX_TIMES')
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
if not isinstance(exception, HttpException):
|
||||
return
|
||||
self.max_metarefresh_delay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY')
|
||||
self.max_redirect_times = settings.getint('REDIRECT_MAX_TIMES')
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
domain = spider.domain_name
|
||||
status = exception.status
|
||||
response = exception.response
|
||||
|
||||
if status in [302, 303] and 'Location' in response.headers:
|
||||
if response.status in [302, 303] and 'Location' in response.headers:
|
||||
redirected_url = urljoin(request.url, response.headers['location'])
|
||||
redirected = request.replace(url=redirected_url, method='GET', body='')
|
||||
redirected.headers.pop('Content-Type', None)
|
||||
redirected.headers.pop('Content-Length', None)
|
||||
return self._redirect(redirected, request, spider, status)
|
||||
return self._redirect(redirected, request, spider, response.status)
|
||||
|
||||
if status in [301, 307] and 'Location' in response.headers:
|
||||
if response.status in [301, 307] and 'Location' in response.headers:
|
||||
redirected_url = urljoin(request.url, response.headers['location'])
|
||||
redirected = request.replace(url=redirected_url)
|
||||
return self._redirect(redirected, request, spider, status)
|
||||
return self._redirect(redirected, request, spider, response.status)
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
interval, url = get_meta_refresh(response)
|
||||
if url and int(interval) < META_REFRESH_MAXSEC:
|
||||
if url and int(interval) < self.max_metarefresh_delay:
|
||||
redirected = request.replace(url=urljoin(request.url, url))
|
||||
return self._redirect(redirected, request, spider, 'meta refresh')
|
||||
|
||||
|
|
|
|||
|
|
@ -19,46 +19,50 @@ About HTTP errors to consider:
|
|||
- You may want to remove 400 from RETRY_HTTP_CODES, if you stick to the HTTP
|
||||
protocol. It's included by default because it's a common code used to
|
||||
indicate server overload, which would be something we want to retry
|
||||
- 200 is included by default (and shoudln't be removed) to check for partial
|
||||
downloads errors, which means the TCP connection has broken in the middle of
|
||||
a HTTP download
|
||||
"""
|
||||
|
||||
from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError, \
|
||||
ConnectionLost
|
||||
from twisted.internet.defer import TimeoutError as UserTimeoutError
|
||||
from twisted.web.client import PartialDownloadError
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.core.exceptions import HttpException
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.response import response_status_message
|
||||
from scrapy.conf import settings
|
||||
|
||||
class RetryMiddleware(object):
|
||||
|
||||
EXCEPTIONS_TO_RETRY = (ServerTimeoutError, UserTimeoutError, DNSLookupError,
|
||||
ConnectionRefusedError, ConnectionDone, ConnectError,
|
||||
ConnectionLost)
|
||||
ConnectionLost, PartialDownloadError)
|
||||
|
||||
def __init__(self):
|
||||
self.max_retry_times = settings.getint('RETRY_TIMES')
|
||||
self.retry_http_codes = map(int, settings.getlist('RETRY_HTTP_CODES'))
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if response.status in self.retry_http_codes:
|
||||
reason = response_status_message(response.status)
|
||||
return self._retry(request, reason, spider) or response
|
||||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
|
||||
or (isinstance(exception, HttpException) \
|
||||
and (int(exception.status) in self.retry_http_codes)):
|
||||
if isinstance(exception, self.EXCEPTIONS_TO_RETRY):
|
||||
return self._retry(request, exception, spider)
|
||||
|
||||
retries = request.meta.get('retry_times', 0) + 1
|
||||
def _retry(self, request, reason, spider):
|
||||
retries = request.meta.get('retry_times', 0) + 1
|
||||
|
||||
if retries <= self.max_retry_times:
|
||||
log.msg("Retrying %s (failed %d times): %s" % (request, retries, exception),
|
||||
domain=spider.domain_name, level=log.DEBUG)
|
||||
retryreq = request.copy()
|
||||
retryreq.meta['retry_times'] = retries
|
||||
retryreq.dont_filter = True
|
||||
return retryreq
|
||||
else:
|
||||
log.msg("Discarding %s (failed %d times): %s" % (request, retries, exception),
|
||||
domain=spider.domain_name, level=log.DEBUG)
|
||||
if retries <= self.max_retry_times:
|
||||
log.msg("Retrying %s (failed %d times): %s" % (request, retries, reason),
|
||||
domain=spider.domain_name, level=log.DEBUG)
|
||||
retryreq = request.copy()
|
||||
retryreq.meta['retry_times'] = retries
|
||||
retryreq.dont_filter = True
|
||||
return retryreq
|
||||
else:
|
||||
log.msg("Discarding %s (failed %d times): %s" % (request, retries, reason),
|
||||
domain=spider.domain_name, level=log.DEBUG)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from scrapy.core.exceptions import HttpException, NotConfigured
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.stats import stats
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
|
@ -18,7 +18,7 @@ class DownloaderStats(object):
|
|||
def __init__(self):
|
||||
if not settings.getbool('DOWNLOADER_STATS'):
|
||||
raise NotConfigured
|
||||
|
||||
|
||||
def process_request(self, request, spider):
|
||||
stats.incpath('_global/downloader/request_count')
|
||||
stats.incpath('%s/downloader/request_count' % spider.domain_name)
|
||||
|
|
@ -30,14 +30,12 @@ class DownloaderStats(object):
|
|||
def process_response(self, request, response, spider):
|
||||
self._inc_response_count(response, spider.domain_name)
|
||||
return response
|
||||
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__)
|
||||
stats.incpath('_global/downloader/exception_count')
|
||||
stats.incpath('%s/downloader/exception_count' % spider.domain_name)
|
||||
stats.incpath('%s/downloader/exception_type_count/%s' % (spider.domain_name, ex_class))
|
||||
if isinstance(exception, HttpException):
|
||||
self._inc_response_count(exception.response, spider.domain_name)
|
||||
|
||||
def _inc_response_count(self, response, domain):
|
||||
stats.incpath('_global/downloader/response_count')
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ from scrapy import log
|
|||
from scrapy.stats import stats
|
||||
from scrapy.utils.misc import md5sum
|
||||
from scrapy.core.exceptions import DropItem, NotConfigured
|
||||
from scrapy.core.exceptions import HttpException, IgnoreRequest
|
||||
from scrapy.conf import settings
|
||||
from scrapy.contrib.pipeline.media import MediaPipeline
|
||||
|
||||
|
|
@ -62,20 +61,25 @@ class BaseImagesPipeline(MediaPipeline):
|
|||
# ('270', (270, 270))
|
||||
)
|
||||
|
||||
|
||||
|
||||
def media_downloaded(self, response, request, info):
|
||||
mtype = self.MEDIA_NAME
|
||||
referer = request.headers.get('Referer')
|
||||
|
||||
if not response or not response.body:
|
||||
msg = 'Image (empty): Empty %s (no content) in %s referred in <%s>: no-content' \
|
||||
if response.status != 200:
|
||||
msg = 'Image (http-error): Error downloading %s from %s referred in <%s>: %s' \
|
||||
% (mtype, request, referer, errmsg)
|
||||
log.msg(msg, level=log.WARNING, domain=info.domain)
|
||||
raise ImageException(msg)
|
||||
|
||||
if not response.body:
|
||||
msg = 'Image (empty-content): Empty %s from %s referred in <%s>: no-content' \
|
||||
% (mtype, request, referer)
|
||||
log.msg(msg, level=log.WARNING, domain=info.domain)
|
||||
raise ImageException(msg)
|
||||
|
||||
status = 'cached' if getattr(response, 'cached', False) else 'downloaded'
|
||||
msg = 'Image (%s): Downloaded %s from %s referred in <%s>' % (status, mtype, request, referer)
|
||||
status = 'cached' if 'cached' in response.flags else 'downloaded'
|
||||
msg = 'Image (%s): Downloaded %s from %s referred in <%s>' % \
|
||||
(status, mtype, request, referer)
|
||||
log.msg(msg, level=log.DEBUG, domain=info.domain)
|
||||
self.inc_stats(info.domain, status)
|
||||
|
||||
|
|
@ -93,18 +97,16 @@ class BaseImagesPipeline(MediaPipeline):
|
|||
|
||||
def media_failed(self, failure, request, info):
|
||||
referer = request.headers.get('Referer')
|
||||
if isinstance(failure.value, (HttpException, IgnoreRequest)):
|
||||
errmsg = str(failure.value)
|
||||
else:
|
||||
errmsg = str(failure)
|
||||
|
||||
msg = 'Image (http-error): Error downloading %s from %s referred in <%s>: %s' \
|
||||
% (self.MEDIA_NAME, request, referer, errmsg)
|
||||
msg = 'Image (unknow-error): Error downloading %s from %s referred in <%s>: %s' \
|
||||
% (self.MEDIA_NAME, request, referer, str(failure))
|
||||
log.msg(msg, level=log.WARNING, domain=info.domain)
|
||||
raise ImageException(msg)
|
||||
|
||||
def media_to_download(self, request, info):
|
||||
def _onsuccess(result):
|
||||
if not result:
|
||||
return # returning None force download
|
||||
|
||||
last_modified = result.get('last_modified', None)
|
||||
if not last_modified:
|
||||
return # returning None force download
|
||||
|
|
|
|||
|
|
@ -59,11 +59,12 @@ class S3ImagesPipeline(BaseImagesPipeline):
|
|||
|
||||
def stat_key(self, key, info):
|
||||
def _onsuccess(response):
|
||||
checksum = response.headers['Etag'].strip('"')
|
||||
last_modified = response.headers['Last-Modified']
|
||||
modified_tuple = rfc822.parsedate_tz(last_modified)
|
||||
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
|
||||
return {'checksum': checksum, 'last_modified': modified_stamp}
|
||||
if response.status == 200:
|
||||
checksum = response.headers['Etag'].strip('"')
|
||||
last_modified = response.headers['Last-Modified']
|
||||
modified_tuple = rfc822.parsedate_tz(last_modified)
|
||||
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
|
||||
return {'checksum': checksum, 'last_modified': modified_stamp}
|
||||
|
||||
req = self.s3_request(key, method='HEAD')
|
||||
dfd = self.s3_download(req, info)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
|
||||
|
||||
class HttpErrorMiddleware(object):
|
||||
"""Filter out response outside of a range of valid status codes
|
||||
|
||||
This middleware filters out every response with status outside of the range 200<=status<300
|
||||
Spiders can add more exceptions using `handle_httpstatus_list` spider attribute.
|
||||
"""
|
||||
|
||||
def process_spider_input(self, response, spider):
|
||||
if not (200 <= response.status < 300 or \
|
||||
response.status in getattr(spider, 'handle_httpstatus_list', [])):
|
||||
return [] # skip response
|
||||
|
|
@ -83,7 +83,7 @@ class DecompressionMiddleware(object):
|
|||
return (response, None)
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if isinstance(response, Response):
|
||||
if isinstance(response, Response) and response.body:
|
||||
response, format = self.extract(response)
|
||||
if format:
|
||||
log.msg('Decompressed response with format: %s' % format, log.DEBUG, domain=spider.domain_name)
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ except ImportError:
|
|||
from scrapy import optional_features
|
||||
from scrapy.core import signals
|
||||
from scrapy.http import Headers
|
||||
from scrapy.core.exceptions import HttpException, NotSupported
|
||||
from scrapy.core.exceptions import NotSupported
|
||||
from scrapy.utils.defer import defer_succeed
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
|
@ -65,13 +65,7 @@ def create_factory(request, spider):
|
|||
signals.send_catch_log(signal=signals.response_downloaded, sender='download_http', response=r, spider=spider)
|
||||
return r
|
||||
|
||||
def _on_success(body):
|
||||
response = _create_response(body)
|
||||
if response.status not in (200, 201, 202):
|
||||
raise HttpException(response.status, None, response)
|
||||
return response
|
||||
|
||||
factory.deferred.addCallbacks(_on_success)
|
||||
factory.deferred.addCallbacks(_create_response)
|
||||
return factory
|
||||
|
||||
def download_http(request, spider):
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from scrapy.conf import settings
|
|||
from scrapy.core import signals
|
||||
from scrapy.core.scheduler import Scheduler, SchedulerMiddlewareManager
|
||||
from scrapy.core.downloader import Downloader
|
||||
from scrapy.core.exceptions import IgnoreRequest, HttpException, DontCloseDomain
|
||||
from scrapy.core.exceptions import IgnoreRequest, DontCloseDomain
|
||||
from scrapy.http import Response, Request
|
||||
from scrapy.item import ScrapedItem
|
||||
from scrapy.item.pipeline import ItemPipelineManager
|
||||
|
|
@ -330,6 +330,7 @@ class ExecutionEngine(object):
|
|||
if self.debug_mode:
|
||||
log.msg('Downloading %s' % request_info(request), log.DEBUG)
|
||||
domain = spider.domain_name
|
||||
referer = request.headers.get('Referer', None)
|
||||
|
||||
def _on_success(response):
|
||||
"""handle the result of a page download"""
|
||||
|
|
@ -338,7 +339,7 @@ class ExecutionEngine(object):
|
|||
log.msg("Requested %s" % request_info(request), level=log.DEBUG, domain=domain)
|
||||
if isinstance(response, Response):
|
||||
response.request = request # tie request to obtained response
|
||||
log.msg("Crawled %s from <%s>" % (response, request.headers.get('referer')), level=log.DEBUG, domain=domain)
|
||||
log.msg("Crawled %s from <%s>" % (response, referer), level=log.DEBUG, domain=domain)
|
||||
return response
|
||||
elif isinstance(response, Request):
|
||||
redirected = response # proper alias
|
||||
|
|
@ -349,11 +350,7 @@ class ExecutionEngine(object):
|
|||
def _on_error(_failure):
|
||||
"""handle an error processing a page"""
|
||||
ex = _failure.value
|
||||
if isinstance(ex, IgnoreRequest):
|
||||
log.msg(_failure.getErrorMessage(), level=log.DEBUG, domain=domain)
|
||||
return _failure
|
||||
referer = request.headers.get('Referer', None)
|
||||
errmsg = str(ex) if isinstance(ex, HttpException) else str(_failure)
|
||||
errmsg = str(_failure) if not isinstance(ex, IgnoreRequest) else _failure.getErrorMessage()
|
||||
log.msg("Downloading <%s> from <%s>: %s" % (request.url, referer, errmsg), log.ERROR, domain=domain)
|
||||
return Failure(IgnoreRequest(str(ex)))
|
||||
|
||||
|
|
|
|||
|
|
@ -21,20 +21,6 @@ class DontCloseDomain(Exception):
|
|||
"""Request the domain not to be closed yet"""
|
||||
pass
|
||||
|
||||
class HttpException(Exception):
|
||||
def __init__(self, status, message, response):
|
||||
if not message:
|
||||
from twisted.web import http
|
||||
message = http.responses.get(int(status))
|
||||
|
||||
self.status = int(status)
|
||||
self.message = message
|
||||
self.response = response
|
||||
Exception.__init__(self, status, message, response)
|
||||
|
||||
def __str__(self):
|
||||
return '%s %s' % (self.status, self.message)
|
||||
|
||||
# Items
|
||||
|
||||
class DropItem(Exception):
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ from __future__ import with_statement
|
|||
from unittest import TestCase
|
||||
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.core.exceptions import HttpException
|
||||
from scrapy.http import Response, Request
|
||||
from scrapy.contrib.downloadermiddleware.cookies import CookiesMiddleware
|
||||
|
||||
|
|
@ -35,21 +34,6 @@ class CookiesMiddlewareTest(TestCase):
|
|||
assert self.mw.process_request(req2, self.spider) is None
|
||||
self.assertEquals(req2.headers.get('Cookie'), "C1=value1")
|
||||
|
||||
def test_http_exception(self):
|
||||
req = Request('http://scrapytest.org/')
|
||||
assert self.mw.process_request(req, self.spider) is None
|
||||
assert 'Cookie' not in req.headers
|
||||
|
||||
headers = {'Set-Cookie': 'C1=value1; path=/'}
|
||||
res = Response('http://scrapytest.org/', headers=headers)
|
||||
exc = HttpException(302, 'Redirect', res)
|
||||
assert self.mw.process_exception(req, exc, self.spider) is None
|
||||
#assert exc.response.cookies
|
||||
|
||||
req2 = Request('http://scrapytest.org/sub1/')
|
||||
assert self.mw.process_request(req2, self.spider) is None
|
||||
self.assertEquals(req2.headers.get('Cookie'), "C1=value1")
|
||||
|
||||
def test_dont_merge_cookies(self):
|
||||
# merge some cookies into jar
|
||||
headers = {'Set-Cookie': 'C1=value1; path=/'}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import unittest
|
||||
|
||||
from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware
|
||||
from scrapy.core.exceptions import HttpException
|
||||
from scrapy.dupefilter import dupefilter
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.http import Request, Response, Headers
|
||||
|
|
@ -22,28 +21,24 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
url = 'http://www.example.com/301'
|
||||
url2 = 'http://www.example.com/redirected'
|
||||
req = Request(url)
|
||||
hdr = Headers({'Location': [url2]})
|
||||
rsp = Response(url, headers=hdr)
|
||||
exc = HttpException('301', None, rsp)
|
||||
rsp = Response(url, headers={'Location': url2}, status=301)
|
||||
|
||||
req2 = self.mw.process_exception(req, exc, self.spider)
|
||||
req2 = self.mw.process_response(req, rsp, self.spider)
|
||||
assert isinstance(req2, Request)
|
||||
self.assertEqual(req2.url, url2)
|
||||
|
||||
# response without Location header but with status code is 3XX should be ignored
|
||||
del rsp.headers['Location']
|
||||
assert self.mw.process_exception(req, exc, self.spider) is None
|
||||
assert self.mw.process_response(req, rsp, self.spider) is rsp
|
||||
|
||||
def test_redirect_302(self):
|
||||
url = 'http://www.example.com/302'
|
||||
url2 = 'http://www.example.com/redirected2'
|
||||
req = Request(url, method='POST', body='test',
|
||||
headers={'Content-Type': 'text/plain', 'Content-length': '4'})
|
||||
hdr = Headers({'Location': [url2]})
|
||||
rsp = Response(url, headers=hdr)
|
||||
exc = HttpException('302', None, rsp)
|
||||
rsp = Response(url, headers={'Location': url2}, status=302)
|
||||
|
||||
req2 = self.mw.process_exception(req, exc, self.spider)
|
||||
req2 = self.mw.process_response(req, rsp, self.spider)
|
||||
assert isinstance(req2, Request)
|
||||
self.assertEqual(req2.url, url2)
|
||||
self.assertEqual(req2.method, 'GET')
|
||||
|
|
@ -56,7 +51,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
|
||||
# response without Location header but with status code is 3XX should be ignored
|
||||
del rsp.headers['Location']
|
||||
assert self.mw.process_exception(req, exc, self.spider) is None
|
||||
assert self.mw.process_response(req, rsp, self.spider) is rsp
|
||||
|
||||
def test_meta_refresh(self):
|
||||
body = """<html>
|
||||
|
|
@ -82,24 +77,24 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
def test_max_redirect_times(self):
|
||||
self.mw.max_redirect_times = 1
|
||||
req = Request('http://scrapytest.org/302')
|
||||
exc = HttpException('302', None, Response('http://www.scrapytest.org/302', headers={'Location': '/redirected'}))
|
||||
rsp = Response('http://scrapytest.org/302', headers={'Location': '/redirected'}, status=302)
|
||||
|
||||
req = self.mw.process_exception(req, exc, self.spider)
|
||||
req = self.mw.process_response(req, rsp, self.spider)
|
||||
assert isinstance(req, Request)
|
||||
assert 'redirect_times' in req.meta
|
||||
self.assertEqual(req.meta['redirect_times'], 1)
|
||||
|
||||
req = self.mw.process_exception(req, exc, self.spider)
|
||||
req = self.mw.process_response(req, rsp, self.spider)
|
||||
self.assertEqual(req, None)
|
||||
|
||||
def test_ttl(self):
|
||||
self.mw.max_redirect_times = 100
|
||||
req = Request('http://scrapytest.org/302', meta={'redirect_ttl': 1})
|
||||
exc = HttpException('302', None, Response('http://www.scrapytest.org/302', headers={'Location': '/redirected'}))
|
||||
rsp = Response('http://www.scrapytest.org/302', headers={'Location': '/redirected'}, status=302)
|
||||
|
||||
req = self.mw.process_exception(req, exc, self.spider)
|
||||
req = self.mw.process_response(req, rsp, self.spider)
|
||||
assert isinstance(req, Request)
|
||||
req = self.mw.process_exception(req, exc, self.spider)
|
||||
req = self.mw.process_response(req, rsp, self.spider)
|
||||
self.assertEqual(req, None)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookup
|
|||
ConnectionLost
|
||||
|
||||
from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware
|
||||
from scrapy.core.exceptions import HttpException
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.http import Request, Response
|
||||
|
||||
|
|
@ -17,20 +16,31 @@ class RetryTest(unittest.TestCase):
|
|||
self.mw = RetryMiddleware()
|
||||
self.mw.max_retry_times = 2
|
||||
|
||||
def test_process_exception_404(self):
|
||||
req404 = Request('http://www.scrapytest.org/404')
|
||||
exc404 = HttpException('404', None, Response('http://www.scrapytest.org/404', body=''))
|
||||
def test_404(self):
|
||||
req = Request('http://www.scrapytest.org/404')
|
||||
rsp = Response('http://www.scrapytest.org/404', body='', status=404)
|
||||
|
||||
# dont retry 404s
|
||||
req = self.mw.process_exception(req404, exc404, self.spider)
|
||||
self.assertTrue(req is None)
|
||||
assert self.mw.process_response(req, rsp, self.spider) is rsp
|
||||
|
||||
def test_process_exception_503(self):
|
||||
req503 = Request('http://www.scrapytest.org/503')
|
||||
exc503 = HttpException('503', None, Response('http://www.scrapytest.org/503', body=''))
|
||||
self._test_retry_exception(req503, exc503)
|
||||
def test_503(self):
|
||||
req = Request('http://www.scrapytest.org/503')
|
||||
rsp = Response('http://www.scrapytest.org/503', body='', status=503)
|
||||
|
||||
def test_process_exception_twistederrors(self):
|
||||
# first retry
|
||||
req = self.mw.process_response(req, rsp, self.spider)
|
||||
assert isinstance(req, Request)
|
||||
self.assertEqual(req.meta['retry_times'], 1)
|
||||
|
||||
# second retry
|
||||
req = self.mw.process_response(req, rsp, self.spider)
|
||||
assert isinstance(req, Request)
|
||||
self.assertEqual(req.meta['retry_times'], 2)
|
||||
|
||||
# discard it
|
||||
assert self.mw.process_response(req, rsp, self.spider) is rsp
|
||||
|
||||
def test_twistederrors(self):
|
||||
for exc in (ServerTimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, ConnectionLost):
|
||||
req = Request('http://www.scrapytest.org/%s' % exc.__name__)
|
||||
self._test_retry_exception(req, exc())
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ scrapy.http.Response objects
|
|||
"""
|
||||
|
||||
import re
|
||||
from twisted.web import http
|
||||
from scrapy.http.response import Response
|
||||
|
||||
def body_or_str(obj, unicode=True):
|
||||
|
|
@ -36,3 +37,14 @@ def get_meta_refresh(response):
|
|||
match = META_REFRESH_RE.search(response.body[0:4096])
|
||||
response.cache['meta_refresh_url'] = match.groups() if match else (None, None)
|
||||
return response.cache['meta_refresh_url']
|
||||
|
||||
def response_status_message(status):
|
||||
"""Return status code plus status text descriptive message
|
||||
|
||||
>>> response_status_message(200)
|
||||
200 OK
|
||||
|
||||
>>> response_status_message(404)
|
||||
404 Not Found
|
||||
"""
|
||||
return '%s %s' % (status, http.responses.get(int(status)))
|
||||
|
|
|
|||
Loading…
Reference in New Issue