diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 160bc7b7f..86ad73e58 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -556,8 +556,7 @@ The :class:`MetaRefreshMiddleware` can be configured through the following settings (see the settings documentation for more info): * :setting:`METAREFRESH_ENABLED` -* :setting:`REDIRECT_MAX_METAREFRESH_DELAY` -* :setting:`REDIRECT_MAX_TIMES` +* :setting:`METAREFRESH_MAXDELAY` This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect` and :reqmeta:`redirect_urls` request meta keys as described for :class:`RedirectMiddleware` diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index 69c772ea9..b31657afb 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -8,8 +8,10 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured class BaseRedirectMiddleware(object): + enabled_setting = 'REDIRECT_ENABLED' + def __init__(self, settings): - if not settings.getbool('REDIRECT_ENABLED'): + if not settings.getbool(self.enabled_setting): raise NotConfigured self.max_redirect_times = settings.getint('REDIRECT_MAX_TIMES') @@ -76,9 +78,12 @@ class RedirectMiddleware(BaseRedirectMiddleware): class MetaRefreshMiddleware(BaseRedirectMiddleware): + enabled_setting = 'METAREFRESH_ENABLED' + def __init__(self, settings): super(MetaRefreshMiddleware, self).__init__(settings) - self.max_metarefresh_delay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY') + self._maxdelay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY', + settings.getint('METAREFRESH_MAXDELAY')) def process_response(self, request, response, spider): if 'dont_redirect' in request.meta or request.method == 'HEAD' or \ @@ -87,7 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): if isinstance(response, HtmlResponse): interval, url = get_meta_refresh(response) - if url and interval < self.max_metarefresh_delay: + if url and interval < self._maxdelay: redirected = self._redirect_request_using_get(request, url) return self._redirect(redirected, request, spider, 'meta refresh') diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index b588ce7ee..f8a8e029d 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -175,13 +175,15 @@ MEMUSAGE_NOTIFY_MAIL = [] MEMUSAGE_REPORT = False MEMUSAGE_WARNING_MB = 0 +METAREFRESH_ENABLED = True +METAREFRESH_MAXDELAY = 100 + NEWSPIDER_MODULE = '' RANDOMIZE_DOWNLOAD_DELAY = True REDIRECT_ENABLED = True -REDIRECT_MAX_METAREFRESH_DELAY = 100 -REDIRECT_MAX_TIMES = 20 # uses Firefox default setting +REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True diff --git a/scrapy/settings/deprecated.py b/scrapy/settings/deprecated.py index 7bdd62b69..23528c156 100644 --- a/scrapy/settings/deprecated.py +++ b/scrapy/settings/deprecated.py @@ -12,6 +12,8 @@ DEPRECATED_SETTINGS = [ ('SELECTORS_BACKEND', 'use SCRAPY_SELECTORS_BACKEND environment variable instead'), ('AUTOTHROTTLE_MIN_DOWNLOAD_DELAY', 'use DOWNLOAD_DELAY instead'), ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'), + ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'), + ('REDIRECT_MAX_METAREFRESH_DELAY', 'use METAREFRESH_MAXDELAY instead'), ] def check_deprecated_settings(settings):