update metarefresh settings

docs/topics/downloader-middleware.rst

@@ -556,8 +556,7 @@ The :class:`MetaRefreshMiddleware` can be configured through the following
 settings (see the settings documentation for more info):
 
 * :setting:`METAREFRESH_ENABLED`
-* :setting:`REDIRECT_MAX_METAREFRESH_DELAY`
-* :setting:`REDIRECT_MAX_TIMES`
+* :setting:`METAREFRESH_MAXDELAY`
 
 This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`
 and :reqmeta:`redirect_urls` request meta keys as described for :class:`RedirectMiddleware`

scrapy/contrib/downloadermiddleware/redirect.py

@@ -8,8 +8,10 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
 
 class BaseRedirectMiddleware(object):
 
+    enabled_setting = 'REDIRECT_ENABLED'
+
     def __init__(self, settings):
-        if not settings.getbool('REDIRECT_ENABLED'):
+        if not settings.getbool(self.enabled_setting):
             raise NotConfigured
 
         self.max_redirect_times = settings.getint('REDIRECT_MAX_TIMES')
@@ -76,9 +78,12 @@ class RedirectMiddleware(BaseRedirectMiddleware):
 
 class MetaRefreshMiddleware(BaseRedirectMiddleware):
 
+    enabled_setting = 'METAREFRESH_ENABLED'
+
     def __init__(self, settings):
         super(MetaRefreshMiddleware, self).__init__(settings)
-        self.max_metarefresh_delay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY')
+        self._maxdelay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY',
+                                         settings.getint('METAREFRESH_MAXDELAY'))
 
     def process_response(self, request, response, spider):
         if 'dont_redirect' in request.meta or request.method == 'HEAD' or \
@@ -87,7 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
 
         if isinstance(response, HtmlResponse):
             interval, url = get_meta_refresh(response)
-            if url and interval < self.max_metarefresh_delay:
+            if url and interval < self._maxdelay:
                 redirected = self._redirect_request_using_get(request, url)
                 return self._redirect(redirected, request, spider, 'meta refresh')
 

scrapy/settings/default_settings.py

@@ -175,13 +175,15 @@ MEMUSAGE_NOTIFY_MAIL = []
 MEMUSAGE_REPORT = False
 MEMUSAGE_WARNING_MB = 0
 
+METAREFRESH_ENABLED = True
+METAREFRESH_MAXDELAY = 100
+
 NEWSPIDER_MODULE = ''
 
 RANDOMIZE_DOWNLOAD_DELAY = True
 
 REDIRECT_ENABLED = True
-REDIRECT_MAX_METAREFRESH_DELAY = 100
-REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
+REDIRECT_MAX_TIMES = 20  # uses Firefox default setting
 REDIRECT_PRIORITY_ADJUST = +2
 
 REFERER_ENABLED = True

scrapy/settings/deprecated.py

@@ -12,6 +12,8 @@ DEPRECATED_SETTINGS = [
     ('SELECTORS_BACKEND', 'use SCRAPY_SELECTORS_BACKEND environment variable instead'),
     ('AUTOTHROTTLE_MIN_DOWNLOAD_DELAY', 'use DOWNLOAD_DELAY instead'),
     ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'),
+    ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'),
+    ('REDIRECT_MAX_METAREFRESH_DELAY', 'use METAREFRESH_MAXDELAY instead'),
 ]
 
 def check_deprecated_settings(settings):