added REFERER_ENABLED setting, to control referer middleware

docs/topics/spider-middleware.rst

@@ -251,6 +251,20 @@ RefererMiddleware
 
    Populates Request referer field, based on the Response which originated it.
 
+RefererMiddleware settings
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. setting:: REFERER_ENABLED
+
+REFERER_ENABLED
+^^^^^^^^^^^^^^^
+
+.. versionadded:: 0.15
+
+Default: ``True``
+
+Whether to enable referer middleware will be enabled.
+
 UrlLengthMiddleware
 -------------------
 

scrapy/contrib/spidermiddleware/referer.py

@@ -4,8 +4,16 @@ originated it.
 """
 
 from scrapy.http import Request
+from scrapy.exceptions import NotConfigured
 
 class RefererMiddleware(object):
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        if not crawler.settings.getbool('REFERER_ENABLED'):
+            raise NotConfigured
+        return cls()
+
     def process_spider_output(self, response, result, spider):
         def _set_referer(r):
             if isinstance(r, Request):

scrapy/settings/default_settings.py

@@ -185,6 +185,8 @@ REDIRECT_MAX_METAREFRESH_DELAY = 100
 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
 REDIRECT_PRIORITY_ADJUST = +2
 
+REFERER_ENABLED = True
+
 RETRY_ENABLED = True
 RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
 RETRY_HTTP_CODES = [500, 503, 504, 400, 408]