From 8933e2f2beabb2c5c09e4edf2be5512e8f7bbe72 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Thu, 22 Mar 2012 16:35:14 -0300
Subject: [PATCH] added REFERER_ENABLED setting, to control referer middleware

---
 docs/topics/spider-middleware.rst          | 14 ++++++++++++++
 scrapy/contrib/spidermiddleware/referer.py |  8 ++++++++
 scrapy/settings/default_settings.py        |  2 ++
 3 files changed, 24 insertions(+)

diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 2ba0b1d05..69078df84 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -251,6 +251,20 @@ RefererMiddleware
 
    Populates Request referer field, based on the Response which originated it.
 
+RefererMiddleware settings
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. setting:: REFERER_ENABLED
+
+REFERER_ENABLED
+^^^^^^^^^^^^^^^
+
+.. versionadded:: 0.15
+
+Default: ``True``
+
+Whether to enable referer middleware will be enabled.
+
 UrlLengthMiddleware
 -------------------
 
diff --git a/scrapy/contrib/spidermiddleware/referer.py b/scrapy/contrib/spidermiddleware/referer.py
index e985e1129..6a8c46543 100644
--- a/scrapy/contrib/spidermiddleware/referer.py
+++ b/scrapy/contrib/spidermiddleware/referer.py
@@ -4,8 +4,16 @@ originated it.
 """
 
 from scrapy.http import Request
+from scrapy.exceptions import NotConfigured
 
 class RefererMiddleware(object):
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        if not crawler.settings.getbool('REFERER_ENABLED'):
+            raise NotConfigured
+        return cls()
+
     def process_spider_output(self, response, result, spider):
         def _set_referer(r):
             if isinstance(r, Request):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index eeff3f501..1cd59d9bd 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -185,6 +185,8 @@ REDIRECT_MAX_METAREFRESH_DELAY = 100
 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
 REDIRECT_PRIORITY_ADJUST = +2
 
+REFERER_ENABLED = True
+
 RETRY_ENABLED = True
 RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
 RETRY_HTTP_CODES = [500, 503, 504, 400, 408]