diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a5931a3d5..15a134dd3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -234,6 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True +REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 44a599433..88041d8f0 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -8,6 +8,7 @@ from scrapy.http import Request from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_native_str from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import load_object LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) @@ -221,16 +222,27 @@ _policy_classes = {p.name: p for p in ( class RefererMiddleware(object): - def __init__(self, policy_class=DefaultReferrerPolicy): - self.default_policy = policy_class + def __init__(self, settings={}): + policy = settings.get('REFERER_POLICY') + if policy is not None: + try: + self.default_policy = load_object(policy) + except ValueError: + try: + self.default_policy = _policy_classes[policy] + except: + raise NotConfigured("Unknown referrer policy name %r" % policy) + else: + self.default_policy = DefaultReferrerPolicy @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured - return cls() + return cls(crawler.settings) def policy(self, response, request): + # policy set in request's meta dict takes precedence over default policy policy_name = request.meta.get('referrer_policy') if policy_name is None: policy_name = to_native_str(