From 537683f945906ff9865cf4e2704a432612f10eb4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 21 Feb 2017 16:47:57 +0100 Subject: [PATCH] Add autoclass directives to document built-in policies --- docs/topics/spider-middleware.rst | 70 +++++++++++++++++++---------- scrapy/spidermiddlewares/referer.py | 6 ++- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 12aa53012..349dbb3a1 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -341,43 +341,65 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` `Referrer Policy`_ to apply when populating Request "Referer" header. -This setting accepts: +.. note:: + You can also set the Referrer Policy per request, + using the special ``"referrer_policy"`` :ref:`Request.meta ` key, + with the same acceptable values as for the ``REFERER_POLICY`` setting. -- a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass, - either a custom one or one of the built-in ones - (see ``scrapy.spidermiddlewares.referer``), -- or one of the standard W3C-defined string values +Acceptable values for REFERER_POLICY +************************************ -======================================= ======================================================================== ======================================================= -String value Class name -======================================= ======================================================================== ======================================================= -`"no-referrer"`_ ``'scrapy.spidermiddlewares.referer.NoReferrerPolicy'`` -`"no-referrer-when-downgrade"`_ ``'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'`` the W3C-recommended default, used by major web browsers -`"same-origin"`_ ``'scrapy.spidermiddlewares.referer.SameOriginPolicy'`` -`"origin"`_ ``'scrapy.spidermiddlewares.referer.OriginPolicy'`` -`"strict-origin"`_ ``'scrapy.spidermiddlewares.referer.StrictOriginPolicy'`` -`"origin-when-cross-origin"`_ ``'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'`` -`"strict-origin-when-cross-origin"`_ ``'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'`` -`"unsafe-url"`_ ``'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'`` NOT recommended -``"scrapy-default"`` ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` Scrapy's default policy (see below) -======================================= ======================================================================== ======================================================= +- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` + subclass — a custom policy or one of the built-in ones (see classes below), +- or one of the standard W3C-defined string values, +- or the special ``"scrapy-default"``. -Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, -with the addition that "Referer" is not sent if the parent request was -using ``file://`` or ``s3://`` scheme. +======================================= ======================================================================== +String value Class name (as a string) +======================================= ======================================================================== +``"scrapy-default"`` (default) :class:`scrapy.spidermiddlewares.referer.DefaultReferrerPolicy` +`"no-referrer"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerPolicy` +`"no-referrer-when-downgrade"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy` +`"same-origin"`_ :class:`scrapy.spidermiddlewares.referer.SameOriginPolicy` +`"origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginPolicy` +`"strict-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginPolicy` +`"origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy` +`"strict-origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy` +`"unsafe-url"`_ :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy` +======================================= ======================================================================== +.. autoclass:: DefaultReferrerPolicy .. warning:: Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_, the W3C-recommended value for browsers — will send a non-empty "Referer" header from any ``http(s)://`` to any ``https://`` URL, even if the domain is different. + `"same-origin"`_ may be a better choice if you want to remove referrer information for cross-domain requests. +.. autoclass:: NoReferrerPolicy + +.. autoclass:: NoReferrerWhenDowngradePolicy .. note:: - You can also override the Referrer Policy per request, - using the special ``"referrer_policy"`` :ref:`Request.meta ` key, - with the same acceptable values as for the ``REFERER_POLICY`` setting. + "no-referrer-when-downgrade" policy is the W3C-recommended default, + and is used by major web browsers. + + However, it is NOT Scrapy's default referrer policy (see :class:`DefaultReferrerPolicy`). + +.. autoclass:: SameOriginPolicy + +.. autoclass:: OriginPolicy + +.. autoclass:: StrictOriginPolicy + +.. autoclass:: OriginWhenCrossOriginPolicy + +.. autoclass:: StrictOriginWhenCrossOriginPolicy + +.. autoclass:: UnsafeUrlPolicy +.. warning:: + "unsafe-url" policy is NOT recommended. .. _Referrer Policy: https://www.w3.org/TR/referrer-policy .. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 4f50db689..c015e13c8 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -240,7 +240,11 @@ class LegacyPolicy(ReferrerPolicy): class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): - + """ + A variant of "no-referrer-when-downgrade", + with the addition that "Referer" is not sent if the parent request was + using ``file://`` or ``s3://`` scheme. + """ NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') name = POLICY_SCRAPY_DEFAULT