mirror of https://github.com/scrapy/scrapy.git
Merge pull request #2306 from redapple/referrer-policy
[MRG] Referrer policies in RefererMiddleware
This commit is contained in:
Mikhail Korobov
GitHub
commit
7e8453cf1e
|
|
@ -307,6 +307,7 @@ Those are:
|
|||
* :reqmeta:`proxy`
|
||||
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
|
||||
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
|
||||
* :reqmeta:`referrer_policy`
|
||||
|
||||
.. reqmeta:: bindaddress
|
||||
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ following methods:
|
|||
it has processed the response.
|
||||
|
||||
:meth:`process_spider_output` must return an iterable of
|
||||
:class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item`
|
||||
:class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item`
|
||||
objects.
|
||||
|
||||
:param response: the response which generated this output from the
|
||||
|
|
@ -328,6 +328,90 @@ Default: ``True``
|
|||
|
||||
Whether to enable referer middleware.
|
||||
|
||||
.. setting:: REFERRER_POLICY
|
||||
|
||||
REFERRER_POLICY
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
.. versionadded:: 1.4
|
||||
|
||||
Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
|
||||
|
||||
.. reqmeta:: referrer_policy
|
||||
|
||||
`Referrer Policy`_ to apply when populating Request "Referer" header.
|
||||
|
||||
.. note::
|
||||
You can also set the Referrer Policy per request,
|
||||
using the special ``"referrer_policy"`` :ref:`Request.meta <topics-request-meta>` key,
|
||||
with the same acceptable values as for the ``REFERRER_POLICY`` setting.
|
||||
|
||||
Acceptable values for REFERRER_POLICY
|
||||
*************************************
|
||||
|
||||
- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy``
|
||||
subclass — a custom policy or one of the built-in ones (see classes below),
|
||||
- or one of the standard W3C-defined string values,
|
||||
- or the special ``"scrapy-default"``.
|
||||
|
||||
======================================= ========================================================================
|
||||
String value Class name (as a string)
|
||||
======================================= ========================================================================
|
||||
``"scrapy-default"`` (default) :class:`scrapy.spidermiddlewares.referer.DefaultReferrerPolicy`
|
||||
`"no-referrer"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerPolicy`
|
||||
`"no-referrer-when-downgrade"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy`
|
||||
`"same-origin"`_ :class:`scrapy.spidermiddlewares.referer.SameOriginPolicy`
|
||||
`"origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginPolicy`
|
||||
`"strict-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginPolicy`
|
||||
`"origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy`
|
||||
`"strict-origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy`
|
||||
`"unsafe-url"`_ :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy`
|
||||
======================================= ========================================================================
|
||||
|
||||
.. autoclass:: DefaultReferrerPolicy
|
||||
.. warning::
|
||||
Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_,
|
||||
the W3C-recommended value for browsers — will send a non-empty
|
||||
"Referer" header from any ``http(s)://`` to any ``https://`` URL,
|
||||
even if the domain is different.
|
||||
|
||||
`"same-origin"`_ may be a better choice if you want to remove referrer
|
||||
information for cross-domain requests.
|
||||
|
||||
.. autoclass:: NoReferrerPolicy
|
||||
|
||||
.. autoclass:: NoReferrerWhenDowngradePolicy
|
||||
.. note::
|
||||
"no-referrer-when-downgrade" policy is the W3C-recommended default,
|
||||
and is used by major web browsers.
|
||||
|
||||
However, it is NOT Scrapy's default referrer policy (see :class:`DefaultReferrerPolicy`).
|
||||
|
||||
.. autoclass:: SameOriginPolicy
|
||||
|
||||
.. autoclass:: OriginPolicy
|
||||
|
||||
.. autoclass:: StrictOriginPolicy
|
||||
|
||||
.. autoclass:: OriginWhenCrossOriginPolicy
|
||||
|
||||
.. autoclass:: StrictOriginWhenCrossOriginPolicy
|
||||
|
||||
.. autoclass:: UnsafeUrlPolicy
|
||||
.. warning::
|
||||
"unsafe-url" policy is NOT recommended.
|
||||
|
||||
.. _Referrer Policy: https://www.w3.org/TR/referrer-policy
|
||||
.. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer
|
||||
.. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
|
||||
.. _"same-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin
|
||||
.. _"origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin
|
||||
.. _"strict-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin
|
||||
.. _"origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin
|
||||
.. _"strict-origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin
|
||||
.. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url
|
||||
|
||||
|
||||
UrlLengthMiddleware
|
||||
-------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -234,6 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
|
|||
REDIRECT_PRIORITY_ADJUST = +2
|
||||
|
||||
REFERER_ENABLED = True
|
||||
REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
|
||||
|
||||
RETRY_ENABLED = True
|
||||
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
|
||||
|
|
|
|||
|
|
@ -2,22 +2,355 @@
|
|||
RefererMiddleware: populates Request referer field, based on the Response which
|
||||
originated it.
|
||||
"""
|
||||
from six.moves.urllib.parse import urlparse
|
||||
import warnings
|
||||
|
||||
from scrapy.http import Request
|
||||
from w3lib.url import safe_url_string
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy import signals
|
||||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.url import strip_url
|
||||
|
||||
|
||||
LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)
|
||||
|
||||
POLICY_NO_REFERRER = "no-referrer"
|
||||
POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade"
|
||||
POLICY_SAME_ORIGIN = "same-origin"
|
||||
POLICY_ORIGIN = "origin"
|
||||
POLICY_STRICT_ORIGIN = "strict-origin"
|
||||
POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin"
|
||||
POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN = "strict-origin-when-cross-origin"
|
||||
POLICY_UNSAFE_URL = "unsafe-url"
|
||||
POLICY_SCRAPY_DEFAULT = "scrapy-default"
|
||||
|
||||
|
||||
class ReferrerPolicy(object):
|
||||
|
||||
NOREFERRER_SCHEMES = LOCAL_SCHEMES
|
||||
|
||||
def referrer(self, response_url, request_url):
|
||||
raise NotImplementedError()
|
||||
|
||||
def stripped_referrer(self, url):
|
||||
if urlparse(url).scheme not in self.NOREFERRER_SCHEMES:
|
||||
return self.strip_url(url)
|
||||
|
||||
def origin_referrer(self, url):
|
||||
if urlparse(url).scheme not in self.NOREFERRER_SCHEMES:
|
||||
return self.origin(url)
|
||||
|
||||
def strip_url(self, url, origin_only=False):
|
||||
"""
|
||||
https://www.w3.org/TR/referrer-policy/#strip-url
|
||||
|
||||
If url is null, return no referrer.
|
||||
If url's scheme is a local scheme, then return no referrer.
|
||||
Set url's username to the empty string.
|
||||
Set url's password to null.
|
||||
Set url's fragment to null.
|
||||
If the origin-only flag is true, then:
|
||||
Set url's path to null.
|
||||
Set url's query to null.
|
||||
Return url.
|
||||
"""
|
||||
if not url:
|
||||
return None
|
||||
return strip_url(url,
|
||||
strip_credentials=True,
|
||||
strip_fragment=True,
|
||||
strip_default_port=True,
|
||||
origin_only=origin_only)
|
||||
|
||||
def origin(self, url):
|
||||
"""Return serialized origin (scheme, host, path) for a request or response URL."""
|
||||
return self.strip_url(url, origin_only=True)
|
||||
|
||||
def potentially_trustworthy(self, url):
|
||||
# Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy
|
||||
parsed_url = urlparse(url)
|
||||
if parsed_url.scheme in ('data',):
|
||||
return False
|
||||
return self.tls_protected(url)
|
||||
|
||||
def tls_protected(self, url):
|
||||
return urlparse(url).scheme in ('https', 'ftps')
|
||||
|
||||
|
||||
class NoReferrerPolicy(ReferrerPolicy):
|
||||
"""
|
||||
https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer
|
||||
|
||||
The simplest policy is "no-referrer", which specifies that no referrer information
|
||||
is to be sent along with requests made from a particular request client to any origin.
|
||||
The header will be omitted entirely.
|
||||
"""
|
||||
name = POLICY_NO_REFERRER
|
||||
|
||||
def referrer(self, response_url, request_url):
|
||||
return None
|
||||
|
||||
|
||||
class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
|
||||
"""
|
||||
https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
|
||||
|
||||
The "no-referrer-when-downgrade" policy sends a full URL along with requests
|
||||
from a TLS-protected environment settings object to a potentially trustworthy URL,
|
||||
and requests from clients which are not TLS-protected to any origin.
|
||||
|
||||
Requests from TLS-protected clients to non-potentially trustworthy URLs,
|
||||
on the other hand, will contain no referrer information.
|
||||
A Referer HTTP header will not be sent.
|
||||
|
||||
This is a user agent's default behavior, if no policy is otherwise specified.
|
||||
"""
|
||||
name = POLICY_NO_REFERRER_WHEN_DOWNGRADE
|
||||
|
||||
def referrer(self, response_url, request_url):
|
||||
if not self.tls_protected(response_url) or self.tls_protected(request_url):
|
||||
return self.stripped_referrer(response_url)
|
||||
|
||||
|
||||
class SameOriginPolicy(ReferrerPolicy):
|
||||
"""
|
||||
https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin
|
||||
|
||||
The "same-origin" policy specifies that a full URL, stripped for use as a referrer,
|
||||
is sent as referrer information when making same-origin requests from a particular request client.
|
||||
|
||||
Cross-origin requests, on the other hand, will contain no referrer information.
|
||||
A Referer HTTP header will not be sent.
|
||||
"""
|
||||
name = POLICY_SAME_ORIGIN
|
||||
|
||||
def referrer(self, response_url, request_url):
|
||||
if self.origin(response_url) == self.origin(request_url):
|
||||
return self.stripped_referrer(response_url)
|
||||
|
||||
|
||||
class OriginPolicy(ReferrerPolicy):
|
||||
"""
|
||||
https://www.w3.org/TR/referrer-policy/#referrer-policy-origin
|
||||
|
||||
The "origin" policy specifies that only the ASCII serialization
|
||||
of the origin of the request client is sent as referrer information
|
||||
when making both same-origin requests and cross-origin requests
|
||||
from a particular request client.
|
||||
"""
|
||||
name = POLICY_ORIGIN
|
||||
|
||||
def referrer(self, response_url, request_url):
|
||||
return self.origin_referrer(response_url)
|
||||
|
||||
|
||||
class StrictOriginPolicy(ReferrerPolicy):
|
||||
"""
|
||||
https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin
|
||||
|
||||
The "strict-origin" policy sends the ASCII serialization
|
||||
of the origin of the request client when making requests:
|
||||
- from a TLS-protected environment settings object to a potentially trustworthy URL, and
|
||||
- from non-TLS-protected environment settings objects to any origin.
|
||||
|
||||
Requests from TLS-protected request clients to non- potentially trustworthy URLs,
|
||||
on the other hand, will contain no referrer information.
|
||||
A Referer HTTP header will not be sent.
|
||||
"""
|
||||
name = POLICY_STRICT_ORIGIN
|
||||
|
||||
def referrer(self, response_url, request_url):
|
||||
if ((self.tls_protected(response_url) and
|
||||
self.potentially_trustworthy(request_url))
|
||||
or not self.tls_protected(response_url)):
|
||||
return self.origin_referrer(response_url)
|
||||
|
||||
|
||||
class OriginWhenCrossOriginPolicy(ReferrerPolicy):
|
||||
"""
|
||||
https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin
|
||||
|
||||
The "origin-when-cross-origin" policy specifies that a full URL,
|
||||
stripped for use as a referrer, is sent as referrer information
|
||||
when making same-origin requests from a particular request client,
|
||||
and only the ASCII serialization of the origin of the request client
|
||||
is sent as referrer information when making cross-origin requests
|
||||
from a particular request client.
|
||||
"""
|
||||
name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
|
||||
|
||||
def referrer(self, response_url, request_url):
|
||||
origin = self.origin(response_url)
|
||||
if origin == self.origin(request_url):
|
||||
return self.stripped_referrer(response_url)
|
||||
else:
|
||||
return origin
|
||||
|
||||
|
||||
class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
|
||||
"""
|
||||
https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin
|
||||
|
||||
The "strict-origin-when-cross-origin" policy specifies that a full URL,
|
||||
stripped for use as a referrer, is sent as referrer information
|
||||
when making same-origin requests from a particular request client,
|
||||
and only the ASCII serialization of the origin of the request client
|
||||
when making cross-origin requests:
|
||||
|
||||
- from a TLS-protected environment settings object to a potentially trustworthy URL, and
|
||||
- from non-TLS-protected environment settings objects to any origin.
|
||||
|
||||
Requests from TLS-protected clients to non- potentially trustworthy URLs,
|
||||
on the other hand, will contain no referrer information.
|
||||
A Referer HTTP header will not be sent.
|
||||
"""
|
||||
name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
|
||||
|
||||
def referrer(self, response_url, request_url):
|
||||
origin = self.origin(response_url)
|
||||
if origin == self.origin(request_url):
|
||||
return self.stripped_referrer(response_url)
|
||||
elif ((self.tls_protected(response_url) and
|
||||
self.potentially_trustworthy(request_url))
|
||||
or not self.tls_protected(response_url)):
|
||||
return self.origin_referrer(response_url)
|
||||
|
||||
|
||||
class UnsafeUrlPolicy(ReferrerPolicy):
|
||||
"""
|
||||
https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url
|
||||
|
||||
The "unsafe-url" policy specifies that a full URL, stripped for use as a referrer,
|
||||
is sent along with both cross-origin requests
|
||||
and same-origin requests made from a particular request client.
|
||||
|
||||
Note: The policy's name doesn't lie; it is unsafe.
|
||||
This policy will leak origins and paths from TLS-protected resources
|
||||
to insecure origins.
|
||||
Carefully consider the impact of setting such a policy for potentially sensitive documents.
|
||||
"""
|
||||
name = POLICY_UNSAFE_URL
|
||||
|
||||
def referrer(self, response_url, request_url):
|
||||
return self.stripped_referrer(response_url)
|
||||
|
||||
|
||||
class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
|
||||
"""
|
||||
A variant of "no-referrer-when-downgrade",
|
||||
with the addition that "Referer" is not sent if the parent request was
|
||||
using ``file://`` or ``s3://`` scheme.
|
||||
"""
|
||||
NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')
|
||||
name = POLICY_SCRAPY_DEFAULT
|
||||
|
||||
|
||||
_policy_classes = {p.name: p for p in (
|
||||
NoReferrerPolicy,
|
||||
NoReferrerWhenDowngradePolicy,
|
||||
SameOriginPolicy,
|
||||
OriginPolicy,
|
||||
StrictOriginPolicy,
|
||||
OriginWhenCrossOriginPolicy,
|
||||
StrictOriginWhenCrossOriginPolicy,
|
||||
UnsafeUrlPolicy,
|
||||
DefaultReferrerPolicy,
|
||||
)}
|
||||
|
||||
|
||||
def _load_policy_class(policy, warning_only=False):
|
||||
"""
|
||||
Expect a string for the path to the policy class,
|
||||
otherwise try to interpret the string as a standard value
|
||||
from https://www.w3.org/TR/referrer-policy/#referrer-policies
|
||||
"""
|
||||
try:
|
||||
return load_object(policy)
|
||||
except ValueError:
|
||||
try:
|
||||
return _policy_classes[policy.lower()]
|
||||
except KeyError:
|
||||
msg = "Could not load referrer policy %r" % policy
|
||||
if not warning_only:
|
||||
raise RuntimeError(msg)
|
||||
else:
|
||||
warnings.warn(msg, RuntimeWarning)
|
||||
return None
|
||||
|
||||
|
||||
class RefererMiddleware(object):
|
||||
|
||||
def __init__(self, settings=None):
|
||||
self.default_policy = DefaultReferrerPolicy
|
||||
if settings is not None:
|
||||
self.default_policy = _load_policy_class(
|
||||
settings.get('REFERRER_POLICY'))
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
if not crawler.settings.getbool('REFERER_ENABLED'):
|
||||
raise NotConfigured
|
||||
return cls()
|
||||
mw = cls(crawler.settings)
|
||||
|
||||
# Note: this hook is a bit of a hack to intercept redirections
|
||||
crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled)
|
||||
|
||||
return mw
|
||||
|
||||
def policy(self, resp_or_url, request):
|
||||
"""
|
||||
Determine Referrer-Policy to use from a parent Response (or URL),
|
||||
and a Request to be sent.
|
||||
|
||||
- if a valid policy is set in Request meta, it is used.
|
||||
- if the policy is set in meta but is wrong (e.g. a typo error),
|
||||
the policy from settings is used
|
||||
- if the policy is not set in Request meta,
|
||||
but there is a Referrer-policy header in the parent response,
|
||||
it is used if valid
|
||||
- otherwise, the policy from settings is used.
|
||||
"""
|
||||
policy_name = request.meta.get('referrer_policy')
|
||||
if policy_name is None:
|
||||
if isinstance(resp_or_url, Response):
|
||||
policy_name = to_native_str(
|
||||
resp_or_url.headers.get('Referrer-Policy', '').decode('latin1'))
|
||||
if policy_name is None:
|
||||
return self.default_policy()
|
||||
|
||||
cls = _load_policy_class(policy_name, warning_only=True)
|
||||
return cls() if cls else self.default_policy()
|
||||
|
||||
def process_spider_output(self, response, result, spider):
|
||||
def _set_referer(r):
|
||||
if isinstance(r, Request):
|
||||
r.headers.setdefault('Referer', response.url)
|
||||
referrer = self.policy(response, r).referrer(response.url, r.url)
|
||||
if referrer is not None:
|
||||
r.headers.setdefault('Referer', referrer)
|
||||
return r
|
||||
return (_set_referer(r) for r in result or ())
|
||||
|
||||
def request_scheduled(self, request, spider):
|
||||
# check redirected request to patch "Referer" header if necessary
|
||||
redirected_urls = request.meta.get('redirect_urls', [])
|
||||
if redirected_urls:
|
||||
request_referrer = request.headers.get('Referer')
|
||||
# we don't patch the referrer value if there is none
|
||||
if request_referrer is not None:
|
||||
# the request's referrer header value acts as a surrogate
|
||||
# for the parent response URL
|
||||
#
|
||||
# Note: if the 3xx response contained a Referrer-Policy header,
|
||||
# the information is not available using this hook
|
||||
parent_url = safe_url_string(request_referrer)
|
||||
policy_referrer = self.policy(parent_url, request).referrer(
|
||||
parent_url, request.url)
|
||||
if policy_referrer != request_referrer:
|
||||
if policy_referrer is None:
|
||||
request.headers.pop('Referer')
|
||||
else:
|
||||
request.headers['Referer'] = policy_referrer
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead.
|
|||
"""
|
||||
import posixpath
|
||||
import re
|
||||
from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse)
|
||||
from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse)
|
||||
|
||||
# scrapy.utils.url was moved to w3lib.url and import * ensures this
|
||||
# move doesn't break old code
|
||||
|
|
@ -103,3 +103,34 @@ def guess_scheme(url):
|
|||
return any_to_uri(url)
|
||||
else:
|
||||
return add_http_if_no_scheme(url)
|
||||
|
||||
|
||||
def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True):
|
||||
|
||||
"""Strip URL string from some of its components:
|
||||
|
||||
- `strip_credentials` removes "user:password@"
|
||||
- `strip_default_port` removes ":80" (resp. ":443", ":21")
|
||||
from http:// (resp. https://, ftp://) URLs
|
||||
- `origin_only` replaces path component with "/", also dropping
|
||||
query and fragment components ; it also strips credentials
|
||||
- `strip_fragment` drops any #fragment component
|
||||
"""
|
||||
|
||||
parsed_url = urlparse(url)
|
||||
netloc = parsed_url.netloc
|
||||
if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password):
|
||||
netloc = netloc.split('@')[-1]
|
||||
if strip_default_port and parsed_url.port:
|
||||
if (parsed_url.scheme, parsed_url.port) in (('http', 80),
|
||||
('https', 443),
|
||||
('ftp', 21)):
|
||||
netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '')
|
||||
return urlunparse((
|
||||
parsed_url.scheme,
|
||||
netloc,
|
||||
'/' if origin_only else parsed_url.path,
|
||||
'' if origin_only else parsed_url.params,
|
||||
'' if origin_only else parsed_url.query,
|
||||
'' if strip_fragment else parsed_url.fragment
|
||||
))
|
||||
|
|
|
|||
|
|
@ -1,21 +1,867 @@
|
|||
from six.moves.urllib.parse import urlparse
|
||||
from unittest import TestCase
|
||||
import warnings
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response, Request
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.spidermiddlewares.referer import RefererMiddleware
|
||||
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
|
||||
from scrapy.spidermiddlewares.referer import RefererMiddleware, \
|
||||
POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \
|
||||
POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \
|
||||
POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \
|
||||
POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, \
|
||||
DefaultReferrerPolicy, \
|
||||
NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \
|
||||
OriginWhenCrossOriginPolicy, OriginPolicy, \
|
||||
StrictOriginWhenCrossOriginPolicy, StrictOriginPolicy, \
|
||||
SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy
|
||||
|
||||
|
||||
class TestRefererMiddleware(TestCase):
|
||||
|
||||
req_meta = {}
|
||||
resp_headers = {}
|
||||
settings = {}
|
||||
scenarii = [
|
||||
('http://scrapytest.org', 'http://scrapytest.org/', b'http://scrapytest.org'),
|
||||
]
|
||||
|
||||
def setUp(self):
|
||||
self.spider = Spider('foo')
|
||||
self.mw = RefererMiddleware()
|
||||
settings = Settings(self.settings)
|
||||
self.mw = RefererMiddleware(settings)
|
||||
|
||||
def test_process_spider_output(self):
|
||||
res = Response('http://scrapytest.org')
|
||||
reqs = [Request('http://scrapytest.org/')]
|
||||
def get_request(self, target):
|
||||
return Request(target, meta=self.req_meta)
|
||||
|
||||
out = list(self.mw.process_spider_output(res, reqs, self.spider))
|
||||
self.assertEquals(out[0].headers.get('Referer'),
|
||||
b'http://scrapytest.org')
|
||||
def get_response(self, origin):
|
||||
return Response(origin, headers=self.resp_headers)
|
||||
|
||||
def test(self):
|
||||
|
||||
for origin, target, referrer in self.scenarii:
|
||||
response = self.get_response(origin)
|
||||
request = self.get_request(target)
|
||||
out = list(self.mw.process_spider_output(response, [request], self.spider))
|
||||
self.assertEquals(out[0].headers.get('Referer'), referrer)
|
||||
|
||||
|
||||
class MixinDefault(object):
|
||||
"""
|
||||
Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
|
||||
|
||||
with some additional filtering of s3://
|
||||
"""
|
||||
scenarii = [
|
||||
('https://example.com/', 'https://scrapy.org/', b'https://example.com/'),
|
||||
('http://example.com/', 'http://scrapy.org/', b'http://example.com/'),
|
||||
('http://example.com/', 'https://scrapy.org/', b'http://example.com/'),
|
||||
('https://example.com/', 'http://scrapy.org/', None),
|
||||
|
||||
# no credentials leak
|
||||
('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'),
|
||||
|
||||
# no referrer leak for local schemes
|
||||
('file:///home/path/to/somefile.html', 'https://scrapy.org/', None),
|
||||
('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),
|
||||
|
||||
# no referrer leak for s3 origins
|
||||
('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None),
|
||||
('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None),
|
||||
]
|
||||
|
||||
|
||||
class MixinNoReferrer(object):
|
||||
scenarii = [
|
||||
('https://example.com/page.html', 'https://example.com/', None),
|
||||
('http://www.example.com/', 'https://scrapy.org/', None),
|
||||
('http://www.example.com/', 'http://scrapy.org/', None),
|
||||
('https://www.example.com/', 'http://scrapy.org/', None),
|
||||
('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),
|
||||
]
|
||||
|
||||
|
||||
class MixinNoReferrerWhenDowngrade(object):
|
||||
scenarii = [
|
||||
# TLS to TLS: send non-empty referrer
|
||||
('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'),
|
||||
('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),
|
||||
('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),
|
||||
('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'),
|
||||
('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'),
|
||||
|
||||
# TLS to non-TLS: do not send referrer
|
||||
('https://example.com/page.html', 'http://not.example.com/', None),
|
||||
('https://example.com/page.html', 'http://scrapy.org/', None),
|
||||
('ftps://example.com/urls.zip', 'http://scrapy.org/', None),
|
||||
|
||||
# non-TLS to TLS or non-TLS: send referrer
|
||||
('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
|
||||
('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'),
|
||||
('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'),
|
||||
('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'),
|
||||
('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'),
|
||||
('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'),
|
||||
('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'),
|
||||
|
||||
# test for user/password stripping
|
||||
('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
|
||||
]
|
||||
|
||||
|
||||
class MixinSameOrigin(object):
|
||||
scenarii = [
|
||||
# Same origin (protocol, host, port): send referrer
|
||||
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
|
||||
|
||||
# Different host: do NOT send referrer
|
||||
('https://example.com/page.html', 'https://not.example.com/otherpage.html', None),
|
||||
('http://example.com/page.html', 'http://not.example.com/otherpage.html', None),
|
||||
('http://example.com/page.html', 'http://www.example.com/otherpage.html', None),
|
||||
|
||||
# Different port: do NOT send referrer
|
||||
('https://example.com:444/page.html', 'https://example.com/not-page.html', None),
|
||||
('http://example.com:81/page.html', 'http://example.com/not-page.html', None),
|
||||
('http://example.com/page.html', 'http://example.com:81/not-page.html', None),
|
||||
|
||||
# Different protocols: do NOT send refferer
|
||||
('https://example.com/page.html', 'http://example.com/not-page.html', None),
|
||||
('https://example.com/page.html', 'http://not.example.com/', None),
|
||||
('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
|
||||
('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None),
|
||||
('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
|
||||
|
||||
# test for user/password stripping
|
||||
('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None),
|
||||
]
|
||||
|
||||
|
||||
class MixinOrigin(object):
|
||||
scenarii = [
|
||||
# TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)
|
||||
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),
|
||||
('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
|
||||
('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
|
||||
('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),
|
||||
|
||||
# test for user/password stripping
|
||||
('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
|
||||
]
|
||||
|
||||
|
||||
class MixinStrictOrigin(object):
|
||||
scenarii = [
|
||||
# TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades
|
||||
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),
|
||||
('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
|
||||
('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),
|
||||
|
||||
# downgrade: send nothing
|
||||
('https://example.com/page.html', 'http://scrapy.org', None),
|
||||
|
||||
# upgrade: send origin
|
||||
('http://example.com/page.html', 'https://scrapy.org', b'http://example.com/'),
|
||||
|
||||
# test for user/password stripping
|
||||
('https://user:password@example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
|
||||
('https://user:password@example.com/page.html', 'http://scrapy.org', None),
|
||||
]
|
||||
|
||||
|
||||
class MixinOriginWhenCrossOrigin(object):
|
||||
scenarii = [
|
||||
# Same origin (protocol, host, port): send referrer
|
||||
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
|
||||
|
||||
# Different host: send origin as referrer
|
||||
('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
|
||||
('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),
|
||||
('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),
|
||||
# exact match required
|
||||
('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),
|
||||
|
||||
# Different port: send origin as referrer
|
||||
('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),
|
||||
('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),
|
||||
|
||||
# Different protocols: send origin as referrer
|
||||
('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'),
|
||||
('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'),
|
||||
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
|
||||
('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),
|
||||
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
|
||||
|
||||
# test for user/password stripping
|
||||
('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),
|
||||
# TLS to non-TLS downgrade: send origin
|
||||
('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'),
|
||||
]
|
||||
|
||||
|
||||
class MixinStrictOriginWhenCrossOrigin(object):
|
||||
scenarii = [
|
||||
# Same origin (protocol, host, port): send referrer
|
||||
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
|
||||
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
|
||||
('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
|
||||
|
||||
# Different host: send origin as referrer
|
||||
('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
|
||||
('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),
|
||||
('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),
|
||||
# exact match required
|
||||
('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),
|
||||
|
||||
# Different port: send origin as referrer
|
||||
('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),
|
||||
('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),
|
||||
|
||||
# downgrade
|
||||
('https://example4.com/page.html', 'http://example4.com/not-page.html', None),
|
||||
('https://example4.com/page.html', 'http://not.example4.com/', None),
|
||||
|
||||
# non-TLS to non-TLS
|
||||
('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),
|
||||
|
||||
# upgrade
|
||||
('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'),
|
||||
('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'),
|
||||
|
||||
# Different protocols: send origin as referrer
|
||||
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
|
||||
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
|
||||
|
||||
# test for user/password stripping
|
||||
('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),
|
||||
|
||||
# TLS to non-TLS downgrade: send nothing
|
||||
('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None),
|
||||
]
|
||||
|
||||
|
||||
class MixinUnsafeUrl(object):
|
||||
scenarii = [
|
||||
# TLS to TLS: send referrer
|
||||
('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'),
|
||||
('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'),
|
||||
('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),
|
||||
('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),
|
||||
('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'),
|
||||
('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'),
|
||||
|
||||
# TLS to non-TLS: send referrer (yes, it's unsafe)
|
||||
('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'),
|
||||
('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'),
|
||||
('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'),
|
||||
|
||||
# non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)
|
||||
('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'),
|
||||
('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'),
|
||||
('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'),
|
||||
('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'),
|
||||
('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'),
|
||||
('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'),
|
||||
('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'),
|
||||
('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'),
|
||||
|
||||
# test for user/password stripping
|
||||
('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'),
|
||||
('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'),
|
||||
]
|
||||
|
||||
|
||||
class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware):
|
||||
pass
|
||||
|
||||
|
||||
# --- Tests using settings to set policy using class path
|
||||
class TestSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}
|
||||
|
||||
|
||||
class TestSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
|
||||
|
||||
|
||||
class TestSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
|
||||
|
||||
|
||||
class TestSettingsOrigin(MixinOrigin, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}
|
||||
|
||||
|
||||
class TestSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}
|
||||
|
||||
|
||||
class TestSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
|
||||
|
||||
|
||||
class TestSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}
|
||||
|
||||
|
||||
class TestSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
|
||||
|
||||
|
||||
class CustomPythonOrgPolicy(ReferrerPolicy):
|
||||
"""
|
||||
A dummy policy that returns referrer as http(s)://python.org
|
||||
depending on the scheme of the target URL.
|
||||
"""
|
||||
def referrer(self, response, request):
|
||||
scheme = urlparse(request).scheme
|
||||
if scheme == 'https':
|
||||
return b'https://python.org/'
|
||||
elif scheme == 'http':
|
||||
return b'http://python.org/'
|
||||
|
||||
|
||||
class TestSettingsCustomPolicy(TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
|
||||
scenarii = [
|
||||
('https://example.com/', 'https://scrapy.org/', b'https://python.org/'),
|
||||
('http://example.com/', 'http://scrapy.org/', b'http://python.org/'),
|
||||
('http://example.com/', 'https://scrapy.org/', b'https://python.org/'),
|
||||
('https://example.com/', 'http://scrapy.org/', b'http://python.org/'),
|
||||
('file:///home/path/to/somefile.html', 'https://scrapy.org/', b'https://python.org/'),
|
||||
('file:///home/path/to/somefile.html', 'http://scrapy.org/', b'http://python.org/'),
|
||||
|
||||
]
|
||||
|
||||
# --- Tests using Request meta dict to set policy
|
||||
class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware):
|
||||
req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}
|
||||
|
||||
|
||||
class TestRequestMetaNoReferrer(MixinNoReferrer, TestRefererMiddleware):
|
||||
req_meta = {'referrer_policy': POLICY_NO_REFERRER}
|
||||
|
||||
|
||||
class TestRequestMetaNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
|
||||
req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}
|
||||
|
||||
|
||||
class TestRequestMetaSameOrigin(MixinSameOrigin, TestRefererMiddleware):
|
||||
req_meta = {'referrer_policy': POLICY_SAME_ORIGIN}
|
||||
|
||||
|
||||
class TestRequestMetaOrigin(MixinOrigin, TestRefererMiddleware):
|
||||
req_meta = {'referrer_policy': POLICY_ORIGIN}
|
||||
|
||||
|
||||
class TestRequestMetaSrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
|
||||
req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN}
|
||||
|
||||
|
||||
class TestRequestMetaOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
|
||||
req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
|
||||
|
||||
|
||||
class TestRequestMetaStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
|
||||
req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
|
||||
|
||||
|
||||
class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
|
||||
req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
|
||||
|
||||
|
||||
class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
|
||||
req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
|
||||
|
||||
|
||||
class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
|
||||
req_meta = {'referrer_policy': POLICY_NO_REFERRER}
|
||||
|
||||
|
||||
class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
|
||||
req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
|
||||
|
||||
|
||||
class TestRequestMetaSettingFallback(TestCase):
|
||||
|
||||
params = [
|
||||
(
|
||||
# When an unknown policy is referenced in Request.meta
|
||||
# (here, a typo error),
|
||||
# the policy defined in settings takes precedence
|
||||
{'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'},
|
||||
{},
|
||||
{'referrer_policy': 'ssscrapy-default'},
|
||||
OriginWhenCrossOriginPolicy,
|
||||
True
|
||||
),
|
||||
(
|
||||
# same as above but with string value for settings policy
|
||||
{'REFERRER_POLICY': 'origin-when-cross-origin'},
|
||||
{},
|
||||
{'referrer_policy': 'ssscrapy-default'},
|
||||
OriginWhenCrossOriginPolicy,
|
||||
True
|
||||
),
|
||||
(
|
||||
# request meta references a wrong policy but it is set,
|
||||
# so the Referrer-Policy header in response is not used,
|
||||
# and the settings' policy is applied
|
||||
{'REFERRER_POLICY': 'origin-when-cross-origin'},
|
||||
{'Referrer-Policy': 'unsafe-url'},
|
||||
{'referrer_policy': 'ssscrapy-default'},
|
||||
OriginWhenCrossOriginPolicy,
|
||||
True
|
||||
),
|
||||
(
|
||||
# here, request meta does not set the policy
|
||||
# so response headers take precedence
|
||||
{'REFERRER_POLICY': 'origin-when-cross-origin'},
|
||||
{'Referrer-Policy': 'unsafe-url'},
|
||||
{},
|
||||
UnsafeUrlPolicy,
|
||||
False
|
||||
),
|
||||
(
|
||||
# here, request meta does not set the policy,
|
||||
# but response headers also use an unknown policy,
|
||||
# so the settings' policy is used
|
||||
{'REFERRER_POLICY': 'origin-when-cross-origin'},
|
||||
{'Referrer-Policy': 'unknown'},
|
||||
{},
|
||||
OriginWhenCrossOriginPolicy,
|
||||
True
|
||||
)
|
||||
]
|
||||
|
||||
def test(self):
|
||||
|
||||
origin = 'http://www.scrapy.org'
|
||||
target = 'http://www.example.com'
|
||||
|
||||
for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]:
|
||||
spider = Spider('foo')
|
||||
mw = RefererMiddleware(Settings(settings))
|
||||
|
||||
response = Response(origin, headers=response_headers)
|
||||
request = Request(target, meta=request_meta)
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
policy = mw.policy(response, request)
|
||||
self.assertIsInstance(policy, policy_class)
|
||||
|
||||
if check_warning:
|
||||
self.assertEqual(len(w), 1)
|
||||
self.assertEqual(w[0].category, RuntimeWarning, w[0].message)
|
||||
|
||||
|
||||
class TestSettingsPolicyByName(TestCase):
|
||||
|
||||
def test_valid_name(self):
|
||||
for s, p in [
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
settings = Settings({'REFERRER_POLICY': s})
|
||||
mw = RefererMiddleware(settings)
|
||||
self.assertEquals(mw.default_policy, p)
|
||||
|
||||
def test_valid_name_casevariants(self):
|
||||
for s, p in [
|
||||
(POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy),
|
||||
(POLICY_NO_REFERRER, NoReferrerPolicy),
|
||||
(POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy),
|
||||
(POLICY_SAME_ORIGIN, SameOriginPolicy),
|
||||
(POLICY_ORIGIN, OriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN, StrictOriginPolicy),
|
||||
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
|
||||
(POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy),
|
||||
(POLICY_UNSAFE_URL, UnsafeUrlPolicy),
|
||||
]:
|
||||
settings = Settings({'REFERRER_POLICY': s.upper()})
|
||||
mw = RefererMiddleware(settings)
|
||||
self.assertEquals(mw.default_policy, p)
|
||||
|
||||
def test_invalid_name(self):
|
||||
settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'})
|
||||
with self.assertRaises(RuntimeError):
|
||||
mw = RefererMiddleware(settings)
|
||||
|
||||
|
||||
class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
|
||||
resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}
|
||||
|
||||
class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
|
||||
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}
|
||||
|
||||
class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
|
||||
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
|
||||
|
||||
|
||||
class TestReferrerOnRedirect(TestRefererMiddleware):
|
||||
|
||||
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
|
||||
scenarii = [
|
||||
( 'http://scrapytest.org/1', # parent
|
||||
'http://scrapytest.org/2', # target
|
||||
(
|
||||
# redirections: code, URL
|
||||
(301, 'http://scrapytest.org/3'),
|
||||
(301, 'http://scrapytest.org/4'),
|
||||
),
|
||||
b'http://scrapytest.org/1', # expected initial referer
|
||||
b'http://scrapytest.org/1', # expected referer for the redirection request
|
||||
),
|
||||
( 'https://scrapytest.org/1',
|
||||
'https://scrapytest.org/2',
|
||||
(
|
||||
# redirecting to non-secure URL
|
||||
(301, 'http://scrapytest.org/3'),
|
||||
),
|
||||
b'https://scrapytest.org/1',
|
||||
b'https://scrapytest.org/1',
|
||||
),
|
||||
( 'https://scrapytest.org/1',
|
||||
'https://scrapytest.com/2',
|
||||
(
|
||||
# redirecting to non-secure URL: different origin
|
||||
(301, 'http://scrapytest.com/3'),
|
||||
),
|
||||
b'https://scrapytest.org/1',
|
||||
b'https://scrapytest.org/1',
|
||||
),
|
||||
]
|
||||
|
||||
def setUp(self):
|
||||
self.spider = Spider('foo')
|
||||
settings = Settings(self.settings)
|
||||
self.referrermw = RefererMiddleware(settings)
|
||||
self.redirectmw = RedirectMiddleware(settings)
|
||||
|
||||
def test(self):
|
||||
|
||||
for parent, target, redirections, init_referrer, final_referrer in self.scenarii:
|
||||
response = self.get_response(parent)
|
||||
request = self.get_request(target)
|
||||
|
||||
out = list(self.referrermw.process_spider_output(response, [request], self.spider))
|
||||
self.assertEquals(out[0].headers.get('Referer'), init_referrer)
|
||||
|
||||
for status, url in redirections:
|
||||
response = Response(request.url, headers={'Location': url}, status=status)
|
||||
request = self.redirectmw.process_response(request, response, self.spider)
|
||||
self.referrermw.request_scheduled(request, self.spider)
|
||||
|
||||
assert isinstance(request, Request)
|
||||
self.assertEquals(request.headers.get('Referer'), final_referrer)
|
||||
|
||||
|
||||
class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
|
||||
"""
|
||||
No Referrer policy never sets the "Referer" header.
|
||||
HTTP redirections should not change that.
|
||||
"""
|
||||
settings = {'REFERRER_POLICY': 'no-referrer'}
|
||||
scenarii = [
|
||||
( 'http://scrapytest.org/1', # parent
|
||||
'http://scrapytest.org/2', # target
|
||||
(
|
||||
# redirections: code, URL
|
||||
(301, 'http://scrapytest.org/3'),
|
||||
(301, 'http://scrapytest.org/4'),
|
||||
),
|
||||
None, # expected initial "Referer"
|
||||
None, # expected "Referer" for the redirection request
|
||||
),
|
||||
( 'https://scrapytest.org/1',
|
||||
'https://scrapytest.org/2',
|
||||
(
|
||||
(301, 'http://scrapytest.org/3'),
|
||||
),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
( 'https://scrapytest.org/1',
|
||||
'https://example.com/2', # different origin
|
||||
(
|
||||
(301, 'http://scrapytest.com/3'),
|
||||
),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
|
||||
"""
|
||||
Same Origin policy sends the full URL as "Referer" if the target origin
|
||||
is the same as the parent response (same protocol, same domain, same port).
|
||||
|
||||
HTTP redirections to a different domain or a lower secure level
|
||||
should have the "Referer" removed.
|
||||
"""
|
||||
settings = {'REFERRER_POLICY': 'same-origin'}
|
||||
scenarii = [
|
||||
( 'http://scrapytest.org/101', # origin
|
||||
'http://scrapytest.org/102', # target
|
||||
(
|
||||
# redirections: code, URL
|
||||
(301, 'http://scrapytest.org/103'),
|
||||
(301, 'http://scrapytest.org/104'),
|
||||
),
|
||||
b'http://scrapytest.org/101', # expected initial "Referer"
|
||||
b'http://scrapytest.org/101', # expected referer for the redirection request
|
||||
),
|
||||
( 'https://scrapytest.org/201',
|
||||
'https://scrapytest.org/202',
|
||||
(
|
||||
# redirecting from secure to non-secure URL == different origin
|
||||
(301, 'http://scrapytest.org/203'),
|
||||
),
|
||||
b'https://scrapytest.org/201',
|
||||
None,
|
||||
),
|
||||
( 'https://scrapytest.org/301',
|
||||
'https://scrapytest.org/302',
|
||||
(
|
||||
# different domain == different origin
|
||||
(301, 'http://example.com/303'),
|
||||
),
|
||||
b'https://scrapytest.org/301',
|
||||
None,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
|
||||
"""
|
||||
Strict Origin policy will always send the "origin" as referrer
|
||||
(think of it as the parent URL without the path part),
|
||||
unless the security level is lower and no "Referer" is sent.
|
||||
|
||||
Redirections from secure to non-secure URLs should have the
|
||||
"Referrer" header removed if necessary.
|
||||
"""
|
||||
settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN}
|
||||
scenarii = [
|
||||
( 'http://scrapytest.org/101',
|
||||
'http://scrapytest.org/102',
|
||||
(
|
||||
(301, 'http://scrapytest.org/103'),
|
||||
(301, 'http://scrapytest.org/104'),
|
||||
),
|
||||
b'http://scrapytest.org/', # send origin
|
||||
b'http://scrapytest.org/', # redirects to same origin: send origin
|
||||
),
|
||||
( 'https://scrapytest.org/201',
|
||||
'https://scrapytest.org/202',
|
||||
(
|
||||
# redirecting to non-secure URL: no referrer
|
||||
(301, 'http://scrapytest.org/203'),
|
||||
),
|
||||
b'https://scrapytest.org/',
|
||||
None,
|
||||
),
|
||||
( 'https://scrapytest.org/301',
|
||||
'https://scrapytest.org/302',
|
||||
(
|
||||
# redirecting to non-secure URL (different domain): no referrer
|
||||
(301, 'http://example.com/303'),
|
||||
),
|
||||
b'https://scrapytest.org/',
|
||||
None,
|
||||
),
|
||||
( 'http://scrapy.org/401',
|
||||
'http://example.com/402',
|
||||
(
|
||||
(301, 'http://scrapytest.org/403'),
|
||||
),
|
||||
b'http://scrapy.org/',
|
||||
b'http://scrapy.org/',
|
||||
),
|
||||
( 'https://scrapy.org/501',
|
||||
'https://example.com/502',
|
||||
(
|
||||
# HTTPS all along, so origin referrer is kept as-is
|
||||
(301, 'https://google.com/503'),
|
||||
(301, 'https://facebook.com/504'),
|
||||
),
|
||||
b'https://scrapy.org/',
|
||||
b'https://scrapy.org/',
|
||||
),
|
||||
( 'https://scrapytest.org/601',
|
||||
'http://scrapytest.org/602', # TLS to non-TLS: no referrer
|
||||
(
|
||||
(301, 'https://scrapytest.org/603'), # TLS URL again: (still) no referrer
|
||||
),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
|
||||
"""
|
||||
Origin When Cross-Origin policy sends the full URL as "Referer",
|
||||
unless the target's origin is different (different domain, different protocol)
|
||||
in which case only the origin is sent.
|
||||
|
||||
Redirections to a different origin should strip the "Referer"
|
||||
to the parent origin.
|
||||
"""
|
||||
settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
|
||||
scenarii = [
|
||||
( 'http://scrapytest.org/101', # origin
|
||||
'http://scrapytest.org/102', # target + redirection
|
||||
(
|
||||
# redirections: code, URL
|
||||
(301, 'http://scrapytest.org/103'),
|
||||
(301, 'http://scrapytest.org/104'),
|
||||
),
|
||||
b'http://scrapytest.org/101', # expected initial referer
|
||||
b'http://scrapytest.org/101', # expected referer for the redirection request
|
||||
),
|
||||
( 'https://scrapytest.org/201',
|
||||
'https://scrapytest.org/202',
|
||||
(
|
||||
# redirecting to non-secure URL: send origin
|
||||
(301, 'http://scrapytest.org/203'),
|
||||
),
|
||||
b'https://scrapytest.org/201',
|
||||
b'https://scrapytest.org/',
|
||||
),
|
||||
( 'https://scrapytest.org/301',
|
||||
'https://scrapytest.org/302',
|
||||
(
|
||||
# redirecting to non-secure URL (different domain): send origin
|
||||
(301, 'http://example.com/303'),
|
||||
),
|
||||
b'https://scrapytest.org/301',
|
||||
b'https://scrapytest.org/',
|
||||
),
|
||||
( 'http://scrapy.org/401',
|
||||
'http://example.com/402',
|
||||
(
|
||||
(301, 'http://scrapytest.org/403'),
|
||||
),
|
||||
b'http://scrapy.org/',
|
||||
b'http://scrapy.org/',
|
||||
),
|
||||
( 'https://scrapy.org/501',
|
||||
'https://example.com/502',
|
||||
(
|
||||
# all different domains: send origin
|
||||
(301, 'https://google.com/503'),
|
||||
(301, 'https://facebook.com/504'),
|
||||
),
|
||||
b'https://scrapy.org/',
|
||||
b'https://scrapy.org/',
|
||||
),
|
||||
( 'https://scrapytest.org/301',
|
||||
'http://scrapytest.org/302', # TLS to non-TLS: send origin
|
||||
(
|
||||
(301, 'https://scrapytest.org/303'), # TLS URL again: send origin (also)
|
||||
),
|
||||
b'https://scrapytest.org/',
|
||||
b'https://scrapytest.org/',
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
|
||||
"""
|
||||
Strict Origin When Cross-Origin policy sends the full URL as "Referer",
|
||||
unless the target's origin is different (different domain, different protocol)
|
||||
in which case only the origin is sent...
|
||||
Unless there's also a downgrade in security and then the "Referer" header
|
||||
is not sent.
|
||||
|
||||
Redirections to a different origin should strip the "Referer" to the parent origin,
|
||||
and from https:// to http:// will remove the "Referer" header.
|
||||
"""
|
||||
settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
|
||||
scenarii = [
|
||||
( 'http://scrapytest.org/101', # origin
|
||||
'http://scrapytest.org/102', # target + redirection
|
||||
(
|
||||
# redirections: code, URL
|
||||
(301, 'http://scrapytest.org/103'),
|
||||
(301, 'http://scrapytest.org/104'),
|
||||
),
|
||||
b'http://scrapytest.org/101', # expected initial referer
|
||||
b'http://scrapytest.org/101', # expected referer for the redirection request
|
||||
),
|
||||
( 'https://scrapytest.org/201',
|
||||
'https://scrapytest.org/202',
|
||||
(
|
||||
# redirecting to non-secure URL: do not send the "Referer" header
|
||||
(301, 'http://scrapytest.org/203'),
|
||||
),
|
||||
b'https://scrapytest.org/201',
|
||||
None,
|
||||
),
|
||||
( 'https://scrapytest.org/301',
|
||||
'https://scrapytest.org/302',
|
||||
(
|
||||
# redirecting to non-secure URL (different domain): send origin
|
||||
(301, 'http://example.com/303'),
|
||||
),
|
||||
b'https://scrapytest.org/301',
|
||||
None,
|
||||
),
|
||||
( 'http://scrapy.org/401',
|
||||
'http://example.com/402',
|
||||
(
|
||||
(301, 'http://scrapytest.org/403'),
|
||||
),
|
||||
b'http://scrapy.org/',
|
||||
b'http://scrapy.org/',
|
||||
),
|
||||
( 'https://scrapy.org/501',
|
||||
'https://example.com/502',
|
||||
(
|
||||
# all different domains: send origin
|
||||
(301, 'https://google.com/503'),
|
||||
(301, 'https://facebook.com/504'),
|
||||
),
|
||||
b'https://scrapy.org/',
|
||||
b'https://scrapy.org/',
|
||||
),
|
||||
( 'https://scrapytest.org/601',
|
||||
'http://scrapytest.org/602', # TLS to non-TLS: do not send "Referer"
|
||||
(
|
||||
(301, 'https://scrapytest.org/603'), # TLS URL again: (still) send nothing
|
||||
),
|
||||
None,
|
||||
None,
|
||||
),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -6,7 +6,8 @@ from six.moves.urllib.parse import urlparse
|
|||
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
|
||||
add_http_if_no_scheme, guess_scheme, parse_url)
|
||||
add_http_if_no_scheme, guess_scheme,
|
||||
parse_url, strip_url)
|
||||
|
||||
__doctests__ = ['scrapy.utils.url']
|
||||
|
||||
|
|
@ -241,5 +242,171 @@ for k, args in enumerate ([
|
|||
setattr (GuessSchemeTest, t_method.__name__, t_method)
|
||||
|
||||
|
||||
class StripUrl(unittest.TestCase):
|
||||
|
||||
def test_noop(self):
|
||||
self.assertEqual(strip_url(
|
||||
'http://www.example.com/index.html'),
|
||||
'http://www.example.com/index.html')
|
||||
|
||||
def test_noop_query_string(self):
|
||||
self.assertEqual(strip_url(
|
||||
'http://www.example.com/index.html?somekey=somevalue'),
|
||||
'http://www.example.com/index.html?somekey=somevalue')
|
||||
|
||||
def test_fragments(self):
|
||||
self.assertEqual(strip_url(
|
||||
'http://www.example.com/index.html?somekey=somevalue#section', strip_fragment=False),
|
||||
'http://www.example.com/index.html?somekey=somevalue#section')
|
||||
|
||||
def test_path(self):
|
||||
for input_url, origin, output_url in [
|
||||
('http://www.example.com/',
|
||||
False,
|
||||
'http://www.example.com/'),
|
||||
|
||||
('http://www.example.com',
|
||||
False,
|
||||
'http://www.example.com'),
|
||||
|
||||
('http://www.example.com',
|
||||
True,
|
||||
'http://www.example.com/'),
|
||||
]:
|
||||
self.assertEqual(strip_url(input_url, origin_only=origin), output_url)
|
||||
|
||||
def test_credentials(self):
|
||||
for i, o in [
|
||||
('http://username@www.example.com/index.html?somekey=somevalue#section',
|
||||
'http://www.example.com/index.html?somekey=somevalue'),
|
||||
|
||||
('https://username:@www.example.com/index.html?somekey=somevalue#section',
|
||||
'https://www.example.com/index.html?somekey=somevalue'),
|
||||
|
||||
('ftp://username:password@www.example.com/index.html?somekey=somevalue#section',
|
||||
'ftp://www.example.com/index.html?somekey=somevalue'),
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_credentials=True), o)
|
||||
|
||||
def test_credentials_encoded_delims(self):
|
||||
for i, o in [
|
||||
# user: "username@"
|
||||
# password: none
|
||||
('http://username%40@www.example.com/index.html?somekey=somevalue#section',
|
||||
'http://www.example.com/index.html?somekey=somevalue'),
|
||||
|
||||
# user: "username:pass"
|
||||
# password: ""
|
||||
('https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section',
|
||||
'https://www.example.com/index.html?somekey=somevalue'),
|
||||
|
||||
# user: "me"
|
||||
# password: "user@domain.com"
|
||||
('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section',
|
||||
'ftp://www.example.com/index.html?somekey=somevalue'),
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_credentials=True), o)
|
||||
|
||||
def test_default_ports_creds_off(self):
|
||||
for i, o in [
|
||||
('http://username:password@www.example.com:80/index.html?somekey=somevalue#section',
|
||||
'http://www.example.com/index.html?somekey=somevalue'),
|
||||
|
||||
('http://username:password@www.example.com:8080/index.html#section',
|
||||
'http://www.example.com:8080/index.html'),
|
||||
|
||||
('http://username:password@www.example.com:443/index.html?somekey=somevalue&someotherkey=sov#section',
|
||||
'http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov'),
|
||||
|
||||
('https://username:password@www.example.com:443/index.html',
|
||||
'https://www.example.com/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:442/index.html',
|
||||
'https://www.example.com:442/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:80/index.html',
|
||||
'https://www.example.com:80/index.html'),
|
||||
|
||||
('ftp://username:password@www.example.com:21/file.txt',
|
||||
'ftp://www.example.com/file.txt'),
|
||||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://www.example.com:221/file.txt'),
|
||||
]:
|
||||
self.assertEqual(strip_url(i), o)
|
||||
|
||||
def test_default_ports(self):
|
||||
for i, o in [
|
||||
('http://username:password@www.example.com:80/index.html',
|
||||
'http://username:password@www.example.com/index.html'),
|
||||
|
||||
('http://username:password@www.example.com:8080/index.html',
|
||||
'http://username:password@www.example.com:8080/index.html'),
|
||||
|
||||
('http://username:password@www.example.com:443/index.html',
|
||||
'http://username:password@www.example.com:443/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:443/index.html',
|
||||
'https://username:password@www.example.com/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:442/index.html',
|
||||
'https://username:password@www.example.com:442/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:80/index.html',
|
||||
'https://username:password@www.example.com:80/index.html'),
|
||||
|
||||
('ftp://username:password@www.example.com:21/file.txt',
|
||||
'ftp://username:password@www.example.com/file.txt'),
|
||||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://username:password@www.example.com:221/file.txt'),
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)
|
||||
|
||||
def test_default_ports_keep(self):
|
||||
for i, o in [
|
||||
('http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section',
|
||||
'http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov'),
|
||||
|
||||
('http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov#section',
|
||||
'http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov'),
|
||||
|
||||
('http://username:password@www.example.com:443/index.html',
|
||||
'http://username:password@www.example.com:443/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:443/index.html',
|
||||
'https://username:password@www.example.com:443/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:442/index.html',
|
||||
'https://username:password@www.example.com:442/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:80/index.html',
|
||||
'https://username:password@www.example.com:80/index.html'),
|
||||
|
||||
('ftp://username:password@www.example.com:21/file.txt',
|
||||
'ftp://username:password@www.example.com:21/file.txt'),
|
||||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://username:password@www.example.com:221/file.txt'),
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)
|
||||
|
||||
def test_origin_only(self):
|
||||
for i, o in [
|
||||
('http://username:password@www.example.com/index.html',
|
||||
'http://www.example.com/'),
|
||||
|
||||
('http://username:password@www.example.com:80/foo/bar?query=value#somefrag',
|
||||
'http://www.example.com/'),
|
||||
|
||||
('http://username:password@www.example.com:8008/foo/bar?query=value#somefrag',
|
||||
'http://www.example.com:8008/'),
|
||||
|
||||
('https://username:password@www.example.com:443/index.html',
|
||||
'https://www.example.com/'),
|
||||
]:
|
||||
self.assertEqual(strip_url(i, origin_only=True), o)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue