RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add tests for the different referrer policies

This commit is contained in:
Paul Tremberth 2016-10-05 17:12:44 +02:00
parent baed7c436f
commit 7ec1b5f6c3
2 changed files with 235 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
scrapy/spidermiddlewares/referer.py
View File

@ -11,6 +11,15 @@ from scrapy.utils.python import to_native_str
LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)
POLICY_NO_REFERRER = "no-referrer"
POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade"
POLICY_SAME_ORIGIN = "same-origin"
POLICY_ORIGIN = "origin"
POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin"
POLICY_UNSAFE_URL = "unsafe-url"
POLICY_SCRAPY_DEFAULT = "scrapy-default"
class ReferrerPolicy(object):
NOREFERRER_SCHEMES = LOCAL_SCHEMES
@ -38,17 +47,29 @@ class ReferrerPolicy(object):
if parsed.scheme in self.NOREFERRER_SCHEMES:
return None
netloc = parsed.netloc
# strip username and password if present
if parsed.username or parsed.password:
netloc = parsed.netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '')
else:
netloc = parsed.netloc
netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '')
# strip standard protocol numbers
# Note: strictly speaking, standard port numbers should only be
# stripped when comparing origins
if parsed.port:
if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)):
netloc = netloc.replace(':{p.port}'.format(p=parsed), '')
return urlunsplit((
parsed.scheme,
netloc,
'' if origin_only else parsed.path,
'/' if origin_only else parsed.path,
'' if origin_only else parsed.query,
''))
def origin(self, url):
return self.strip_url(url, origin_only=True)
class NoReferrerPolicy(ReferrerPolicy):
"""
@ -58,7 +79,7 @@ class NoReferrerPolicy(ReferrerPolicy):
is to be sent along with requests made from a particular request client to any origin.
The header will be omitted entirely.
"""
name = "no-referrer"
name = POLICY_NO_REFERRER
def referrer(self, response, request):
return None
@ -79,7 +100,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
This is a user agent's default behavior, if no policy is otherwise specified.
"""
name = "no-referrer-when-downgrade"
name = POLICY_NO_REFERRER_WHEN_DOWNGRADE
def referrer(self, response, request):
target_url = request.url
@ -108,12 +129,12 @@ class SameOriginPolicy(ReferrerPolicy):
Cross-origin requests, on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
"""
name = "same-origin"
name = POLICY_SAME_ORIGIN
def referrer(self, response, request):
target_url = request.url
referrer_source = response.url
if urlsplit(referrer_source).netloc == urlsplit(target_url).netloc:
if self.origin(referrer_source) == self.origin(target_url):
return self.strip_url(referrer_source)
else:
return None
@ -128,10 +149,10 @@ class OriginPolicy(ReferrerPolicy):
when making both same-origin requests and cross-origin requests
from a particular request client.
"""
name = "origin"
name = POLICY_ORIGIN
def referrer(self, response, request):
return self.strip_url(referrer_source, origin_only=True)
return self.strip_url(response.url, origin_only=True)
class OriginWhenCrossOriginPolicy(ReferrerPolicy):
@ -145,17 +166,17 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy):
is sent as referrer information when making cross-origin requests
from a particular request client.
"""
name = "origin-when-cross-origin"
name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
def referrer(self, response, request):
target_url = request.url
referrer_source = response.url
source_origin = self.origin(referrer_source)
if source_origin == self.origin(target_url):
return self.strip_url(referrer_source, origin_only=False)
else:
return source_origin
# same origin --> send full referrer
# different origin --> send only "origin" as referrer
if urlsplit(referrer_source).netloc != urlsplit(target_url).netloc:
origin_only = True
return self.strip_url(referrer_source, origin_only=origin_only)
class UnsafeUrlPolicy(ReferrerPolicy):
@ -171,7 +192,7 @@ class UnsafeUrlPolicy(ReferrerPolicy):
to insecure origins.
Carefully consider the impact of setting such a policy for potentially sensitive documents.
"""
name = "unsafe-url"
name = POLICY_UNSAFE_URL
def referrer(self, response, request):
referrer_source = response.url
@ -186,15 +207,17 @@ class LegacyPolicy(ReferrerPolicy):
class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')
name = POLICY_SCRAPY_DEFAULT
_policies = {p.name: p for p in (
_policy_classes = {p.name: p for p in (
NoReferrerPolicy,
NoReferrerWhenDowngradePolicy,
SameOriginPolicy,
OriginPolicy,
OriginWhenCrossOriginPolicy,
UnsafeUrlPolicy,
DefaultReferrerPolicy,
)}
class RefererMiddleware(object):
@ -211,10 +234,11 @@ class RefererMiddleware(object):
def policy(self, response, request):
policy_name = request.meta.get('referrer_policy')
if policy_name is None:
policy_name = to_native_str(response.headers.get('Referrer-Policy', '').decode('latin1'))
policy_name = to_native_str(
response.headers.get('Referrer-Policy', '').decode('latin1'))
policy_class = _policies.get(policy_name.lower(), self.default_policy)
return policy_class()
cls = _policy_classes.get(policy_name.lower(), self.default_policy)
return cls()
def process_spider_output(self, response, result, spider):
def _set_referer(r):

242
tests/test_spidermiddleware_referer.py
View File

@ -2,7 +2,10 @@ from unittest import TestCase
from scrapy.http import Response, Request
from scrapy.spiders import Spider
from scrapy.spidermiddlewares.referer import RefererMiddleware
from scrapy.spidermiddlewares.referer import RefererMiddleware, \
POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \
POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \
POLICY_UNSAFE_URL
class TestRefererMiddleware(TestCase):
@ -25,71 +28,206 @@ class TestRefererMiddleware(TestCase):
with some additional filtering of s3://
"""
# a) https:// --> https:// -- include Referer header
origin = Response('https://example.com/')
target = Request('https://scrapy.org/')
for origin, target, referrer in [
('https://example.com/', 'https://scrapy.org/', b'https://example.com/'),
('http://example.com/', 'http://scrapy.org/', b'http://example.com/'),
('http://example.com/', 'https://scrapy.org/', b'http://example.com/'),
('https://example.com/', 'http://scrapy.org/', None),
out = list(self.mw.process_spider_output(origin, [target], self.spider))
self.assertEquals(out[0].headers.get('Referer'),
b'https://example.com/')
# no credentials leak
('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'),
# b.1) http:// --> http:// -- include Referer header
origin = Response('http://example.com/')
target = Request('http://scrapy.org/')
# no referrer leak for local schemes
('file:///home/path/to/somefile.html', 'https://scrapy.org/', None),
('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),
out = list(self.mw.process_spider_output(origin, [target], self.spider))
self.assertEquals(out[0].headers.get('Referer'),
b'http://example.com/')
# no referrer leak for s3 origins
('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None),
('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None),
]:
response = Response(origin)
request = Request(target)
# b.2) http:// --> https:// -- include Referer header
origin = Response('http://example.com/')
target = Request('https://scrapy.org/')
out = list(self.mw.process_spider_output(response, [request], self.spider))
self.assertEquals(out[0].headers.get('Referer'), referrer)
out = list(self.mw.process_spider_output(origin, [target], self.spider))
self.assertEquals(out[0].headers.get('Referer'),
b'http://example.com/')
def test_policy_no_referrer(self):
# c) https:// --> http:// -- Referer header NOT sent
origin = Response('https://example.com/')
target = Request('http://scrapy.org/')
for origin, target, referrer in [
('https://example.com/page.html', 'https://example.com/', None),
('http://www.example.com/', 'https://scrapy.org/', None),
('http://www.example.com/', 'http://scrapy.org/', None),
('https://www.example.com/', 'http://scrapy.org/', None),
('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),
]:
response = Response(origin)
request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER})
out = list(self.mw.process_spider_output(origin, [target], self.spider))
self.assertEquals(out[0].headers.get('Referer'), None)
out = list(self.mw.process_spider_output(response, [request], self.spider))
self.assertEquals(out[0].headers.get('Referer'), referrer)
def test_policy_default_no_credentials_leak(self):
origin = Response('http://user:password@example.com/')
target = Request('https://scrapy.org/')
def test_policy_no_referrer_when_downgrade(self):
out = list(self.mw.process_spider_output(origin, [target], self.spider))
self.assertEquals(out[0].headers.get('Referer'),
b'http://example.com/')
for origin, target, referrer in [
# TLS to TLS: send non-empty referrer
('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'),
('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),
('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),
('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'),
('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'),
def test_policy_default_file_no_referrer_leak(self):
# file:// --> https:// -- Referrer NOT sent
origin = Response('file:///home/path/to/somefile.html')
target = Request('https://scrapy.org/')
# TLS to non-TLS: do not send referrer
('https://example.com/page.html', 'http://not.example.com/', None),
('https://example.com/page.html', 'http://scrapy.org/', None),
('ftps://example.com/urls.zip', 'http://scrapy.org/', None),
out = list(self.mw.process_spider_output(origin, [target], self.spider))
self.assertEquals(out[0].headers.get('Referer'), None)
# non-TLS to TLS or non-TLS: send referrer
('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'),
('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'),
('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'),
('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'),
('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'),
('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'),
('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'),
# file:// --> http:// -- Referrer NOT sent
origin = Response('file:///home/path/to/somefile.html')
target = Request('http://scrapy.org/')
# test for user/password stripping
('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
]:
response = Response(origin)
request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE})
out = list(self.mw.process_spider_output(origin, [target], self.spider))
self.assertEquals(out[0].headers.get('Referer'), None)
out = list(self.mw.process_spider_output(response, [request], self.spider))
self.assertEquals(out[0].headers.get('Referer'), referrer)
def test_policy_default_s3_no_referrer_leak(self):
# s3:// --> https:// -- Referrer NOT sent
origin = Response('s3://mybucket/path/to/data.csv')
target = Request('https://scrapy.org/')
def test_policy_same_origin(self):
out = list(self.mw.process_spider_output(origin, [target], self.spider))
self.assertEquals(out[0].headers.get('Referer'), None)
for origin, target, referrer in [
# Same origin (protocol, host, port): send referrer
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
# s3:// --> http:// -- Referrer NOT sent
origin = Response('s3://mybucket/path/to/data.csv')
target = Request('http://scrapy.org/')
# Different host: do NOT send referrer
('https://example.com/page.html', 'https://not.example.com/otherpage.html', None),
('http://example.com/page.html', 'http://not.example.com/otherpage.html', None),
('http://example.com/page.html', 'http://www.example.com/otherpage.html', None),
out = list(self.mw.process_spider_output(origin, [target], self.spider))
self.assertEquals(out[0].headers.get('Referer'), None)
# Different port: do NOT send referrer
('https://example.com:444/page.html', 'https://example.com/not-page.html', None),
('http://example.com:81/page.html', 'http://example.com/not-page.html', None),
('http://example.com/page.html', 'http://example.com:81/not-page.html', None),
# Different protocols: do NOT send refferer
('https://example.com/page.html', 'http://example.com/not-page.html', None),
('https://example.com/page.html', 'http://not.example.com/', None),
('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None),
('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
# test for user/password stripping
('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None),
]:
response = Response(origin)
request = Request(target, meta={'referrer_policy': POLICY_SAME_ORIGIN})
out = list(self.mw.process_spider_output(response, [request], self.spider))
self.assertEquals(out[0].headers.get('Referer'), referrer)
def test_policy_origin(self):
for origin, target, referrer in [
# TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),
('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),
# test for user/password stripping
('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
]:
response = Response(origin)
request = Request(target, meta={'referrer_policy': POLICY_ORIGIN})
out = list(self.mw.process_spider_output(response, [request], self.spider))
self.assertEquals(out[0].headers.get('Referer'), referrer)
def test_policy_origin_when_cross_origin(self):
for origin, target, referrer in [
# Same origin (protocol, host, port): send referrer
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
# Different host: send origin as referrer
('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),
('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),
# exact match required
('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),
# Different port: send origin as referrer
('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),
('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),
# Different protocols: send origin as referrer
('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'),
('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'),
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
# test for user/password stripping
('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),
# TLS to non-TLS downgrade: send origin
('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'),
]:
response = Response(origin)
request = Request(target, meta={'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN})
out = list(self.mw.process_spider_output(response, [request], self.spider))
self.assertEquals(out[0].headers.get('Referer'), referrer)
def test_policy_unsafe_url(self):
for origin, target, referrer in [
# TLS to TLS: send referrer
('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'),
('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'),
('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),
('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),
('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'),
('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'),
# TLS to non-TLS: send referrer (yes, it's unsafe)
('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'),
('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'),
('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'),
# non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)
('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'),
('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'),
('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'),
('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'),
('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'),
('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'),
('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'),
('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'),
# test for user/password stripping
('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'),
('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'),
]:
response = Response(origin)
request = Request(target, meta={'referrer_policy': POLICY_UNSAFE_URL})
out = list(self.mw.process_spider_output(response, [request], self.spider))
self.assertEquals(out[0].headers.get('Referer'), referrer)