From b6ab1ae9c3ffa7aeee0e7ee77e2dd55d7b663a30 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 3 Jan 2017 15:14:59 -0200 Subject: [PATCH 001/109] docs: installation instructions, mention conda in the beginning (closes #2475) --- docs/intro/install.rst | 74 +++++++++++------------------------------- 1 file changed, 19 insertions(+), 55 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 767749ec5..86387ef5e 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,14 +7,25 @@ Installation guide Installing Scrapy ================= -Scrapy runs on Python 2.7 and Python 3.3 or above -(except on Windows where Python 3 is not supported yet). +Scrapy runs on Python 2.7 and Python 3.3 or above. -If you’re already familiar with installation of Python packages, +If you're using `Anaconda`_ or `Miniconda`_, you can install the package from +the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows +and OS X. + +To install Scrapy using ``conda``, run:: + + conda install -c conda-forge scrapy + +Alternatively, if you’re already familiar with installation of Python packages, you can install Scrapy and its dependencies from PyPI with:: pip install Scrapy +Note that sometimes this may require solving compilation issues for some Scrapy +dependencies depending on your operating system, so be sure to check the +:ref:`intro-install-platform-notes`. + We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, to avoid conflicting with your system packages. @@ -108,42 +119,14 @@ Platform specific installation notes Windows ------- -* Install Python 2.7 from https://www.python.org/downloads/ +Though it's possible to install Scrapy on Windows using pip, we recommend you +to install `Anaconda`_ or `Miniconda`_ and use the package from the +`conda-forge`_ channel, which will avoid most installation issues. - You need to adjust ``PATH`` environment variable to include paths to - the Python executable and additional scripts. The following paths need to be - added to ``PATH``:: +Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: - C:\Python27\;C:\Python27\Scripts\; + conda install -c conda-forge scrapy - To update the ``PATH`` open a Command prompt and run:: - - c:\python27\python.exe c:\python27\tools\scripts\win_add2path.py - - Close the command prompt window and reopen it so changes take effect, run the - following command and check it shows the expected Python version:: - - python --version - -* Install `pywin32` from http://sourceforge.net/projects/pywin32/ - - Be sure you download the architecture (win32 or amd64) that matches your system - -* *(Only required for Python<2.7.9)* Install `pip`_ from - https://pip.pypa.io/en/latest/installing/ - - Now open a Command prompt to check ``pip`` is installed correctly:: - - pip --version - -* At this point Python 2.7 and ``pip`` package manager must be working, let's - install Scrapy:: - - pip install Scrapy - -.. note:: - Python 3 is not supported on Windows. This is because Scrapy core requirement Twisted does not support - Python 3 on Windows. Ubuntu 12.04 or above --------------------- @@ -234,27 +217,8 @@ After any of these workarounds you should be able to install Scrapy:: pip install Scrapy -Anaconda --------- - - -Using Anaconda is an alternative to using a virtualenv and installing with ``pip``. - -.. note:: - - For Windows users, or if you have issues installing through ``pip``, this is - the recommended way to install Scrapy. - -If you already have `Anaconda`_ or `Miniconda`_ installed, the `conda-forge`_ -community have up-to-date packages for Linux, Windows and OS X. - -To install Scrapy using ``conda``, run:: - - conda install -c conda-forge scrapy - .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing/ -.. _Control Panel: https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx .. _lxml: http://lxml.de/ .. _parsel: https://pypi.python.org/pypi/parsel .. _w3lib: https://pypi.python.org/pypi/w3lib From bb7d99ed81e70a5a59d47e69995ac5c025a6a8c0 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 01:30:45 +0500 Subject: [PATCH 002/109] drop unneeded urlparse call --- scrapy/linkextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index e5d21e174..8676c3b92 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -101,7 +101,7 @@ class FilteringLinkExtractor(object): links = [x for x in links if self._link_allowed(x)] if self.canonicalize: for link in links: - link.url = canonicalize_url(urlparse(link.url)) + link.url = canonicalize_url(link.url) links = self.link_extractor._process_links(links) return links From 47f7da8724b5a453979c2b0d10114c23ac1bd170 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 02:03:26 +0500 Subject: [PATCH 003/109] canonicalize=False by default for LinkExtractor. Fixes GH-1941. --- docs/topics/link-extractors.rst | 9 +++++-- scrapy/linkextractors/lxmlhtml.py | 22 +++++++++++---- .../link_extractor/sgml_linkextractor.html | 1 + tests/test_linkextractors.py | 27 +++++++++++++++++++ 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 75bdb4142..01d7f0b97 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -51,7 +51,7 @@ LxmlLinkExtractor :synopsis: lxml's HTMLParser-based link extractors -.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, strip=True) +.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, strip=True) LxmlLinkExtractor is the recommended link extractor with handy filtering options. It is implemented using lxml's robust HTMLParser. @@ -103,7 +103,12 @@ LxmlLinkExtractor :type attrs: list :param canonicalize: canonicalize each extracted url (using - w3lib.url.canonicalize_url). Defaults to ``True``. + w3lib.url.canonicalize_url). Defaults to ``False``. + Note that canonicalize_url is meant for duplicate checking; + it can change the URL visible at server side, so the response can be + different for requests with canonicalized and raw URLs. If you're + using LinkExtractor to follow links it is more robust to + keep the default ``canonicalize=False``. :type canonicalize: boolean :param unique: whether duplicate filtering should be applied to extracted diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index c284f1905..a7092f9b8 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -6,6 +6,7 @@ from six.moves.urllib.parse import urljoin import lxml.etree as etree from w3lib.html import strip_html5_whitespace +from w3lib.url import canonicalize_url from scrapy.link import Link from scrapy.utils.misc import arg_to_iter, rel_has_nofollow @@ -29,12 +30,17 @@ def _nons(tag): class LxmlParserLinkExtractor(object): def __init__(self, tag="a", attr="href", process=None, unique=False, - strip=True): + strip=True, canonicalized=False): self.scan_tag = tag if callable(tag) else lambda t: t == tag self.scan_attr = attr if callable(attr) else lambda a: a == attr self.process_attr = process if callable(process) else lambda v: v self.unique = unique self.strip = strip + if canonicalized: + self.link_key = lambda link: link.url + else: + self.link_key = lambda link: canonicalize_url(link.url, + keep_fragments=True) def _iter_links(self, document): for el in document.iter(etree.Element): @@ -82,21 +88,27 @@ class LxmlParserLinkExtractor(object): def _deduplicate_if_needed(self, links): if self.unique: - return unique_list(links, key=lambda link: link.url) + return unique_list(links, key=self.link_key) return links class LxmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=True, + tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, deny_extensions=None, restrict_css=(), strip=True): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) tag_func = lambda x: x in tags attr_func = lambda x: x in attrs - lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process=process_value, strip=strip) + lx = LxmlParserLinkExtractor( + tag=tag_func, + attr=attr_func, + unique=unique, + process=process_value, + strip=strip, + canonicalized=canonicalize + ) super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, allow_domains=allow_domains, deny_domains=deny_domains, diff --git a/tests/sample_data/link_extractor/sgml_linkextractor.html b/tests/sample_data/link_extractor/sgml_linkextractor.html index fbb803f2d..7d5db368a 100644 --- a/tests/sample_data/link_extractor/sgml_linkextractor.html +++ b/tests/sample_data/link_extractor/sgml_linkextractor.html @@ -11,6 +11,7 @@ sample 3 text sample 3 repetition +sample 3 repetition with fragment inner tag href with whitespaces diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 340c64f35..50484f060 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -30,6 +30,7 @@ class Base: Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), @@ -41,6 +42,7 @@ class Base: Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) def test_extract_filter_allow_with_duplicates(self): @@ -50,6 +52,27 @@ class Base: Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') + ]) + + def test_extract_filter_allow_with_duplicates_canonicalize(self): + lx = self.extractor_cls(allow=('sample', ), unique=False, + canonicalize=True) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment') + ]) + + def test_extract_filter_allow_no_duplicates_canonicalize(self): + lx = self.extractor_cls(allow=('sample',), unique=True, + canonicalize=True) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), ]) def test_extract_filter_allow_and_deny(self): @@ -73,6 +96,8 @@ class Base: Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', + text='sample 3 repetition with fragment') ]) lx = self.extractor_cls(allow='sample', deny='3') @@ -280,6 +305,7 @@ class Base: Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), @@ -291,6 +317,7 @@ class Base: Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample2.jpg', text=u''), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), From df446d167f6b539c4a79fb64d91cbc1607f30acb Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 03:11:59 +0500 Subject: [PATCH 004/109] fix deprecated link extractors --- scrapy/linkextractors/sgml.py | 18 +++++++++++------- tests/test_linkextractors_deprecated.py | 2 ++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 11ff7a261..f4ca4262a 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -6,7 +6,7 @@ from six.moves.urllib.parse import urljoin import warnings from sgmllib import SGMLParser -from w3lib.url import safe_url_string +from w3lib.url import safe_url_string, canonicalize_url from w3lib.html import strip_html5_whitespace from scrapy.link import Link @@ -20,7 +20,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning class BaseSgmlLinkExtractor(SGMLParser): def __init__(self, tag="a", attr="href", unique=False, process_value=None, - strip=True): + strip=True, canonicalized=False): warnings.warn( "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. " "Please use scrapy.linkextractors.LinkExtractor", @@ -33,6 +33,11 @@ class BaseSgmlLinkExtractor(SGMLParser): self.current_link = None self.unique = unique self.strip = strip + if canonicalized: + self.link_key = lambda link: link.url + else: + self.link_key = lambda link: canonicalize_url(link.url, + keep_fragments=True) def _extract_links(self, response_text, response_url, response_encoding, base_url=None): """ Do the real extraction work """ @@ -61,8 +66,7 @@ class BaseSgmlLinkExtractor(SGMLParser): The subclass should override it if necessary """ - links = unique_list(links, key=lambda link: link.url) if self.unique else links - return links + return unique_list(links, key=self.link_key) if self.unique else links def extract_links(self, response): # wrapper needed to allow to work directly with text @@ -107,10 +111,9 @@ class BaseSgmlLinkExtractor(SGMLParser): class SgmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, + tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, deny_extensions=None, restrict_css=(), strip=True): - warnings.warn( "SgmlLinkExtractor is deprecated and will be removed in future releases. " "Please use scrapy.linkextractors.LinkExtractor", @@ -124,7 +127,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor): with warnings.catch_warnings(): warnings.simplefilter('ignore', ScrapyDeprecationWarning) lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process_value=process_value, strip=strip) + unique=unique, process_value=process_value, strip=strip, + canonicalized=canonicalize) super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, allow_domains=allow_domains, deny_domains=deny_domains, diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py index fef227aa1..794f85e0f 100644 --- a/tests/test_linkextractors_deprecated.py +++ b/tests/test_linkextractors_deprecated.py @@ -121,6 +121,7 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), @@ -190,6 +191,7 @@ class RegexLinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_links(self.response), [Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'),]) From 2b4d46315f2ce7ce20e7355fe4ad184bb041d6f4 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 21 Feb 2017 00:05:40 +0500 Subject: [PATCH 005/109] TST fixed compatibility with new link extractor whitespace handling --- tests/test_http_response.py | 1 + tests/test_linkextractors.py | 17 ++++++++++++++--- tests/test_linkextractors_deprecated.py | 1 + 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 924bb7979..779f5a71c 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -392,6 +392,7 @@ class TextResponseTest(BaseResponseTest): 'http://example.com/sample2.html', 'http://example.com/sample3.html', 'http://example.com/sample3.html', + 'http://example.com/sample3.html#foo', 'http://www.google.com/something', 'http://example.com/innertag.html' ] diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 50484f060..1d7c4f311 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -13,6 +13,7 @@ from tests import get_testdata class Base: class LinkExtractorTestCase(unittest.TestCase): extractor_cls = None + escapes_whitespace = False def setUp(self): body = get_testdata('link_extractor', 'sgml_linkextractor.html') @@ -26,6 +27,11 @@ class Base: def test_extract_all_links(self): lx = self.extractor_cls() + if self.escapes_whitespace: + page4_url = 'http://example.com/page%204.html' + else: + page4_url = 'http://example.com/page 4.html' + self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), @@ -33,7 +39,7 @@ class Base: Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), + Link(url=page4_url, text=u'href with whitespaces'), ]) def test_extract_filter_allow(self): @@ -301,6 +307,11 @@ class Base: def test_attrs(self): lx = self.extractor_cls(attrs="href") + if self.escapes_whitespace: + page4_url = 'http://example.com/page%204.html' + else: + page4_url = 'http://example.com/page 4.html' + self.assertEqual(lx.extract_links(self.response), [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), @@ -308,7 +319,7 @@ class Base: Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), + Link(url=page4_url, text=u'href with whitespaces'), ]) lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=()) @@ -320,7 +331,7 @@ class Base: Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), + Link(url=page4_url, text=u'href with whitespaces'), ]) lx = self.extractor_cls(attrs=None) diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py index 794f85e0f..1366971be 100644 --- a/tests/test_linkextractors_deprecated.py +++ b/tests/test_linkextractors_deprecated.py @@ -143,6 +143,7 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase): class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = SgmlLinkExtractor + escapes_whitespace = True def test_deny_extensions(self): html = """asd and """ From d60642e1755b004e46b87e2011c1873f25895bb8 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Fri, 12 Aug 2016 00:45:42 -0300 Subject: [PATCH 006/109] data URI download handler. --- scrapy/core/downloader/handlers/data.py | 93 +++++++++++++++++++++++++ tests/test_downloader_handlers.py | 58 ++++++++++++++- 2 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 scrapy/core/downloader/handlers/data.py diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py new file mode 100644 index 000000000..637a6f9c0 --- /dev/null +++ b/scrapy/core/downloader/handlers/data.py @@ -0,0 +1,93 @@ +import base64 +import re +from six.moves.urllib.parse import unquote + +from scrapy.http import TextResponse +from scrapy.responsetypes import responsetypes +from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.decorators import defers + + +# ASCII characters. +_char = set(map(chr, range(127))) + +# RFC 2045 token. +_token = r'[{}]+'.format(re.escape(''.join(_char - + # Control characters. + set(map(chr, range(0, 32))) - + # tspecials and space. + set('()<>@,;:\\"/[]?= ')))) + +# RFC 822 quoted-string, without surrounding quotation marks. +_quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format( + re.escape(''.join(_char - {'"', '\\', '\r'})), + re.escape(''.join(_char)) +) + +# RFC 2397 mediatype. +_mediatype_pattern = re.compile(r'{token}/{token}'.format(token=_token)) + +_mediatype_parameter_pattern = re.compile( + r';({token})=(?:({token})|"({quoted})")'.format(token=_token, + quoted=_quoted_string) +) + + +class DataURIDownloadHandler(object): + def __init__(self, settings): + super(DataURIDownloadHandler, self).__init__() + + @defers + def download_request(self, request, spider): + url = request.url + + scheme, url = url.split(':', 1) + if scheme != 'data': + raise ValueError("not a data URI") + + # RFC 3986 section 2.1 allows percent encoding to escape characters + # that would be interpreted as delimiters, implying that actual + # delimiters should not be percent-encoded. + # Decoding before parsing will allow malformed URIs with + # percent-encoded delimiters, but it makes parsing easier and should + # not affect well-formed URIs, as the delimiters used in this URI + # scheme are not allowed, percent-encoded or not, in tokens. + url = unquote(url) + + media_type = "text/plain" + media_type_params = CaselessDict() + + m = _mediatype_pattern.match(url) + if m: + media_type = m.group() + url = url[m.end():] + else: + media_type_params['charset'] = "US-ASCII" + + while True: + m = _mediatype_parameter_pattern.match(url) + if m: + attribute, value, value_quoted = m.groups() + if value_quoted: + value = re.sub(r'\\(.)', '\1', value_quoted) + media_type_params[attribute] = value + url = url[m.end():] + else: + break + + is_base64, data = url.split(',', 1) + if is_base64: + if is_base64 != ";base64": + raise ValueError("invalid data URI") + data = base64.b64decode(data) + + respcls = responsetypes.from_mimetype(media_type) + + resp_kwargs = {} + + if media_type: + media_type = media_type.split('/') + if issubclass(respcls, TextResponse) and media_type[0] == 'text': + resp_kwargs['encoding'] = media_type_params.get('charset') + + return respcls(url=request.url, body=data, **resp_kwargs) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c1683fb3e..c21a1670f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -6,7 +6,6 @@ try: from unittest import mock except ImportError: import mock -import shutil from twisted.trial import unittest from twisted.protocols.policies import WrappingFactory @@ -20,6 +19,7 @@ from twisted.cred import portal, checkers, credentials from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers +from scrapy.core.downloader.handlers.data import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler @@ -29,6 +29,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spiders import Spider from scrapy.http import Request from scrapy.http.response.text import TextResponse +from scrapy.responsetypes import responsetypes from scrapy.settings import Settings from scrapy.utils.test import get_crawler, skip_if_no_boto from scrapy.utils.python import to_bytes @@ -828,3 +829,58 @@ class AnonymousFTPTestCase(BaseFTPTestCase): def tearDown(self): shutil.rmtree(self.directory) + + +class DataURITestCase(unittest.TestCase): + + def setUp(self): + self.download_handler = DataURIDownloadHandler(Settings()) + self.download_request = self.download_handler.download_request + self.spider = Spider('foo') + + def test_default_mediatype_encoding(self): + def _test(response): + self.assertEquals(response.text, 'A brief note') + self.assertEquals(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEquals(response.encoding, "US-ASCII") + + request = Request("data:,A%20brief%20note") + return self.download_request(request, self.spider).addCallback(_test) + + def test_default_mediatype(self): + def _test(response): + self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEquals(response.encoding, "iso-8859-7") + + request = Request("data:;charset=iso-8859-7,%be%d3%be") + return self.download_request(request, self.spider).addCallback(_test) + + def test_text_charset(self): + def _test(response): + self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(response.encoding, "iso-8859-7") + + request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") + return self.download_request(request, self.spider).addCallback(_test) + + def test_mediatype_parameters(self): + def _test(response): + self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEquals(response.encoding, "utf-8") + + request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;' + 'charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22' + ',%CE%8E%CE%A3%CE%8E') + return self.download_request(request, self.spider).addCallback(_test) + + def test_base64(self): + def _test(response): + self.assertEquals(response.text, 'Hello, world.') + + request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D') + return self.download_request(request, self.spider).addCallback(_test) From 7e9f2c31d78dcc798a281e5ba6ddf91714dc9cc7 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Fri, 12 Aug 2016 02:09:35 -0300 Subject: [PATCH 007/109] Ensure bytes objects when needed in data URI downloader. --- scrapy/core/downloader/handlers/data.py | 29 ++++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py index 637a6f9c0..30c39865f 100644 --- a/scrapy/core/downloader/handlers/data.py +++ b/scrapy/core/downloader/handlers/data.py @@ -1,6 +1,12 @@ import base64 import re -from six.moves.urllib.parse import unquote + +import six + +if six.PY2: + from urllib import unquote +else: + from urllib.parse import unquote_to_bytes as unquote from scrapy.http import TextResponse from scrapy.responsetypes import responsetypes @@ -24,12 +30,19 @@ _quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format( re.escape(''.join(_char)) ) +# Encode the regular expression strings to make them into bytes, as Python 3 +# bytes have no format() method, but bytes must be passed to re.compile() in +# order to make a pattern object that can be used to match on bytes. + # RFC 2397 mediatype. -_mediatype_pattern = re.compile(r'{token}/{token}'.format(token=_token)) +_mediatype_pattern = re.compile( + r'{token}/{token}'.format(token=_token).encode() +) _mediatype_parameter_pattern = re.compile( r';({token})=(?:({token})|"({quoted})")'.format(token=_token, - quoted=_quoted_string) + quoted=_quoted_string + ).encode() ) @@ -59,7 +72,7 @@ class DataURIDownloadHandler(object): m = _mediatype_pattern.match(url) if m: - media_type = m.group() + media_type = m.group().decode() url = url[m.end():] else: media_type_params['charset'] = "US-ASCII" @@ -69,15 +82,15 @@ class DataURIDownloadHandler(object): if m: attribute, value, value_quoted = m.groups() if value_quoted: - value = re.sub(r'\\(.)', '\1', value_quoted) - media_type_params[attribute] = value + value = re.sub(br'\\(.)', r'\1', value_quoted) + media_type_params[attribute.decode()] = value.decode() url = url[m.end():] else: break - is_base64, data = url.split(',', 1) + is_base64, data = url.split(b',', 1) if is_base64: - if is_base64 != ";base64": + if is_base64 != b";base64": raise ValueError("invalid data URI") data = base64.b64decode(data) From 3397d27574d03a222b6a790004fcaa1a03711d75 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 8 Feb 2017 12:32:00 -0200 Subject: [PATCH 008/109] Test for binary body content from data URI downloader. --- tests/test_downloader_handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c21a1670f..cfcdcd8f8 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -861,6 +861,7 @@ class DataURITestCase(unittest.TestCase): def test_text_charset(self): def _test(response): self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(response.body, b'\xbe\xd3\xbe') self.assertEquals(response.encoding, "iso-8859-7") request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") From c847e7d4d00647a953a3c5458d198439c30b87f7 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 8 Feb 2017 12:38:49 -0200 Subject: [PATCH 009/109] Use w3lib data URI parser. --- scrapy/core/downloader/handlers/data.py | 99 ++----------------------- 1 file changed, 8 insertions(+), 91 deletions(-) diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py index 30c39865f..d102f2b73 100644 --- a/scrapy/core/downloader/handlers/data.py +++ b/scrapy/core/downloader/handlers/data.py @@ -1,106 +1,23 @@ -import base64 -import re - -import six - -if six.PY2: - from urllib import unquote -else: - from urllib.parse import unquote_to_bytes as unquote +from w3lib.url import parse_data_uri from scrapy.http import TextResponse from scrapy.responsetypes import responsetypes -from scrapy.utils.datatypes import CaselessDict from scrapy.utils.decorators import defers -# ASCII characters. -_char = set(map(chr, range(127))) - -# RFC 2045 token. -_token = r'[{}]+'.format(re.escape(''.join(_char - - # Control characters. - set(map(chr, range(0, 32))) - - # tspecials and space. - set('()<>@,;:\\"/[]?= ')))) - -# RFC 822 quoted-string, without surrounding quotation marks. -_quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format( - re.escape(''.join(_char - {'"', '\\', '\r'})), - re.escape(''.join(_char)) -) - -# Encode the regular expression strings to make them into bytes, as Python 3 -# bytes have no format() method, but bytes must be passed to re.compile() in -# order to make a pattern object that can be used to match on bytes. - -# RFC 2397 mediatype. -_mediatype_pattern = re.compile( - r'{token}/{token}'.format(token=_token).encode() -) - -_mediatype_parameter_pattern = re.compile( - r';({token})=(?:({token})|"({quoted})")'.format(token=_token, - quoted=_quoted_string - ).encode() -) - - class DataURIDownloadHandler(object): def __init__(self, settings): super(DataURIDownloadHandler, self).__init__() @defers def download_request(self, request, spider): - url = request.url - - scheme, url = url.split(':', 1) - if scheme != 'data': - raise ValueError("not a data URI") - - # RFC 3986 section 2.1 allows percent encoding to escape characters - # that would be interpreted as delimiters, implying that actual - # delimiters should not be percent-encoded. - # Decoding before parsing will allow malformed URIs with - # percent-encoded delimiters, but it makes parsing easier and should - # not affect well-formed URIs, as the delimiters used in this URI - # scheme are not allowed, percent-encoded or not, in tokens. - url = unquote(url) - - media_type = "text/plain" - media_type_params = CaselessDict() - - m = _mediatype_pattern.match(url) - if m: - media_type = m.group().decode() - url = url[m.end():] - else: - media_type_params['charset'] = "US-ASCII" - - while True: - m = _mediatype_parameter_pattern.match(url) - if m: - attribute, value, value_quoted = m.groups() - if value_quoted: - value = re.sub(br'\\(.)', r'\1', value_quoted) - media_type_params[attribute.decode()] = value.decode() - url = url[m.end():] - else: - break - - is_base64, data = url.split(b',', 1) - if is_base64: - if is_base64 != b";base64": - raise ValueError("invalid data URI") - data = base64.b64decode(data) - - respcls = responsetypes.from_mimetype(media_type) + uri = parse_data_uri(request.url) + respcls = responsetypes.from_mimetype(uri.media_type) resp_kwargs = {} + if (issubclass(respcls, TextResponse) and + uri.media_type.split('/')[0] == 'text'): + charset = uri.media_type_parameters.get('charset') + resp_kwargs['encoding'] = charset - if media_type: - media_type = media_type.split('/') - if issubclass(respcls, TextResponse) and media_type[0] == 'text': - resp_kwargs['encoding'] = media_type_params.get('charset') - - return respcls(url=request.url, body=data, **resp_kwargs) + return respcls(url=request.url, body=uri.data, **resp_kwargs) From 121a668a479c3b681581a36c7da090518fbc2120 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Sun, 12 Feb 2017 11:23:21 -0200 Subject: [PATCH 010/109] Rename data URI downloader module. --- scrapy/core/downloader/handlers/{data.py => datauri.py} | 0 tests/test_downloader_handlers.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename scrapy/core/downloader/handlers/{data.py => datauri.py} (100%) diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/datauri.py similarity index 100% rename from scrapy/core/downloader/handlers/data.py rename to scrapy/core/downloader/handlers/datauri.py diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index cfcdcd8f8..b27245a36 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -19,7 +19,7 @@ from twisted.cred import portal, checkers, credentials from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers -from scrapy.core.downloader.handlers.data import DataURIDownloadHandler +from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler From 3139f4a5f700a100b6ab6890677dac808b01835f Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Sun, 12 Feb 2017 11:23:56 -0200 Subject: [PATCH 011/109] Add data URI download handler to settings. --- scrapy/settings/default_settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index e0e39120c..2251d3db5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -67,6 +67,7 @@ DOWNLOAD_DELAY = 0 DOWNLOAD_HANDLERS = {} DOWNLOAD_HANDLERS_BASE = { + 'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler', 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', From 96a570a93a78c977f7554179face6123d338ade0 Mon Sep 17 00:00:00 2001 From: MikeinRealLife Date: Wed, 22 Feb 2017 21:17:34 -0800 Subject: [PATCH 012/109] fixed ticket #2574 --- docs/topics/request-response.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 9a6e0d1b6..018d14100 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -304,6 +304,8 @@ Those are: * :reqmeta:`download_maxsize` * :reqmeta:`download_latency` * :reqmeta:`proxy` +* ``ftp_user`` (See :setting:`FTP_USER` for more info) +* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) .. reqmeta:: bindaddress @@ -332,6 +334,8 @@ used to control Scrapy behavior, this one is supposed to be read-only. .. _topics-request-response-ref-request-subclasses: +.. _topics-request-response-ref-request-subclasses: + Request subclasses ================== From 441f25507ea37b098e14bf3a1f9de7ccbe5a1cc5 Mon Sep 17 00:00:00 2001 From: MikeinRealLife Date: Wed, 22 Feb 2017 21:23:27 -0800 Subject: [PATCH 013/109] fixed typo removed duplicate line --- docs/topics/request-response.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 018d14100..06e245b16 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -334,8 +334,6 @@ used to control Scrapy behavior, this one is supposed to be read-only. .. _topics-request-response-ref-request-subclasses: -.. _topics-request-response-ref-request-subclasses: - Request subclasses ================== From f01ae6ffcd431b73f5358f9f876f8e9ee9be0113 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Thu, 23 Feb 2017 11:42:34 -0400 Subject: [PATCH 014/109] Handle data loss gracefully. Websites that return a wrong ``Content-Length`` header may cause a data loss error. Also when a chunked response is not finished properly. This change adds a new setting ``DOWNLOAD_FAIL_ON_DATALOSS`` (default: ``True``) and request.meta key ``download_fail_on_dataloss``. --- docs/topics/request-response.rst | 9 +++ docs/topics/settings.rst | 26 ++++++ scrapy/core/downloader/handlers/http11.py | 41 +++++++--- scrapy/settings/default_settings.py | 2 + tests/test_downloader_handlers.py | 99 ++++++++++++++++++++++- 5 files changed, 167 insertions(+), 10 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 9a6e0d1b6..214ac5640 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -303,6 +303,7 @@ Those are: * :reqmeta:`download_timeout` * :reqmeta:`download_maxsize` * :reqmeta:`download_latency` +* :reqmeta:`download_fail_on_dataloss` * :reqmeta:`proxy` .. reqmeta:: bindaddress @@ -330,6 +331,14 @@ started, i.e. HTTP message sent over the network. This meta key only becomes available when the response has been downloaded. While most other meta keys are used to control Scrapy behavior, this one is supposed to be read-only. +.. reqmeta:: download_fail_on_dataloss + +download_fail_on_dataloss +------------------------- + +Whether or not to fail on broken responses. See: +:setting:`DOWNLOAD_FAIL_ON_DATALOSS`. + .. _topics-request-response-ref-request-subclasses: Request subclasses diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f616742c4..ccdd02c4e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -604,6 +604,32 @@ If you want to disable it set to 0. This feature needs Twisted >= 11.1. +.. setting:: DOWNLOAD_FAIL_ON_DATALOSS + +DOWNLOAD_FAIL_ON_DATALOSS +------------------------- + +Default: ``True`` + +Whether or not to fail on broken responses, that is, declared +``Content-Length`` does not match content sent by the server or chunked +response was not properly finish. If ``True``, these responses raise a +``ResponseFailed([_DataLoss])`` error. If ``False``, these responses +are passed through and the flag ``dataloss`` is added to the response, i.e.: +``'dataloss' in response.flags`` is ``True``. + +Optionally, this can be set per-request basis by using the +:reqmeta:`download_fail_on_dataloss` Request.meta key to ``False``. + +.. note:: + + A broken response, or data loss error, may happen under several + circumstances, from server misconfiguration to network errors to data + corruption. It is up to the user to decide if it makes sense to process + broken responses considering they may contain partial or incomplete content. + If setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``, + the ``ResponseFailed([_DataLoss])`` failure will be retried as usual. + .. setting:: DUPEFILTER_CLASS DUPEFILTER_CLASS diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index b96c8c6fe..37e836809 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -12,9 +12,9 @@ from twisted.internet import defer, reactor, protocol from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from twisted.internet.error import TimeoutError -from twisted.web.http import PotentialDataLoss +from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool + HTTPConnectionPool, ResponseFailed from twisted.internet.endpoints import TCP4ClientEndpoint from scrapy.http import Headers @@ -51,13 +51,15 @@ class HTTP11DownloadHandler(object): warnings.warn(msg) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') + self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') self._disconnect_timeout = 1 def download_request(self, request, spider): """Return a deferred for the HTTP download""" agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool, maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), - warnsize=getattr(spider, 'download_warnsize', self._default_warnsize)) + warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), + fail_on_dataloss=self._fail_on_dataloss) return agent.download_request(request) def close(self): @@ -233,13 +235,14 @@ class ScrapyAgent(object): _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, - maxsize=0, warnsize=0): + maxsize=0, warnsize=0, fail_on_dataloss=True): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress self._pool = pool self._maxsize = maxsize self._warnsize = warnsize + self._fail_on_dataloss = fail_on_dataloss self._txresponse = None def _get_agent(self, request, timeout): @@ -326,6 +329,7 @@ class ScrapyAgent(object): maxsize = request.meta.get('download_maxsize', self._maxsize) warnsize = request.meta.get('download_warnsize', self._warnsize) expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 + fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss) if maxsize and expected_size > maxsize: error_msg = ("Cancelling download of %(url)s: expected response " @@ -345,7 +349,8 @@ class ScrapyAgent(object): txresponse._transport._producer.loseConnection() d = defer.Deferred(_cancel) - txresponse.deliverBody(_ResponseReader(d, txresponse, request, maxsize, warnsize)) + txresponse.deliverBody(_ResponseReader( + d, txresponse, request, maxsize, warnsize, fail_on_dataloss)) # save response for timeouts self._txresponse = txresponse @@ -380,13 +385,16 @@ class _RequestBodyProducer(object): class _ResponseReader(protocol.Protocol): - def __init__(self, finished, txresponse, request, maxsize, warnsize): + def __init__(self, finished, txresponse, request, maxsize, warnsize, + fail_on_dataloss): self._finished = finished self._txresponse = txresponse self._request = request self._bodybuf = BytesIO() self._maxsize = maxsize self._warnsize = warnsize + self._fail_on_dataloss = fail_on_dataloss + self._fail_on_dataloss_warned = False self._reached_warnsize = False self._bytes_received = 0 @@ -415,7 +423,22 @@ class _ResponseReader(protocol.Protocol): body = self._bodybuf.getvalue() if reason.check(ResponseDone): self._finished.callback((self._txresponse, body, None)) - elif reason.check(PotentialDataLoss): + return + + if reason.check(PotentialDataLoss): self._finished.callback((self._txresponse, body, ['partial'])) - else: - self._finished.errback(reason) + return + + if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons): + if not self._fail_on_dataloss: + self._finished.callback((self._txresponse, body, ['dataloss'])) + return + + elif not self._fail_on_dataloss_warned: + logger.warn("Got data loss in %s. If you want to process broken " + "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" + " -- This message won't be shown in further requests", + self._txresponse.request.absoluteURI.decode()) + self._fail_on_dataloss_warned = True + + self._finished.errback(reason) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index e0e39120c..a5931a3d5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -79,6 +79,8 @@ DOWNLOAD_TIMEOUT = 180 # 3mins DOWNLOAD_MAXSIZE = 1024*1024*1024 # 1024m DOWNLOAD_WARNSIZE = 32*1024*1024 # 32m +DOWNLOAD_FAIL_ON_DATALOSS = True + DOWNLOADER = 'scrapy.core.downloader.Downloader' DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index e49a514b8..4e63b2038 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -13,9 +13,11 @@ from twisted.protocols.policies import WrappingFactory from twisted.python.filepath import FilePath from twisted.internet import reactor, defer, error from twisted.web import server, static, util, resource +from twisted.web._newclient import ResponseFailed +from twisted.web.http import _DataLoss from twisted.web.test.test_webclient import ForeverTakingResource, \ NoLengthResource, HostHeaderResource, \ - PayloadResource, BrokenDownloadResource + PayloadResource from twisted.cred import portal, checkers, credentials from w3lib.url import path_to_file_uri @@ -118,6 +120,52 @@ class ContentLengthHeaderResource(resource.Resource): return request.requestHeaders.getRawHeaders(b"content-length")[0] +class ChunkedResource(resource.Resource): + + def render(self, request): + def response(): + request.write(b"chunked ") + request.write(b"content\n") + request.finish() + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class BrokenChunkedResource(resource.Resource): + + def render(self, request): + def response(): + request.write(b"chunked ") + request.write(b"content\n") + # Disable terminating chunk on finish. + request.chunked = False + closeConnection(request) + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class BrokenDownloadResource(resource.Resource): + + def render(self, request): + def response(): + request.setHeader(b"Content-Length", b"20") + request.write(b"partial") + closeConnection(request) + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +def closeConnection(request): + # We have to force a disconnection for HTTP/1.1 clients. Otherwise + # client keeps the connection open waiting for more data. + if hasattr(request.channel, 'loseConnection'): # twisted >=16.3.0 + request.channel.loseConnection() + else: + request.channel.transport.loseConnection() + request.finish() + + class EmptyContentTypeHeaderResource(resource.Resource): """ A testing resource which renders itself as the value of request body @@ -149,6 +197,8 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"host", HostHeaderResource()) r.putChild(b"payload", PayloadResource()) r.putChild(b"broken", BrokenDownloadResource()) + r.putChild(b"chunked", ChunkedResource()) + r.putChild(b"broken-chunked", BrokenChunkedResource()) r.putChild(b"contentlength", ContentLengthHeaderResource()) r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) self.site = server.Site(r, timeout=None) @@ -341,6 +391,53 @@ class Http11TestCase(HttpTestCase): d.addCallback(self.assertEquals, b"0123456789") return d + def test_download_chunked_content(self): + request = Request(self.getURL('chunked')) + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEquals, b"chunked content\n") + return d + + def test_download_broken_content_cause_data_loss(self, url='broken'): + request = Request(self.getURL(url)) + d = self.download_request(request, Spider('foo')) + + def checkDataLoss(failure): + if failure.check(ResponseFailed): + if any(r.check(_DataLoss) for r in failure.value.reasons): + return None + return failure + + d.addCallback(lambda _: self.fail("No DataLoss exception")) + d.addErrback(checkDataLoss) + return d + + def test_download_broken_chunked_content_cause_data_loss(self): + return self.test_download_broken_content_cause_data_loss('broken-chunked') + + def test_download_broken_content_allow_data_loss(self, url='broken'): + request = Request(self.getURL(url), meta={'download_fail_on_dataloss': False}) + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.flags) + d.addCallback(self.assertEqual, ['dataloss']) + return d + + def test_download_broken_chunked_content_allow_data_loss(self): + return self.test_download_broken_content_allow_data_loss('broken-chunked') + + def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): + download_handler = self.download_handler_cls(Settings({ + 'DOWNLOAD_FAIL_ON_DATALOSS': False, + })) + request = Request(self.getURL(url)) + d = download_handler.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.flags) + d.addCallback(self.assertEqual, ['dataloss']) + return d + + def test_download_broken_chunked_content_allow_data_loss_via_setting(self): + return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked') + class Https11TestCase(Http11TestCase): scheme = 'https' From baed7c436f8c6e52a1aab62fbaf900c0a0f3bbda Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 11:18:26 +0200 Subject: [PATCH 015/109] WIP Add Referrer policies --- scrapy/spidermiddlewares/referer.py | 206 ++++++++++++++++++++++++- tests/test_spidermiddleware_referer.py | 74 +++++++++ 2 files changed, 279 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 6a8c46543..21b340f22 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,22 +2,226 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ +from six.moves.urllib.parse import urlsplit, urlunsplit from scrapy.http import Request from scrapy.exceptions import NotConfigured +from scrapy.utils.python import to_native_str + + +LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) + +class ReferrerPolicy(object): + + NOREFERRER_SCHEMES = LOCAL_SCHEMES + + def referrer(self, response, request): + raise NotImplementedError() + + def strip_url(self, url, origin_only=False): + """ + https://www.w3.org/TR/referrer-policy/#strip-url + + If url is null, return no referrer. + If url's scheme is a local scheme, then return no referrer. + Set url's username to the empty string. + Set url's password to null. + Set url's fragment to null. + If the origin-only flag is true, then: + Set url's path to null. + Set url's query to null. + Return url. + """ + if url is None or not url: + return None + parsed = urlsplit(url, allow_fragments=True) + + if parsed.scheme in self.NOREFERRER_SCHEMES: + return None + if parsed.username or parsed.password: + netloc = parsed.netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') + else: + netloc = parsed.netloc + return urlunsplit(( + parsed.scheme, + netloc, + '' if origin_only else parsed.path, + '' if origin_only else parsed.query, + '')) + + +class NoReferrerPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer + + The simplest policy is "no-referrer", which specifies that no referrer information + is to be sent along with requests made from a particular request client to any origin. + The header will be omitted entirely. + """ + name = "no-referrer" + + def referrer(self, response, request): + return None + + +class NoReferrerWhenDowngradePolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + + The "no-referrer-when-downgrade" policy sends a full URL + along with requests from a TLS-protected environment settings object + to a a priori authenticated URL, + and requests from request clients which are not TLS-protected to any origin. + + Requests from TLS-protected request clients to non-a priori authenticated URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + + This is a user agent's default behavior, if no policy is otherwise specified. + """ + name = "no-referrer-when-downgrade" + + def referrer(self, response, request): + target_url = request.url + + referrer_source = response.url + referrer_url = self.strip_url(referrer_source) + + # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer: + # + # If environment is TLS-protected + # and the origin of request's current URL is not an a priori authenticated URL, + # then return no referrer. + if urlsplit(referrer_source).scheme in ('https', 'ftps') and \ + urlsplit(target_url).scheme in ('http',): + return None + return referrer_url + + +class SameOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin + + The "same-origin" policy specifies that a full URL, stripped for use as a referrer, + is sent as referrer information when making same-origin requests from a particular request client. + + Cross-origin requests, on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = "same-origin" + + def referrer(self, response, request): + target_url = request.url + referrer_source = response.url + if urlsplit(referrer_source).netloc == urlsplit(target_url).netloc: + return self.strip_url(referrer_source) + else: + return None + + +class OriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-origin + + The "origin" policy specifies that only the ASCII serialization + of the origin of the request client is sent as referrer information + when making both same-origin requests and cross-origin requests + from a particular request client. + """ + name = "origin" + + def referrer(self, response, request): + return self.strip_url(referrer_source, origin_only=True) + + +class OriginWhenCrossOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin + + The "origin-when-cross-origin" policy specifies that a full URL, + stripped for use as a referrer, is sent as referrer information + when making same-origin requests from a particular request client, + and only the ASCII serialization of the origin of the request client + is sent as referrer information when making cross-origin requests + from a particular request client. + """ + name = "origin-when-cross-origin" + + def referrer(self, response, request): + target_url = request.url + referrer_source = response.url + + # same origin --> send full referrer + # different origin --> send only "origin" as referrer + if urlsplit(referrer_source).netloc != urlsplit(target_url).netloc: + origin_only = True + return self.strip_url(referrer_source, origin_only=origin_only) + + +class UnsafeUrlPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url + + The "unsafe-url" policy specifies that a full URL, stripped for use as a referrer, + is sent along with both cross-origin requests + and same-origin requests made from a particular request client. + + Note: The policy's name doesn't lie; it is unsafe. + This policy will leak origins and paths from TLS-protected resources + to insecure origins. + Carefully consider the impact of setting such a policy for potentially sensitive documents. + """ + name = "unsafe-url" + + def referrer(self, response, request): + referrer_source = response.url + return self.strip_url(referrer_source) + + +class LegacyPolicy(ReferrerPolicy): + def referrer(self, response, request): + return response.url + + +class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): + + NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') + + +_policies = {p.name: p for p in ( + NoReferrerPolicy, + NoReferrerWhenDowngradePolicy, + SameOriginPolicy, + OriginPolicy, + OriginWhenCrossOriginPolicy, + UnsafeUrlPolicy, +)} class RefererMiddleware(object): + def __init__(self, policy_class=DefaultReferrerPolicy): + self.default_policy = policy_class + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured return cls() + def policy(self, response, request): + policy_name = request.meta.get('referrer_policy') + if policy_name is None: + policy_name = to_native_str(response.headers.get('Referrer-Policy', '').decode('latin1')) + + policy_class = _policies.get(policy_name.lower(), self.default_policy) + return policy_class() + def process_spider_output(self, response, result, spider): def _set_referer(r): if isinstance(r, Request): - r.headers.setdefault('Referer', response.url) + referrer = self.policy(response, r).referrer(response, r) + if referrer is not None: + r.headers.setdefault('Referer', referrer) return r return (_set_referer(r) for r in result or ()) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index bd7673efb..f109bb248 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -19,3 +19,77 @@ class TestRefererMiddleware(TestCase): self.assertEquals(out[0].headers.get('Referer'), b'http://scrapytest.org') + def test_policy_default(self): + """ + Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + + with some additional filtering of s3:// + """ + # a) https:// --> https:// -- include Referer header + origin = Response('https://example.com/') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), + b'https://example.com/') + + # b.1) http:// --> http:// -- include Referer header + origin = Response('http://example.com/') + target = Request('http://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), + b'http://example.com/') + + # b.2) http:// --> https:// -- include Referer header + origin = Response('http://example.com/') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), + b'http://example.com/') + + # c) https:// --> http:// -- Referer header NOT sent + origin = Response('https://example.com/') + target = Request('http://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) + + def test_policy_default_no_credentials_leak(self): + origin = Response('http://user:password@example.com/') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), + b'http://example.com/') + + def test_policy_default_file_no_referrer_leak(self): + # file:// --> https:// -- Referrer NOT sent + origin = Response('file:///home/path/to/somefile.html') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) + + # file:// --> http:// -- Referrer NOT sent + origin = Response('file:///home/path/to/somefile.html') + target = Request('http://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) + + def test_policy_default_s3_no_referrer_leak(self): + # s3:// --> https:// -- Referrer NOT sent + origin = Response('s3://mybucket/path/to/data.csv') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) + + # s3:// --> http:// -- Referrer NOT sent + origin = Response('s3://mybucket/path/to/data.csv') + target = Request('http://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) From 7ec1b5f6c316f6c251821441f513e3f71cf63da0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 17:12:44 +0200 Subject: [PATCH 016/109] Add tests for the different referrer policies --- scrapy/spidermiddlewares/referer.py | 66 ++++--- tests/test_spidermiddleware_referer.py | 242 +++++++++++++++++++------ 2 files changed, 235 insertions(+), 73 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 21b340f22..1895aa95d 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -11,6 +11,15 @@ from scrapy.utils.python import to_native_str LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) +POLICY_NO_REFERRER = "no-referrer" +POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade" +POLICY_SAME_ORIGIN = "same-origin" +POLICY_ORIGIN = "origin" +POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin" +POLICY_UNSAFE_URL = "unsafe-url" +POLICY_SCRAPY_DEFAULT = "scrapy-default" + + class ReferrerPolicy(object): NOREFERRER_SCHEMES = LOCAL_SCHEMES @@ -38,17 +47,29 @@ class ReferrerPolicy(object): if parsed.scheme in self.NOREFERRER_SCHEMES: return None + + netloc = parsed.netloc + # strip username and password if present if parsed.username or parsed.password: - netloc = parsed.netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') - else: - netloc = parsed.netloc + netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') + + # strip standard protocol numbers + # Note: strictly speaking, standard port numbers should only be + # stripped when comparing origins + if parsed.port: + if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): + netloc = netloc.replace(':{p.port}'.format(p=parsed), '') + return urlunsplit(( parsed.scheme, netloc, - '' if origin_only else parsed.path, + '/' if origin_only else parsed.path, '' if origin_only else parsed.query, '')) + def origin(self, url): + return self.strip_url(url, origin_only=True) + class NoReferrerPolicy(ReferrerPolicy): """ @@ -58,7 +79,7 @@ class NoReferrerPolicy(ReferrerPolicy): is to be sent along with requests made from a particular request client to any origin. The header will be omitted entirely. """ - name = "no-referrer" + name = POLICY_NO_REFERRER def referrer(self, response, request): return None @@ -79,7 +100,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): This is a user agent's default behavior, if no policy is otherwise specified. """ - name = "no-referrer-when-downgrade" + name = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response, request): target_url = request.url @@ -108,12 +129,12 @@ class SameOriginPolicy(ReferrerPolicy): Cross-origin requests, on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = "same-origin" + name = POLICY_SAME_ORIGIN def referrer(self, response, request): target_url = request.url referrer_source = response.url - if urlsplit(referrer_source).netloc == urlsplit(target_url).netloc: + if self.origin(referrer_source) == self.origin(target_url): return self.strip_url(referrer_source) else: return None @@ -128,10 +149,10 @@ class OriginPolicy(ReferrerPolicy): when making both same-origin requests and cross-origin requests from a particular request client. """ - name = "origin" + name = POLICY_ORIGIN def referrer(self, response, request): - return self.strip_url(referrer_source, origin_only=True) + return self.strip_url(response.url, origin_only=True) class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -145,17 +166,17 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): is sent as referrer information when making cross-origin requests from a particular request client. """ - name = "origin-when-cross-origin" + name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response, request): target_url = request.url referrer_source = response.url + source_origin = self.origin(referrer_source) + if source_origin == self.origin(target_url): + return self.strip_url(referrer_source, origin_only=False) + else: + return source_origin - # same origin --> send full referrer - # different origin --> send only "origin" as referrer - if urlsplit(referrer_source).netloc != urlsplit(target_url).netloc: - origin_only = True - return self.strip_url(referrer_source, origin_only=origin_only) class UnsafeUrlPolicy(ReferrerPolicy): @@ -171,7 +192,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): to insecure origins. Carefully consider the impact of setting such a policy for potentially sensitive documents. """ - name = "unsafe-url" + name = POLICY_UNSAFE_URL def referrer(self, response, request): referrer_source = response.url @@ -186,15 +207,17 @@ class LegacyPolicy(ReferrerPolicy): class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') + name = POLICY_SCRAPY_DEFAULT -_policies = {p.name: p for p in ( +_policy_classes = {p.name: p for p in ( NoReferrerPolicy, NoReferrerWhenDowngradePolicy, SameOriginPolicy, OriginPolicy, OriginWhenCrossOriginPolicy, UnsafeUrlPolicy, + DefaultReferrerPolicy, )} class RefererMiddleware(object): @@ -211,10 +234,11 @@ class RefererMiddleware(object): def policy(self, response, request): policy_name = request.meta.get('referrer_policy') if policy_name is None: - policy_name = to_native_str(response.headers.get('Referrer-Policy', '').decode('latin1')) + policy_name = to_native_str( + response.headers.get('Referrer-Policy', '').decode('latin1')) - policy_class = _policies.get(policy_name.lower(), self.default_policy) - return policy_class() + cls = _policy_classes.get(policy_name.lower(), self.default_policy) + return cls() def process_spider_output(self, response, result, spider): def _set_referer(r): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index f109bb248..8458fe90b 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -2,7 +2,10 @@ from unittest import TestCase from scrapy.http import Response, Request from scrapy.spiders import Spider -from scrapy.spidermiddlewares.referer import RefererMiddleware +from scrapy.spidermiddlewares.referer import RefererMiddleware, \ + POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ + POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ + POLICY_UNSAFE_URL class TestRefererMiddleware(TestCase): @@ -25,71 +28,206 @@ class TestRefererMiddleware(TestCase): with some additional filtering of s3:// """ - # a) https:// --> https:// -- include Referer header - origin = Response('https://example.com/') - target = Request('https://scrapy.org/') + for origin, target, referrer in [ + ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), + ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), + ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), + ('https://example.com/', 'http://scrapy.org/', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'https://example.com/') + # no credentials leak + ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), - # b.1) http:// --> http:// -- include Referer header - origin = Response('http://example.com/') - target = Request('http://scrapy.org/') + # no referrer leak for local schemes + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://example.com/') + # no referrer leak for s3 origins + ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), + ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), + ]: + response = Response(origin) + request = Request(target) - # b.2) http:// --> https:// -- include Referer header - origin = Response('http://example.com/') - target = Request('https://scrapy.org/') + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://example.com/') + def test_policy_no_referrer(self): - # c) https:// --> http:// -- Referer header NOT sent - origin = Response('https://example.com/') - target = Request('http://scrapy.org/') + for origin, target, referrer in [ + ('https://example.com/page.html', 'https://example.com/', None), + ('http://www.example.com/', 'https://scrapy.org/', None), + ('http://www.example.com/', 'http://scrapy.org/', None), + ('https://www.example.com/', 'http://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER}) - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) - def test_policy_default_no_credentials_leak(self): - origin = Response('http://user:password@example.com/') - target = Request('https://scrapy.org/') + def test_policy_no_referrer_when_downgrade(self): - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://example.com/') + for origin, target, referrer in [ + # TLS to TLS: send non-empty referrer + ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), + ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), + ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), - def test_policy_default_file_no_referrer_leak(self): - # file:// --> https:// -- Referrer NOT sent - origin = Response('file:///home/path/to/somefile.html') - target = Request('https://scrapy.org/') + # TLS to non-TLS: do not send referrer + ('https://example.com/page.html', 'http://not.example.com/', None), + ('https://example.com/page.html', 'http://scrapy.org/', None), + ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + # non-TLS to TLS or non-TLS: send referrer + ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), + ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), + ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), + ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), - # file:// --> http:// -- Referrer NOT sent - origin = Response('file:///home/path/to/somefile.html') - target = Request('http://scrapy.org/') + # test for user/password stripping + ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}) - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) - def test_policy_default_s3_no_referrer_leak(self): - # s3:// --> https:// -- Referrer NOT sent - origin = Response('s3://mybucket/path/to/data.csv') - target = Request('https://scrapy.org/') + def test_policy_same_origin(self): - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + for origin, target, referrer in [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), - # s3:// --> http:// -- Referrer NOT sent - origin = Response('s3://mybucket/path/to/data.csv') - target = Request('http://scrapy.org/') + # Different host: do NOT send referrer + ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + # Different port: do NOT send referrer + ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), + ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), + ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), + + # Different protocols: do NOT send refferer + ('https://example.com/page.html', 'http://example.com/not-page.html', None), + ('https://example.com/page.html', 'http://not.example.com/', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_SAME_ORIGIN}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) + + def test_policy_origin(self): + + for origin, target, referrer in [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_ORIGIN}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) + + def test_policy_origin_when_cross_origin(self): + + for origin, target, referrer in [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # Different protocols: send origin as referrer + ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), + ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + # TLS to non-TLS downgrade: send origin + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) + + def test_policy_unsafe_url(self): + + for origin, target, referrer in [ + # TLS to TLS: send referrer + ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), + ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), + ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), + ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), + + # TLS to non-TLS: send referrer (yes, it's unsafe) + ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), + ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), + ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), + + # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) + ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), + ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), + ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), + ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), + ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_UNSAFE_URL}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) From 3af88a2877f947c74c6d9003da620629b27d5a17 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 18:27:18 +0200 Subject: [PATCH 017/109] Use urlparse_cached() on request and responses --- scrapy/spidermiddlewares/referer.py | 79 ++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 19 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 1895aa95d..cbbcd3e97 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,11 +2,12 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ -from six.moves.urllib.parse import urlsplit, urlunsplit +from six.moves.urllib.parse import urlparse, urlunparse, ParseResult from scrapy.http import Request from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_native_str +from scrapy.utils.httpobj import urlparse_cached LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) @@ -27,7 +28,7 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() - def strip_url(self, url, origin_only=False): + def strip_url_parsed(self, req_or_resp, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -41,9 +42,9 @@ class ReferrerPolicy(object): Set url's query to null. Return url. """ - if url is None or not url: + if req_or_resp.url is None or not req_or_resp.url: return None - parsed = urlsplit(url, allow_fragments=True) + parsed = urlparse_cached(req_or_resp) if parsed.scheme in self.NOREFERRER_SCHEMES: return None @@ -60,16 +61,61 @@ class ReferrerPolicy(object): if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): netloc = netloc.replace(':{p.port}'.format(p=parsed), '') - return urlunsplit(( + return ParseResult(parsed.scheme, + netloc, + '/' if origin_only else parsed.path, + '' if origin_only else parsed.params, + '' if origin_only else parsed.query, + '') + + def strip_url(self, url, origin_only=False): + """ + https://www.w3.org/TR/referrer-policy/#strip-url + + If url is null, return no referrer. + If url's scheme is a local scheme, then return no referrer. + Set url's username to the empty string. + Set url's password to null. + Set url's fragment to null. + If the origin-only flag is true, then: + Set url's path to null. + Set url's query to null. + Return url. + """ + if url is None or not url: + return None + parsed = urlparse(url, allow_fragments=True) + + if parsed.scheme in self.NOREFERRER_SCHEMES: + return None + + netloc = parsed.netloc + # strip username and password if present + if parsed.username or parsed.password: + netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') + + # strip standard protocol numbers + # Note: strictly speaking, standard port numbers should only be + # stripped when comparing origins + if parsed.port: + if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): + netloc = netloc.replace(':{p.port}'.format(p=parsed), '') + + return urlunparse(( parsed.scheme, netloc, '/' if origin_only else parsed.path, + '' if origin_only else parsed.params, '' if origin_only else parsed.query, '')) def origin(self, url): return self.strip_url(url, origin_only=True) + def origin_parsed(self, req_or_resp): + """Return (scheme, host, path) tuple for a request or response URL.""" + return tuple(self.strip_url_parsed(req_or_resp, origin_only=True)[:3]) + class NoReferrerPolicy(ReferrerPolicy): """ @@ -103,20 +149,17 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): name = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response, request): - target_url = request.url - - referrer_source = response.url - referrer_url = self.strip_url(referrer_source) - # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer: # # If environment is TLS-protected # and the origin of request's current URL is not an a priori authenticated URL, # then return no referrer. - if urlsplit(referrer_source).scheme in ('https', 'ftps') and \ - urlsplit(target_url).scheme in ('http',): + if urlparse_cached(response).scheme in ('https', 'ftps') and \ + urlparse_cached(request).scheme in ('http',): return None - return referrer_url + stripped = self.strip_url_parsed(response) + if stripped is not None: + return urlunparse(stripped) class SameOriginPolicy(ReferrerPolicy): @@ -132,12 +175,10 @@ class SameOriginPolicy(ReferrerPolicy): name = POLICY_SAME_ORIGIN def referrer(self, response, request): - target_url = request.url - referrer_source = response.url - if self.origin(referrer_source) == self.origin(target_url): - return self.strip_url(referrer_source) - else: - return None + if self.origin_parsed(response) == self.origin_parsed(request): + stripped = self.strip_url_parsed(response) + if stripped is not None: + return urlunparse(stripped) class OriginPolicy(ReferrerPolicy): From f2ee6be3bb311bb8f89ccd929bc0ebd8412d0183 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 18:33:31 +0200 Subject: [PATCH 018/109] Use urlparse_cached() for OriginPolicy --- scrapy/spidermiddlewares/referer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index cbbcd3e97..bf2a3c037 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -193,7 +193,9 @@ class OriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN def referrer(self, response, request): - return self.strip_url(response.url, origin_only=True) + stripped = self.strip_url_parsed(response, origin_only=True) + if stripped is not None: + return urlunparse(stripped) class OriginWhenCrossOriginPolicy(ReferrerPolicy): From 59cb884ace1cb1c3339f1d0f05895501dd4d0447 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 18:44:46 +0200 Subject: [PATCH 019/109] Use urlparse_cached() for OriginWhenCrossOriginPolicy --- scrapy/spidermiddlewares/referer.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index bf2a3c037..60f21ae84 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -193,9 +193,9 @@ class OriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN def referrer(self, response, request): - stripped = self.strip_url_parsed(response, origin_only=True) - if stripped is not None: - return urlunparse(stripped) + origin = self.strip_url_parsed(response, origin_only=True) + if origin is not None: + return urlunparse(origin) class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -212,14 +212,13 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response, request): - target_url = request.url - referrer_source = response.url - source_origin = self.origin(referrer_source) - if source_origin == self.origin(target_url): - return self.strip_url(referrer_source, origin_only=False) + origin = self.origin_parsed(response) + if origin == self.origin_parsed(request): + stripped = self.strip_url_parsed(response) + if stripped is not None: + return urlunparse(stripped) else: - return source_origin - + return urlunparse(origin + ('', '', '')) class UnsafeUrlPolicy(ReferrerPolicy): From f6a800fde67447d94480d4271de62f5e9a60bfb0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 18:55:35 +0200 Subject: [PATCH 020/109] Remove all non-cached urlparsing references --- scrapy/spidermiddlewares/referer.py | 71 ++++++----------------------- 1 file changed, 14 insertions(+), 57 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 60f21ae84..46351576f 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,7 +2,7 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ -from six.moves.urllib.parse import urlparse, urlunparse, ParseResult +from six.moves.urllib.parse import ParseResult, urlunparse from scrapy.http import Request from scrapy.exceptions import NotConfigured @@ -28,7 +28,7 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() - def strip_url_parsed(self, req_or_resp, origin_only=False): + def strip_url(self, req_or_resp, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -68,53 +68,9 @@ class ReferrerPolicy(object): '' if origin_only else parsed.query, '') - def strip_url(self, url, origin_only=False): - """ - https://www.w3.org/TR/referrer-policy/#strip-url - - If url is null, return no referrer. - If url's scheme is a local scheme, then return no referrer. - Set url's username to the empty string. - Set url's password to null. - Set url's fragment to null. - If the origin-only flag is true, then: - Set url's path to null. - Set url's query to null. - Return url. - """ - if url is None or not url: - return None - parsed = urlparse(url, allow_fragments=True) - - if parsed.scheme in self.NOREFERRER_SCHEMES: - return None - - netloc = parsed.netloc - # strip username and password if present - if parsed.username or parsed.password: - netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') - - # strip standard protocol numbers - # Note: strictly speaking, standard port numbers should only be - # stripped when comparing origins - if parsed.port: - if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): - netloc = netloc.replace(':{p.port}'.format(p=parsed), '') - - return urlunparse(( - parsed.scheme, - netloc, - '/' if origin_only else parsed.path, - '' if origin_only else parsed.params, - '' if origin_only else parsed.query, - '')) - - def origin(self, url): - return self.strip_url(url, origin_only=True) - - def origin_parsed(self, req_or_resp): + def origin(self, req_or_resp): """Return (scheme, host, path) tuple for a request or response URL.""" - return tuple(self.strip_url_parsed(req_or_resp, origin_only=True)[:3]) + return tuple(self.strip_url(req_or_resp, origin_only=True)[:3]) class NoReferrerPolicy(ReferrerPolicy): @@ -157,7 +113,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): if urlparse_cached(response).scheme in ('https', 'ftps') and \ urlparse_cached(request).scheme in ('http',): return None - stripped = self.strip_url_parsed(response) + stripped = self.strip_url(response) if stripped is not None: return urlunparse(stripped) @@ -175,8 +131,8 @@ class SameOriginPolicy(ReferrerPolicy): name = POLICY_SAME_ORIGIN def referrer(self, response, request): - if self.origin_parsed(response) == self.origin_parsed(request): - stripped = self.strip_url_parsed(response) + if self.origin(response) == self.origin(request): + stripped = self.strip_url(response) if stripped is not None: return urlunparse(stripped) @@ -193,7 +149,7 @@ class OriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN def referrer(self, response, request): - origin = self.strip_url_parsed(response, origin_only=True) + origin = self.strip_url(response, origin_only=True) if origin is not None: return urlunparse(origin) @@ -212,9 +168,9 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response, request): - origin = self.origin_parsed(response) - if origin == self.origin_parsed(request): - stripped = self.strip_url_parsed(response) + origin = self.origin(response) + if origin == self.origin(request): + stripped = self.strip_url(response) if stripped is not None: return urlunparse(stripped) else: @@ -237,8 +193,9 @@ class UnsafeUrlPolicy(ReferrerPolicy): name = POLICY_UNSAFE_URL def referrer(self, response, request): - referrer_source = response.url - return self.strip_url(referrer_source) + stripped = self.strip_url(response) + if stripped is not None: + return urlunparse(stripped) class LegacyPolicy(ReferrerPolicy): From f6205778f31dae3ac0f60528a7aa4ef3ef6a181a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 19:06:26 +0200 Subject: [PATCH 021/109] Refactor ReferrerPolicy methods --- scrapy/spidermiddlewares/referer.py | 30 ++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 46351576f..44a599433 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -28,6 +28,16 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() + def stripped_referrer(self, req_or_resp): + stripped = self.strip_url(req_or_resp) + if stripped is not None: + return urlunparse(stripped) + + def origin_referrer(self, req_or_resp): + stripped = self.strip_url(req_or_resp, origin_only=True) + if stripped is not None: + return urlunparse(stripped) + def strip_url(self, req_or_resp, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -113,9 +123,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): if urlparse_cached(response).scheme in ('https', 'ftps') and \ urlparse_cached(request).scheme in ('http',): return None - stripped = self.strip_url(response) - if stripped is not None: - return urlunparse(stripped) + return self.stripped_referrer(response) class SameOriginPolicy(ReferrerPolicy): @@ -132,9 +140,7 @@ class SameOriginPolicy(ReferrerPolicy): def referrer(self, response, request): if self.origin(response) == self.origin(request): - stripped = self.strip_url(response) - if stripped is not None: - return urlunparse(stripped) + return self.stripped_referrer(response) class OriginPolicy(ReferrerPolicy): @@ -149,9 +155,7 @@ class OriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN def referrer(self, response, request): - origin = self.strip_url(response, origin_only=True) - if origin is not None: - return urlunparse(origin) + return self.origin_referrer(response) class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -170,9 +174,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): def referrer(self, response, request): origin = self.origin(response) if origin == self.origin(request): - stripped = self.strip_url(response) - if stripped is not None: - return urlunparse(stripped) + return self.stripped_referrer(response) else: return urlunparse(origin + ('', '', '')) @@ -193,9 +195,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): name = POLICY_UNSAFE_URL def referrer(self, response, request): - stripped = self.strip_url(response) - if stripped is not None: - return urlunparse(stripped) + return self.stripped_referrer(response) class LegacyPolicy(ReferrerPolicy): From 842ce131aa666dce0db61aeee701c420050a5a9c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 11 Oct 2016 18:27:31 +0200 Subject: [PATCH 022/109] Make default referrer policy customizable via settings --- scrapy/settings/default_settings.py | 1 + scrapy/spidermiddlewares/referer.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a5931a3d5..15a134dd3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -234,6 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True +REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 44a599433..88041d8f0 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -8,6 +8,7 @@ from scrapy.http import Request from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_native_str from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import load_object LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) @@ -221,16 +222,27 @@ _policy_classes = {p.name: p for p in ( class RefererMiddleware(object): - def __init__(self, policy_class=DefaultReferrerPolicy): - self.default_policy = policy_class + def __init__(self, settings={}): + policy = settings.get('REFERER_POLICY') + if policy is not None: + try: + self.default_policy = load_object(policy) + except ValueError: + try: + self.default_policy = _policy_classes[policy] + except: + raise NotConfigured("Unknown referrer policy name %r" % policy) + else: + self.default_policy = DefaultReferrerPolicy @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured - return cls() + return cls(crawler.settings) def policy(self, response, request): + # policy set in request's meta dict takes precedence over default policy policy_name = request.meta.get('referrer_policy') if policy_name is None: policy_name = to_native_str( From e72b6e33611f61dbb358f1ff97c5c195a7db696f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 11 Oct 2016 18:27:56 +0200 Subject: [PATCH 023/109] Add tests for referrer policy via settings and via Request meta --- tests/test_spidermiddleware_referer.py | 501 ++++++++++++++----------- 1 file changed, 292 insertions(+), 209 deletions(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 8458fe90b..b724d7999 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,233 +1,316 @@ from unittest import TestCase +from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request +from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.spidermiddlewares.referer import RefererMiddleware, \ POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ - POLICY_UNSAFE_URL + POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \ + DefaultReferrerPolicy, \ + NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ + OriginWhenCrossOriginPolicy, OriginPolicy, \ + SameOriginPolicy, UnsafeUrlPolicy class TestRefererMiddleware(TestCase): + req_meta = {} + resp_headers = {} + settings = {} + scenarii = [ + ('http://scrapytest.org', 'http://scrapytest.org/', b'http://scrapytest.org'), + ] + def setUp(self): self.spider = Spider('foo') - self.mw = RefererMiddleware() + settings = Settings(self.settings) + self.mw = RefererMiddleware(settings) - def test_process_spider_output(self): - res = Response('http://scrapytest.org') - reqs = [Request('http://scrapytest.org/')] + def get_request(self, target): + return Request(target, meta=self.req_meta) - out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://scrapytest.org') + def get_response(self, origin): + return Response(origin, headers=self.resp_headers) - def test_policy_default(self): - """ - Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + def test(self): - with some additional filtering of s3:// - """ - for origin, target, referrer in [ - ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), - ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), - ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), - ('https://example.com/', 'http://scrapy.org/', None), - - # no credentials leak - ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), - - # no referrer leak for local schemes - ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), - ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), - - # no referrer leak for s3 origins - ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), - ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), - ]: - response = Response(origin) - request = Request(target) + for origin, target, referrer in self.scenarii: + response = self.get_response(origin) + request = self.get_request(target) out = list(self.mw.process_spider_output(response, [request], self.spider)) self.assertEquals(out[0].headers.get('Referer'), referrer) - def test_policy_no_referrer(self): - for origin, target, referrer in [ - ('https://example.com/page.html', 'https://example.com/', None), - ('http://www.example.com/', 'https://scrapy.org/', None), - ('http://www.example.com/', 'http://scrapy.org/', None), - ('https://www.example.com/', 'http://scrapy.org/', None), - ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), +class MixinDefault(object): + """ + Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + + with some additional filtering of s3:// + """ + scenarii = [ + ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), + ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), + ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), + ('https://example.com/', 'http://scrapy.org/', None), + + # no credentials leak + ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), + + # no referrer leak for local schemes + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + + # no referrer leak for s3 origins + ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), + ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), + ] + + +class MixinNoReferrer(object): + scenarii = [ + ('https://example.com/page.html', 'https://example.com/', None), + ('http://www.example.com/', 'https://scrapy.org/', None), + ('http://www.example.com/', 'http://scrapy.org/', None), + ('https://www.example.com/', 'http://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + ] + + +class MixinNoReferrerWhenDowngrade(object): + scenarii = [ + # TLS to TLS: send non-empty referrer + ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), + ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), + ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), + + # TLS to non-TLS: do not send referrer + ('https://example.com/page.html', 'http://not.example.com/', None), + ('https://example.com/page.html', 'http://scrapy.org/', None), + ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), + + # non-TLS to TLS or non-TLS: send referrer + ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), + ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), + ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), + ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ] + + +class MixinSameOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: do NOT send referrer + ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), + + # Different port: do NOT send referrer + ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), + ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), + ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), + + # Different protocols: do NOT send refferer + ('https://example.com/page.html', 'http://example.com/not-page.html', None), + ('https://example.com/page.html', 'http://not.example.com/', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ] + + +class MixinOrigin(object): + scenarii = [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ] + + +class MixinOriginWhenCrossOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # Different protocols: send origin as referrer + ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), + ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + # TLS to non-TLS downgrade: send origin + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), + ] + + +class MixinUnsafeUrl(object): + scenarii = [ + # TLS to TLS: send referrer + ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), + ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), + ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), + ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), + + # TLS to non-TLS: send referrer (yes, it's unsafe) + ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), + ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), + ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), + + # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) + ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), + ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), + ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), + ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), + ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), + ] + + +class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware): + pass + + +# --- Tests using settings to set policy using class path +class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} + + +class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + + +class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + + +class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} + + +class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + + +class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + + +# --- Tests using Request meta dict to set policy +class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} + + +class TestRefererMiddlewareNoReferrer(MixinNoReferrer, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_NO_REFERRER} + + +class TestRefererMiddlewareNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE} + + +class TestRefererMiddlewareSameOrigin(MixinSameOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_SAME_ORIGIN} + + +class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_ORIGIN} + + +class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} + + +class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + + +class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + +class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + req_meta = {'referrer_policy': POLICY_NO_REFERRER} + + +class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + +class TestRefererMiddlewareSettingsPolicyByName(TestCase): + + def test_valid_name(self): + for s, p in [ + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER}) + settings = Settings({'REFERER_POLICY': s}) + mw = RefererMiddleware(settings) + self.assertEquals(mw.default_policy, p) - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_no_referrer_when_downgrade(self): - - for origin, target, referrer in [ - # TLS to TLS: send non-empty referrer - ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), - ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), - ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), - ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), - ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), - - # TLS to non-TLS: do not send referrer - ('https://example.com/page.html', 'http://not.example.com/', None), - ('https://example.com/page.html', 'http://scrapy.org/', None), - ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), - - # non-TLS to TLS or non-TLS: send referrer - ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), - ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), - ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), - ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), - ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), - ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), - ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), - - # test for user/password stripping - ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_same_origin(self): - - for origin, target, referrer in [ - # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), - - # Different host: do NOT send referrer - ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), - ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), - ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), - - # Different port: do NOT send referrer - ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), - ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), - ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), - - # Different protocols: do NOT send refferer - ('https://example.com/page.html', 'http://example.com/not-page.html', None), - ('https://example.com/page.html', 'http://not.example.com/', None), - ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), - ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), - ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), - - # test for user/password stripping - ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_SAME_ORIGIN}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_origin(self): - - for origin, target, referrer in [ - # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), - ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), - ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), - ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), - - # test for user/password stripping - ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_ORIGIN}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_origin_when_cross_origin(self): - - for origin, target, referrer in [ - # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), - - # Different host: send origin as referrer - ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), - ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), - ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), - # exact match required - ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), - - # Different port: send origin as referrer - ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), - ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), - - # Different protocols: send origin as referrer - ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), - ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), - ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), - ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), - ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), - - # test for user/password stripping - ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), - # TLS to non-TLS downgrade: send origin - ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_unsafe_url(self): - - for origin, target, referrer in [ - # TLS to TLS: send referrer - ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), - ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), - ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), - ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), - ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), - ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), - - # TLS to non-TLS: send referrer (yes, it's unsafe) - ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), - ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), - ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), - - # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) - ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), - ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), - ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), - ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), - ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), - ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), - ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), - ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), - - # test for user/password stripping - ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), - ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_UNSAFE_URL}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) + def test_invalid_name(self): + settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) + with self.assertRaises(NotConfigured): + mw = RefererMiddleware(settings) From 0344f57fefc0877bf9048a084002fc719335e31c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 11 Oct 2016 19:53:15 +0200 Subject: [PATCH 024/109] Support case-insensitive policy names in settings --- scrapy/spidermiddlewares/referer.py | 2 +- tests/test_spidermiddleware_referer.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 88041d8f0..deda7b284 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -229,7 +229,7 @@ class RefererMiddleware(object): self.default_policy = load_object(policy) except ValueError: try: - self.default_policy = _policy_classes[policy] + self.default_policy = _policy_classes[policy.lower()] except: raise NotConfigured("Unknown referrer policy name %r" % policy) else: diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index b724d7999..b1ab366a7 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -310,6 +310,20 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): mw = RefererMiddleware(settings) self.assertEquals(mw.default_policy, p) + def test_valid_name_casevariants(self): + for s, p in [ + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), + ]: + settings = Settings({'REFERER_POLICY': s.upper()}) + mw = RefererMiddleware(settings) + self.assertEquals(mw.default_policy, p) + def test_invalid_name(self): settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) with self.assertRaises(NotConfigured): From ec8b4c1a9bfd004960d730b41078b9f0633627d8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 11 Oct 2016 20:00:34 +0200 Subject: [PATCH 025/109] Change __init__ default "settings" arg handling --- scrapy/spidermiddlewares/referer.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index deda7b284..2a3790bde 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -222,18 +222,18 @@ _policy_classes = {p.name: p for p in ( class RefererMiddleware(object): - def __init__(self, settings={}): - policy = settings.get('REFERER_POLICY') - if policy is not None: - try: - self.default_policy = load_object(policy) - except ValueError: + def __init__(self, settings=None): + self.default_policy = DefaultReferrerPolicy + if settings is not None: + policy = settings.get('REFERER_POLICY') + if policy is not None: try: - self.default_policy = _policy_classes[policy.lower()] - except: - raise NotConfigured("Unknown referrer policy name %r" % policy) - else: - self.default_policy = DefaultReferrerPolicy + self.default_policy = load_object(policy) + except ValueError: + try: + self.default_policy = _policy_classes[policy.lower()] + except: + raise NotConfigured("Unknown referrer policy name %r" % policy) @classmethod def from_crawler(cls, crawler): From e50e670eff2fe1b109f6b5dd026c4c706a93585c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 12 Oct 2016 16:16:53 +0200 Subject: [PATCH 026/109] Add test for custom referrer policy via settings --- scrapy/spidermiddlewares/referer.py | 4 ++++ tests/test_spidermiddleware_referer.py | 30 ++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 2a3790bde..0bba63cb7 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -227,9 +227,12 @@ class RefererMiddleware(object): if settings is not None: policy = settings.get('REFERER_POLICY') if policy is not None: + # expect a string for the path to the policy class try: self.default_policy = load_object(policy) except ValueError: + # otherwise try to interpret the string as standard + # https://www.w3.org/TR/referrer-policy/#referrer-policies try: self.default_policy = _policy_classes[policy.lower()] except: @@ -239,6 +242,7 @@ class RefererMiddleware(object): def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured + return cls(crawler.settings) def policy(self, response, request): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index b1ab366a7..cfc4b5296 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -11,7 +11,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \ DefaultReferrerPolicy, \ NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ OriginWhenCrossOriginPolicy, OriginPolicy, \ - SameOriginPolicy, UnsafeUrlPolicy + SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy class TestRefererMiddleware(TestCase): @@ -249,6 +249,33 @@ class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddlewa settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} +class CustomPythonOrgPolicy(ReferrerPolicy): + """ + A dummy policy that returns referrer as http(s)://python.org + depending on the scheme of the target URL. + """ + def referrer(self, response, request): + from scrapy.utils.httpobj import urlparse_cached + + scheme = urlparse_cached(request).scheme + if scheme == 'https': + return b'https://python.org/' + elif scheme == 'http': + return b'http://python.org/' + + +class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware): + settings = {'REFERER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} + scenarii = [ + ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), + ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), + ('http://example.com/', 'https://scrapy.org/', b'https://python.org/'), + ('https://example.com/', 'http://scrapy.org/', b'http://python.org/'), + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', b'https://python.org/'), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', b'http://python.org/'), + + ] + # --- Tests using Request meta dict to set policy class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} @@ -278,7 +305,6 @@ class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_UNSAFE_URL} - class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} From d3d4d66ce8e5d01aa1fd6013a6d63337bb931460 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 12 Oct 2016 16:30:25 +0200 Subject: [PATCH 027/109] Add tests for referrer-policy set in response HTTP headers --- tests/test_spidermiddleware_referer.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index cfc4b5296..9555817d9 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -354,3 +354,16 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) with self.assertRaises(NotConfigured): mw = RefererMiddleware(settings) + + +class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} + +class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} + +class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} From 285d5bc03a7ebe8eaa558a5c24ff0693353c3e87 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 12 Oct 2016 17:34:12 +0200 Subject: [PATCH 028/109] Patch "Referer" header on HTTP redirects if necessary --- scrapy/spidermiddlewares/referer.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 0bba63cb7..01f1fdf85 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -4,8 +4,9 @@ originated it. """ from six.moves.urllib.parse import ParseResult, urlunparse -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured +from scrapy import signals from scrapy.utils.python import to_native_str from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object @@ -242,8 +243,9 @@ class RefererMiddleware(object): def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured - - return cls(crawler.settings) + mw = cls(crawler.settings) + crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled) + return mw def policy(self, response, request): # policy set in request's meta dict takes precedence over default policy @@ -264,3 +266,18 @@ class RefererMiddleware(object): return r return (_set_referer(r) for r in result or ()) + def request_scheduled(self, request, spider): + # check redirected request to patch "Referer" header if necessary + redirected_urls = request.meta.get('redirect_urls', []) + if redirected_urls: + request_referrer = request.headers.get('Referer') + # we don't patch the referrer value if there is none + if request_referrer is not None: + faked_response = Response(redirected_urls[0]) + policy_referrer = self.policy(faked_response, + request).referrer(faked_response, request) + if policy_referrer != request_referrer: + if policy_referrer is None: + request.headers.pop('Referer') + else: + request.headers['Referer'] = policy_referrer From c9c59db489575131d573e130015fd1fb6b133882 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 12 Oct 2016 18:29:47 +0200 Subject: [PATCH 029/109] Update documentation about REFERER_POLICY setting --- docs/topics/spider-middleware.rst | 38 ++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8360827e8..a9d3d4568 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -95,7 +95,7 @@ following methods: it has processed the response. :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` + :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` objects. :param response: the response which generated this output from the @@ -328,6 +328,42 @@ Default: ``True`` Whether to enable referer middleware. +.. setting:: REFERER_POLICY + +REFERER_POLICY +^^^^^^^^^^^^^^ + +.. versionadded:: 1.3 + +Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` + +`Referrer Policy`_ to apply when populating Request "Referer" header. + +This setting accepts: + +- a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass, + either a custom one or one of the built-in ones + (see ``scrapy.spidermiddlewares.referer``), +- or one of the standard W3C-defined string values, i.e. ``"no-referrer"``, + ``"no-referrer-when-downgrade"``, ``"same-origin"``, ``"origin"``, + ``"origin-when-cross-origin"`` or ``"unsafe-url"``. + (It can also be the non-standard value ``"scrapy-default"`` to use + Scrapy's default referrer policy.) + +Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, +with the addition that "Referrer" is not sent if the parent request was +using ``file://`` or ``s3://`` scheme. + +.. warning:: + By default, Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_, + will send a non-empty "Referer" header from any ``https://`` to any ``https://`` URL, + even if the domain is different. + ``same-origin`` may be a better choice if you want to remove referrer + information for cross-domain requests. + +.. _Referrer Policy: https://www.w3.org/TR/referrer-policy +.. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + UrlLengthMiddleware ------------------- From 5dd7311cd48e676147138746229d7ab2b429b8a9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 19 Oct 2016 14:45:33 +0200 Subject: [PATCH 030/109] Move URL credentials stripping to a helper function --- scrapy/spidermiddlewares/referer.py | 43 +++----------- scrapy/utils/url.py | 34 ++++++++++- tests/test_utils_url.py | 92 ++++++++++++++++++++++++++++- 3 files changed, 133 insertions(+), 36 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 01f1fdf85..e40e798b8 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,14 +2,13 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ -from six.moves.urllib.parse import ParseResult, urlunparse - from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals from scrapy.utils.python import to_native_str from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object +from scrapy.utils.url import strip_url_credentials LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) @@ -31,14 +30,10 @@ class ReferrerPolicy(object): raise NotImplementedError() def stripped_referrer(self, req_or_resp): - stripped = self.strip_url(req_or_resp) - if stripped is not None: - return urlunparse(stripped) + return self.strip_url(req_or_resp) def origin_referrer(self, req_or_resp): - stripped = self.strip_url(req_or_resp, origin_only=True) - if stripped is not None: - return urlunparse(stripped) + return self.strip_url(req_or_resp, origin_only=True) def strip_url(self, req_or_resp, origin_only=False): """ @@ -56,33 +51,13 @@ class ReferrerPolicy(object): """ if req_or_resp.url is None or not req_or_resp.url: return None - parsed = urlparse_cached(req_or_resp) - - if parsed.scheme in self.NOREFERRER_SCHEMES: - return None - - netloc = parsed.netloc - # strip username and password if present - if parsed.username or parsed.password: - netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') - - # strip standard protocol numbers - # Note: strictly speaking, standard port numbers should only be - # stripped when comparing origins - if parsed.port: - if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): - netloc = netloc.replace(':{p.port}'.format(p=parsed), '') - - return ParseResult(parsed.scheme, - netloc, - '/' if origin_only else parsed.path, - '' if origin_only else parsed.params, - '' if origin_only else parsed.query, - '') + parsed_url = urlparse_cached(req_or_resp) + if parsed_url.scheme not in self.NOREFERRER_SCHEMES: + return strip_url_credentials(parsed_url, origin_only=origin_only) def origin(self, req_or_resp): - """Return (scheme, host, path) tuple for a request or response URL.""" - return tuple(self.strip_url(req_or_resp, origin_only=True)[:3]) + """Return serialized origin (scheme, host, path) for a request or response URL.""" + return self.strip_url(req_or_resp, origin_only=True) class NoReferrerPolicy(ReferrerPolicy): @@ -178,7 +153,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): if origin == self.origin(request): return self.stripped_referrer(response) else: - return urlunparse(origin + ('', '', '')) + return origin class UnsafeUrlPolicy(ReferrerPolicy): diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index dc1cce4ac..f3ccfb0e8 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead. """ import posixpath import re -from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse) +from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse) # scrapy.utils.url was moved to w3lib.url and import * ensures this # move doesn't break old code @@ -103,3 +103,35 @@ def guess_scheme(url): return any_to_uri(url) else: return add_http_if_no_scheme(url) + + +def strip_url_credentials(url, origin_only=False, keep_fragments=False): + + if url is None: + return None + + if not isinstance(url, ParseResult): + parsed_url = urlparse(url) + else: + parsed_url = url + + netloc = parsed_url.netloc + # strip username and password if present + if parsed_url.username or parsed_url.password: + netloc = netloc.split('@')[-1] + + # strip standard protocol numbers + # Note: strictly speaking, standard port numbers should only be + # stripped when comparing origins + if parsed_url.port: + if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443)): + netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '') + + return urlunparse(( + parsed_url.scheme, + netloc, + '/' if origin_only else parsed_url.path, + '' if origin_only else parsed_url.params, + '' if origin_only else parsed_url.query, + '' if not keep_fragments else parsed_url.fragment + )) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index f46d1d927..f1a5c3196 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -6,7 +6,8 @@ from six.moves.urllib.parse import urlparse from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, - add_http_if_no_scheme, guess_scheme, parse_url) + add_http_if_no_scheme, guess_scheme, + parse_url, strip_url_credentials) __doctests__ = ['scrapy.utils.url'] @@ -241,5 +242,94 @@ for k, args in enumerate ([ setattr (GuessSchemeTest, t_method.__name__, t_method) +class StripUrlCredentials(unittest.TestCase): + + def test_noop(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com/index.html'), + 'http://www.example.com/index.html') + + def test_noop_query_string(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com/index.html?somekey=somevalue'), + 'http://www.example.com/index.html?somekey=somevalue') + + def test_fragments(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com/index.html?somekey=somevalue#section', keep_fragments=True), + 'http://www.example.com/index.html?somekey=somevalue#section') + + def test_noop_trailing_path(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com/'), + 'http://www.example.com/') + + def test_noop_trailing_path2(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com'), + 'http://www.example.com') + + def test_trailing_path_origin(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com', origin_only=True), + 'http://www.example.com/') + + def test_username(self): + # username is stripped (and fragment too) + self.assertEqual(strip_url_credentials( + 'http://username@www.example.com/index.html?somekey=somevalue#section'), + 'http://www.example.com/index.html?somekey=somevalue') + + def test_username_empty_pass(self): + # same as above + self.assertEqual(strip_url_credentials( + 'https://username:@www.example.com/index.html?somekey=somevalue#section'), + 'https://www.example.com/index.html?somekey=somevalue') + + def test_username_password(self): + self.assertEqual(strip_url_credentials( + 'ftp://username:password@www.example.com/index.html?somekey=somevalue#section'), + 'ftp://www.example.com/index.html?somekey=somevalue') + + def test_default_http_port(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com:80/index.html'), + 'http://www.example.com/index.html') + + def test_non_default_http_port(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com:8080/index.html'), + 'http://www.example.com:8080/index.html') + + def test_default_https_port(self): + self.assertEqual(strip_url_credentials( + 'https://username:password@www.example.com:443/index.html'), + 'https://www.example.com/index.html') + + def test_non_default_https_port(self): + self.assertEqual(strip_url_credentials( + 'https://username:password@www.example.com:442/index.html'), + 'https://www.example.com:442/index.html') + + def test_origin_only(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com/index.html', origin_only=True), + 'http://www.example.com/') + + def test_default_http_port_origin_only(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com:80/index.html', origin_only=True), + 'http://www.example.com/') + + def test_non_default_http_port_origin_only(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com:8008/index.html', origin_only=True), + 'http://www.example.com:8008/') + + def test_default_https_port_origin_only(self): + self.assertEqual(strip_url_credentials( + 'https://username:password@www.example.com:443/index.html', origin_only=True), + 'https://www.example.com/') + if __name__ == "__main__": unittest.main() From 8864d0e8c19d0ddb52175be1952a4013b13cd86f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 24 Oct 2016 18:21:49 +0200 Subject: [PATCH 031/109] Rename helper function to strip_url() + add more tests --- scrapy/spidermiddlewares/referer.py | 8 +- scrapy/utils/url.py | 30 +++-- tests/test_utils_url.py | 190 +++++++++++++++++++--------- 3 files changed, 150 insertions(+), 78 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index e40e798b8..24e5eac40 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -8,7 +8,7 @@ from scrapy import signals from scrapy.utils.python import to_native_str from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object -from scrapy.utils.url import strip_url_credentials +from scrapy.utils.url import strip_url LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) @@ -53,7 +53,11 @@ class ReferrerPolicy(object): return None parsed_url = urlparse_cached(req_or_resp) if parsed_url.scheme not in self.NOREFERRER_SCHEMES: - return strip_url_credentials(parsed_url, origin_only=origin_only) + return strip_url(parsed_url, + strip_credentials=True, + strip_fragment=True, + strip_default_port=True, + origin_only=origin_only) def origin(self, req_or_resp): """Return serialized origin (scheme, host, path) for a request or response URL.""" diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index f3ccfb0e8..9864f353d 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -105,33 +105,37 @@ def guess_scheme(url): return add_http_if_no_scheme(url) -def strip_url_credentials(url, origin_only=False, keep_fragments=False): +def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True): + + """Strip URL string from some of its components: + + - `strip_credentials` removes "user:password@" + - `strip_default_port` removes ":80" (resp. ":443", ":21") + from http:// (resp. https://, ftp://) URLs + - `origin_only` replaces path component with "/", also dropping + query and fragment components ; it also strips credentials + - `strip_fragment` drops any #fragment component + """ if url is None: return None - if not isinstance(url, ParseResult): parsed_url = urlparse(url) else: parsed_url = url - netloc = parsed_url.netloc - # strip username and password if present - if parsed_url.username or parsed_url.password: + if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password): netloc = netloc.split('@')[-1] - - # strip standard protocol numbers - # Note: strictly speaking, standard port numbers should only be - # stripped when comparing origins - if parsed_url.port: - if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443)): + if strip_default_port and parsed_url.port: + if (parsed_url.scheme, parsed_url.port) in (('http', 80), + ('https', 443), + ('ftp', 21)): netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '') - return urlunparse(( parsed_url.scheme, netloc, '/' if origin_only else parsed_url.path, '' if origin_only else parsed_url.params, '' if origin_only else parsed_url.query, - '' if not keep_fragments else parsed_url.fragment + '' if strip_fragment else parsed_url.fragment )) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index f1a5c3196..1f9845d82 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -7,7 +7,7 @@ from six.moves.urllib.parse import urlparse from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, add_http_if_no_scheme, guess_scheme, - parse_url, strip_url_credentials) + parse_url, strip_url) __doctests__ = ['scrapy.utils.url'] @@ -242,94 +242,158 @@ for k, args in enumerate ([ setattr (GuessSchemeTest, t_method.__name__, t_method) -class StripUrlCredentials(unittest.TestCase): +class StripUrl(unittest.TestCase): def test_noop(self): - self.assertEqual(strip_url_credentials( + self.assertEqual(strip_url( 'http://www.example.com/index.html'), 'http://www.example.com/index.html') def test_noop_query_string(self): - self.assertEqual(strip_url_credentials( + self.assertEqual(strip_url( 'http://www.example.com/index.html?somekey=somevalue'), 'http://www.example.com/index.html?somekey=somevalue') def test_fragments(self): - self.assertEqual(strip_url_credentials( - 'http://www.example.com/index.html?somekey=somevalue#section', keep_fragments=True), + self.assertEqual(strip_url( + 'http://www.example.com/index.html?somekey=somevalue#section', strip_fragment=False), 'http://www.example.com/index.html?somekey=somevalue#section') - def test_noop_trailing_path(self): - self.assertEqual(strip_url_credentials( - 'http://www.example.com/'), - 'http://www.example.com/') + def test_path(self): + for input_url, origin, output_url in [ + ('http://www.example.com/', + False, + 'http://www.example.com/'), - def test_noop_trailing_path2(self): - self.assertEqual(strip_url_credentials( - 'http://www.example.com'), - 'http://www.example.com') + ('http://www.example.com', + False, + 'http://www.example.com'), - def test_trailing_path_origin(self): - self.assertEqual(strip_url_credentials( - 'http://www.example.com', origin_only=True), - 'http://www.example.com/') + ('http://www.example.com', + True, + 'http://www.example.com/'), + ]: + self.assertEqual(strip_url(input_url, origin_only=origin), output_url) + self.assertEqual(strip_url(urlparse(input_url), origin_only=origin), output_url) - def test_username(self): - # username is stripped (and fragment too) - self.assertEqual(strip_url_credentials( - 'http://username@www.example.com/index.html?somekey=somevalue#section'), - 'http://www.example.com/index.html?somekey=somevalue') + def test_credentials(self): + for i, o in [ + ('http://username@www.example.com/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), - def test_username_empty_pass(self): - # same as above - self.assertEqual(strip_url_credentials( - 'https://username:@www.example.com/index.html?somekey=somevalue#section'), - 'https://www.example.com/index.html?somekey=somevalue') + ('https://username:@www.example.com/index.html?somekey=somevalue#section', + 'https://www.example.com/index.html?somekey=somevalue'), - def test_username_password(self): - self.assertEqual(strip_url_credentials( - 'ftp://username:password@www.example.com/index.html?somekey=somevalue#section'), - 'ftp://www.example.com/index.html?somekey=somevalue') + ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section', + 'ftp://www.example.com/index.html?somekey=somevalue'), + ]: + self.assertEqual(strip_url(i, strip_credentials=True), o) + self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) - def test_default_http_port(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com:80/index.html'), - 'http://www.example.com/index.html') + def test_default_ports_creds_off(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), - def test_non_default_http_port(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com:8080/index.html'), - 'http://www.example.com:8080/index.html') + ('http://username:password@www.example.com:8080/index.html#section', + 'http://www.example.com:8080/index.html'), - def test_default_https_port(self): - self.assertEqual(strip_url_credentials( - 'https://username:password@www.example.com:443/index.html'), - 'https://www.example.com/index.html') + ('http://username:password@www.example.com:443/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov'), - def test_non_default_https_port(self): - self.assertEqual(strip_url_credentials( - 'https://username:password@www.example.com:442/index.html'), - 'https://www.example.com:442/index.html') + ('https://username:password@www.example.com:443/index.html', + 'https://www.example.com/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://www.example.com/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i), o) + self.assertEqual(strip_url(urlparse(i)), o) + + def test_default_ports(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html', + 'http://username:password@www.example.com/index.html'), + + ('http://username:password@www.example.com:8080/index.html', + 'http://username:password@www.example.com:8080/index.html'), + + ('http://username:password@www.example.com:443/index.html', + 'http://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:443/index.html', + 'https://username:password@www.example.com/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://username:password@www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://username:password@www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://username:password@www.example.com/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://username:password@www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o) + self.assertEqual(strip_url(urlparse(i), strip_default_port=True, strip_credentials=False), o) + + def test_default_ports_keep(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov'), + + ('http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov'), + + ('http://username:password@www.example.com:443/index.html', + 'http://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:443/index.html', + 'https://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://username:password@www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://username:password@www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://username:password@www.example.com:21/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://username:password@www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o) + self.assertEqual(strip_url(urlparse(i), strip_default_port=False, strip_credentials=False), o) def test_origin_only(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com/index.html', origin_only=True), - 'http://www.example.com/') + for i, o in [ + ('http://username:password@www.example.com/index.html', + 'http://www.example.com/'), - def test_default_http_port_origin_only(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com:80/index.html', origin_only=True), - 'http://www.example.com/') + ('http://username:password@www.example.com:80/foo/bar?query=value#somefrag', + 'http://www.example.com/'), - def test_non_default_http_port_origin_only(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com:8008/index.html', origin_only=True), - 'http://www.example.com:8008/') + ('http://username:password@www.example.com:8008/foo/bar?query=value#somefrag', + 'http://www.example.com:8008/'), + + ('https://username:password@www.example.com:443/index.html', + 'https://www.example.com/'), + ]: + self.assertEqual(strip_url(i, origin_only=True), o) + self.assertEqual(strip_url(urlparse(i), origin_only=True), o) - def test_default_https_port_origin_only(self): - self.assertEqual(strip_url_credentials( - 'https://username:password@www.example.com:443/index.html', origin_only=True), - 'https://www.example.com/') if __name__ == "__main__": unittest.main() From 0a0b60a59f75ef1b4976b811011bd50b78f9cec8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 26 Oct 2016 12:41:00 +0200 Subject: [PATCH 032/109] Add tests for stripping userinfo with percent-encoded delimiters --- tests/test_utils_url.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 1f9845d82..9182d0fda 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -290,6 +290,26 @@ class StripUrl(unittest.TestCase): self.assertEqual(strip_url(i, strip_credentials=True), o) self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) + def test_credentials_encoded_delims(self): + for i, o in [ + # user: "username@" + # password: none + ('http://username%40@www.example.com/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), + + # user: "username:pass" + # password: "" + ('https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section', + 'https://www.example.com/index.html?somekey=somevalue'), + + # user: "me" + # password: "user@domain.com" + ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section', + 'ftp://www.example.com/index.html?somekey=somevalue'), + ]: + self.assertEqual(strip_url(i, strip_credentials=True), o) + self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) + def test_default_ports_creds_off(self): for i, o in [ ('http://username:password@www.example.com:80/index.html?somekey=somevalue#section', From c808a97c74718f72f53a3c00eca77f60c26fb6ba Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 12 Jan 2017 18:22:18 +0100 Subject: [PATCH 033/109] Add new "strict-" policies --- scrapy/spidermiddlewares/referer.py | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 24e5eac40..5d2a0e7b1 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -17,7 +17,9 @@ POLICY_NO_REFERRER = "no-referrer" POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade" POLICY_SAME_ORIGIN = "same-origin" POLICY_ORIGIN = "origin" +POLICY_STRICT_ORIGIN = "strict-origin" POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin" +POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN = "strict-origin-when-cross-origin" POLICY_UNSAFE_URL = "unsafe-url" POLICY_SCRAPY_DEFAULT = "scrapy-default" @@ -139,6 +141,26 @@ class OriginPolicy(ReferrerPolicy): return self.origin_referrer(response) +class StrictOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin + + The "strict-origin" policy sends the ASCII serialization + of the origin of the request client when making requests: + - from a TLS-protected environment settings object to a potentially trustworthy URL, and + - from non-TLS-protected environment settings objects to any origin. + + Requests from TLS-protected request clients to non- potentially trustworthy URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = POLICY_STRICT_ORIGIN + + def referrer(self, response, request): + if urlparse_cached(response).scheme == urlparse_cached(request).scheme: + return self.origin_referrer(response) + + class OriginWhenCrossOriginPolicy(ReferrerPolicy): """ https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin @@ -160,6 +182,33 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): return origin +class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin + + The "strict-origin-when-cross-origin" policy specifies that a full URL, + stripped for use as a referrer, is sent as referrer information + when making same-origin requests from a particular request client, + and only the ASCII serialization of the origin of the request client + when making cross-origin requests: + + - from a TLS-protected environment settings object to a potentially trustworthy URL, and + - from non-TLS-protected environment settings objects to any origin. + + Requests from TLS-protected clients to non- potentially trustworthy URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN + + def referrer(self, response, request): + origin = self.origin(response) + if origin == self.origin(request): + return self.stripped_referrer(response) + else: + return origin + + class UnsafeUrlPolicy(ReferrerPolicy): """ https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url From 5cef67ae75dca229bd1b1f3ac61f52309d1ca5a4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 14:18:33 +0100 Subject: [PATCH 034/109] Update Referrer tests for "strict-" policies --- tests/test_spidermiddleware_referer.py | 75 +++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 9555817d9..df20dfbb9 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -8,6 +8,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \ POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \ + POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, \ DefaultReferrerPolicy, \ NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ OriginWhenCrossOriginPolicy, OriginPolicy, \ @@ -39,7 +40,6 @@ class TestRefererMiddleware(TestCase): for origin, target, referrer in self.scenarii: response = self.get_response(origin) request = self.get_request(target) - out = list(self.mw.process_spider_output(response, [request], self.spider)) self.assertEquals(out[0].headers.get('Referer'), referrer) @@ -154,6 +154,25 @@ class MixinOrigin(object): ] +class MixinStrictOrigin(object): + scenarii = [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # downgrade: send nothing + ('https://example.com/page.html', 'http://scrapy.org', None), + + # upgrade: send origin + ('http://example.com/page.html', 'https://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://user:password@example.com/page.html', 'http://scrapy.org', None), + ] + + class MixinOriginWhenCrossOrigin(object): scenarii = [ # Same origin (protocol, host, port): send referrer @@ -189,6 +208,44 @@ class MixinOriginWhenCrossOrigin(object): ] +class MixinStrictOriginWhenCrossOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # downgrade + ('https://example4.com/page.html', 'http://example4.com/not-page.html', None), + ('https://example4.com/page.html', 'http://not.example4.com/', None), + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', None), + + # Different protocols: send origin as referrer + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + + # TLS to non-TLS downgrade: send nothing + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None), + ] + + class MixinUnsafeUrl(object): scenarii = [ # TLS to TLS: send referrer @@ -241,10 +298,18 @@ class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} +class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} + + class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} +class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} + + class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} @@ -297,10 +362,18 @@ class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_ORIGIN} +class TestRefererMiddlewareSrictOrigin(MixinStrictOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN} + + class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} +class TestRefererMiddlewareStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} + + class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_UNSAFE_URL} From 77aec5a79681128c0608e65bc77e317252172d98 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 14:19:19 +0100 Subject: [PATCH 035/109] Fix implementation --- scrapy/spidermiddlewares/referer.py | 40 ++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 5d2a0e7b1..a6316cd0c 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -31,13 +31,13 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() - def stripped_referrer(self, req_or_resp): - return self.strip_url(req_or_resp) + def stripped_referrer(self, r): + return self.strip_url(r) - def origin_referrer(self, req_or_resp): - return self.strip_url(req_or_resp, origin_only=True) + def origin_referrer(self, r): + return self.strip_url(r, origin_only=True) - def strip_url(self, req_or_resp, origin_only=False): + def strip_url(self, r, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -51,9 +51,9 @@ class ReferrerPolicy(object): Set url's query to null. Return url. """ - if req_or_resp.url is None or not req_or_resp.url: + if r is None or not r.url: return None - parsed_url = urlparse_cached(req_or_resp) + parsed_url = urlparse_cached(r) if parsed_url.scheme not in self.NOREFERRER_SCHEMES: return strip_url(parsed_url, strip_credentials=True, @@ -61,9 +61,19 @@ class ReferrerPolicy(object): strip_default_port=True, origin_only=origin_only) - def origin(self, req_or_resp): + def origin(self, r): """Return serialized origin (scheme, host, path) for a request or response URL.""" - return self.strip_url(req_or_resp, origin_only=True) + return self.strip_url(r, origin_only=True) + + def potentially_trustworthy(self, r): + # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy + parsed_url = urlparse_cached(r) + if parsed_url.scheme in ('data',): + return False + return self.tls_protected(r) + + def tls_protected(self, r): + return urlparse_cached(r).scheme in ('https', 'ftps') class NoReferrerPolicy(ReferrerPolicy): @@ -157,7 +167,9 @@ class StrictOriginPolicy(ReferrerPolicy): name = POLICY_STRICT_ORIGIN def referrer(self, response, request): - if urlparse_cached(response).scheme == urlparse_cached(request).scheme: + if ((urlparse_cached(response).scheme == 'https' and + self.potentially_trustworthy(request)) + or urlparse_cached(response).scheme == 'http'): return self.origin_referrer(response) @@ -205,8 +217,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response) if origin == self.origin(request): return self.stripped_referrer(response) - else: - return origin + elif ((urlparse_cached(response).scheme in ('https', 'ftps') and + self.potentially_trustworthy(request)) + or urlparse_cached(response).scheme == 'http'): + return self.origin_referrer(response) class UnsafeUrlPolicy(ReferrerPolicy): @@ -244,7 +258,9 @@ _policy_classes = {p.name: p for p in ( NoReferrerWhenDowngradePolicy, SameOriginPolicy, OriginPolicy, + StrictOriginPolicy, OriginWhenCrossOriginPolicy, + StrictOriginWhenCrossOriginPolicy, UnsafeUrlPolicy, DefaultReferrerPolicy, )} From deb8567116db8488a8f5b890aad975df095ddcfe Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 16:22:49 +0100 Subject: [PATCH 036/109] Update NoReferrerWhenDowngradePolicy --- scrapy/spidermiddlewares/referer.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a6316cd0c..d64c79108 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -94,12 +94,11 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): """ https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade - The "no-referrer-when-downgrade" policy sends a full URL - along with requests from a TLS-protected environment settings object - to a a priori authenticated URL, - and requests from request clients which are not TLS-protected to any origin. + The "no-referrer-when-downgrade" policy sends a full URL along with requests + from a TLS-protected environment settings object to a potentially trustworthy URL, + and requests from clients which are not TLS-protected to any origin. - Requests from TLS-protected request clients to non-a priori authenticated URLs, + Requests from TLS-protected clients to non-potentially trustworthy URLs, on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. @@ -108,15 +107,8 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): name = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response, request): - # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer: - # - # If environment is TLS-protected - # and the origin of request's current URL is not an a priori authenticated URL, - # then return no referrer. - if urlparse_cached(response).scheme in ('https', 'ftps') and \ - urlparse_cached(request).scheme in ('http',): - return None - return self.stripped_referrer(response) + if not self.tls_protected(response) or self.tls_protected(request): + return self.stripped_referrer(response) class SameOriginPolicy(ReferrerPolicy): From ebcacd3f549de034d05ed946c446e0f71b11bc6f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 17:18:00 +0100 Subject: [PATCH 037/109] Update StrictOriginPolicy --- scrapy/spidermiddlewares/referer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index d64c79108..8ee04120a 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -159,9 +159,8 @@ class StrictOriginPolicy(ReferrerPolicy): name = POLICY_STRICT_ORIGIN def referrer(self, response, request): - if ((urlparse_cached(response).scheme == 'https' and - self.potentially_trustworthy(request)) - or urlparse_cached(response).scheme == 'http'): + if ((self.tls_protected(response) and self.potentially_trustworthy(request)) + or not self.tls_protected(response)): return self.origin_referrer(response) From b6c761d2b4c9cafe94010075ebf39807921dec9c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 17:57:17 +0100 Subject: [PATCH 038/109] Fix tests --- scrapy/spidermiddlewares/referer.py | 7 ++++--- tests/test_spidermiddleware_referer.py | 8 +++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 8ee04120a..4f50db689 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -159,7 +159,8 @@ class StrictOriginPolicy(ReferrerPolicy): name = POLICY_STRICT_ORIGIN def referrer(self, response, request): - if ((self.tls_protected(response) and self.potentially_trustworthy(request)) + if ((self.tls_protected(response) and + self.potentially_trustworthy(request)) or not self.tls_protected(response)): return self.origin_referrer(response) @@ -208,9 +209,9 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response) if origin == self.origin(request): return self.stripped_referrer(response) - elif ((urlparse_cached(response).scheme in ('https', 'ftps') and + elif ((self.tls_protected(response) and self.potentially_trustworthy(request)) - or urlparse_cached(response).scheme == 'http'): + or not self.tls_protected(response)): return self.origin_referrer(response) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index df20dfbb9..4779b0ed1 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -232,7 +232,13 @@ class MixinStrictOriginWhenCrossOrigin(object): # downgrade ('https://example4.com/page.html', 'http://example4.com/not-page.html', None), ('https://example4.com/page.html', 'http://not.example4.com/', None), - ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', None), + + # non-TLS to non-TLS + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + + # upgrade + ('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'), + ('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'), # Different protocols: send origin as referrer ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), From c86f568b9cd1fc69c67762ba39714e38c9ba70fb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 22:31:29 +0100 Subject: [PATCH 039/109] Update docs with "strict-..." policies --- docs/topics/spider-middleware.rst | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a9d3d4568..e8325d7ef 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -344,9 +344,18 @@ This setting accepts: - a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass, either a custom one or one of the built-in ones (see ``scrapy.spidermiddlewares.referer``), -- or one of the standard W3C-defined string values, i.e. ``"no-referrer"``, - ``"no-referrer-when-downgrade"``, ``"same-origin"``, ``"origin"``, - ``"origin-when-cross-origin"`` or ``"unsafe-url"``. +- or one of the standard W3C-defined string values: + + - `"no-referrer" `_, + - `"no-referrer-when-downgrade" `_, + - `"same-origin" `_, + - `"origin" `_, + - `"strict-origin" `_, + - `"origin-when-cross-origin" `_, + - `"strict-origin-when-cross-origin" `_, + - or `"unsafe-url" `_ + (not recommended). + (It can also be the non-standard value ``"scrapy-default"`` to use Scrapy's default referrer policy.) From e249abc32bcb171e4676b8fd9f11f04201f80a75 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 18 Jan 2017 15:48:28 +0100 Subject: [PATCH 040/109] Update docs --- docs/topics/spider-middleware.rst | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index e8325d7ef..0ddf027ea 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -333,7 +333,7 @@ Whether to enable referer middleware. REFERER_POLICY ^^^^^^^^^^^^^^ -.. versionadded:: 1.3 +.. versionadded:: 1.4 Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` @@ -346,14 +346,14 @@ This setting accepts: (see ``scrapy.spidermiddlewares.referer``), - or one of the standard W3C-defined string values: - - `"no-referrer" `_, - - `"no-referrer-when-downgrade" `_, - - `"same-origin" `_, - - `"origin" `_, - - `"strict-origin" `_, - - `"origin-when-cross-origin" `_, - - `"strict-origin-when-cross-origin" `_, - - or `"unsafe-url" `_ + - `"no-referrer"`_, + - `"no-referrer-when-downgrade"`_, + - `"same-origin"`_, + - `"origin"`_, + - `"strict-origin"`_, + - `"origin-when-cross-origin"`_, + - `"strict-origin-when-cross-origin"`_, + - or `"unsafe-url"`_ (not recommended). (It can also be the non-standard value ``"scrapy-default"`` to use @@ -364,14 +364,22 @@ with the addition that "Referrer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. .. warning:: - By default, Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_, + Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_, will send a non-empty "Referer" header from any ``https://`` to any ``https://`` URL, even if the domain is different. ``same-origin`` may be a better choice if you want to remove referrer information for cross-domain requests. .. _Referrer Policy: https://www.w3.org/TR/referrer-policy +.. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer .. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade +.. _"same-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin +.. _"origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin +.. _"strict-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin +.. _"origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin +.. _"strict-origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin +.. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url + UrlLengthMiddleware ------------------- From 03ff19d1882edb709cf627795f3ce22934235254 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 18 Jan 2017 16:29:20 +0100 Subject: [PATCH 041/109] Update docs for new "referrer_policy" Request.meta key --- docs/topics/request-response.rst | 1 + docs/topics/spider-middleware.rst | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 3d110b02d..67f8ec285 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -307,6 +307,7 @@ Those are: * :reqmeta:`proxy` * ``ftp_user`` (See :setting:`FTP_USER` for more info) * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) +* :reqmeta:`referrer_policy` .. reqmeta:: bindaddress diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 0ddf027ea..a4ac45b41 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -337,6 +337,8 @@ REFERER_POLICY Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` +.. reqmeta:: referrer_policy + `Referrer Policy`_ to apply when populating Request "Referer" header. This setting accepts: @@ -370,6 +372,11 @@ using ``file://`` or ``s3://`` scheme. ``same-origin`` may be a better choice if you want to remove referrer information for cross-domain requests. +.. note:: + You can also override the Referrer Policy per request, + using the special ``"referrer_policy"`` :ref:`Request.meta ` key, + with the same acceptable values as for the ``REFERER_POLICY`` setting. + .. _Referrer Policy: https://www.w3.org/TR/referrer-policy .. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer .. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade From eb07285a63d6fdeefe113051943f4bf7e36f7b33 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 18 Jan 2017 17:20:35 +0100 Subject: [PATCH 042/109] Reword warning on no-referrer-when-downgrade policy --- docs/topics/spider-middleware.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a4ac45b41..ada4c46c0 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -349,7 +349,8 @@ This setting accepts: - or one of the standard W3C-defined string values: - `"no-referrer"`_, - - `"no-referrer-when-downgrade"`_, + - `"no-referrer-when-downgrade"`_ + (the W3C-recommended default, used by major web browsers), - `"same-origin"`_, - `"origin"`_, - `"strict-origin"`_, @@ -358,18 +359,19 @@ This setting accepts: - or `"unsafe-url"`_ (not recommended). - (It can also be the non-standard value ``"scrapy-default"`` to use - Scrapy's default referrer policy.) +It can also be the non-standard value ``"scrapy-default"`` to use +Scrapy's default referrer policy. Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, with the addition that "Referrer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. .. warning:: - Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_, - will send a non-empty "Referer" header from any ``https://`` to any ``https://`` URL, + Scrapy's default referrer policy—just like `"no-referrer-when-downgrade"`_, + the W3C-recommended value for browsers—will send a non-empty + "Referer" header from any ``http(s)://`` to any ``https://`` URL, even if the domain is different. - ``same-origin`` may be a better choice if you want to remove referrer + `"same-origin"`_ may be a better choice if you want to remove referrer information for cross-domain requests. .. note:: From 605935f015b1862b83f051380cd08b2931cce784 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 26 Jan 2017 12:09:34 +0100 Subject: [PATCH 043/109] Edit text --- docs/topics/spider-middleware.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index ada4c46c0..a792364b3 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -363,12 +363,12 @@ It can also be the non-standard value ``"scrapy-default"`` to use Scrapy's default referrer policy. Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, -with the addition that "Referrer" is not sent if the parent request was +with the addition that "Referer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. .. warning:: - Scrapy's default referrer policy—just like `"no-referrer-when-downgrade"`_, - the W3C-recommended value for browsers—will send a non-empty + Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_, + the W3C-recommended value for browsers — will send a non-empty "Referer" header from any ``http(s)://`` to any ``https://`` URL, even if the domain is different. `"same-origin"`_ may be a better choice if you want to remove referrer From 3dc09eeceb16ca97a0c5851bfdc41f921772910b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 30 Jan 2017 15:25:49 +0100 Subject: [PATCH 044/109] Use table for referrer policy options --- docs/topics/spider-middleware.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a792364b3..12aa53012 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -346,21 +346,21 @@ This setting accepts: - a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass, either a custom one or one of the built-in ones (see ``scrapy.spidermiddlewares.referer``), -- or one of the standard W3C-defined string values: +- or one of the standard W3C-defined string values - - `"no-referrer"`_, - - `"no-referrer-when-downgrade"`_ - (the W3C-recommended default, used by major web browsers), - - `"same-origin"`_, - - `"origin"`_, - - `"strict-origin"`_, - - `"origin-when-cross-origin"`_, - - `"strict-origin-when-cross-origin"`_, - - or `"unsafe-url"`_ - (not recommended). - -It can also be the non-standard value ``"scrapy-default"`` to use -Scrapy's default referrer policy. +======================================= ======================================================================== ======================================================= +String value Class name +======================================= ======================================================================== ======================================================= +`"no-referrer"`_ ``'scrapy.spidermiddlewares.referer.NoReferrerPolicy'`` +`"no-referrer-when-downgrade"`_ ``'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'`` the W3C-recommended default, used by major web browsers +`"same-origin"`_ ``'scrapy.spidermiddlewares.referer.SameOriginPolicy'`` +`"origin"`_ ``'scrapy.spidermiddlewares.referer.OriginPolicy'`` +`"strict-origin"`_ ``'scrapy.spidermiddlewares.referer.StrictOriginPolicy'`` +`"origin-when-cross-origin"`_ ``'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'`` +`"strict-origin-when-cross-origin"`_ ``'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'`` +`"unsafe-url"`_ ``'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'`` NOT recommended +``"scrapy-default"`` ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` Scrapy's default policy (see below) +======================================= ======================================================================== ======================================================= Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, with the addition that "Referer" is not sent if the parent request was From 537683f945906ff9865cf4e2704a432612f10eb4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 21 Feb 2017 16:47:57 +0100 Subject: [PATCH 045/109] Add autoclass directives to document built-in policies --- docs/topics/spider-middleware.rst | 70 +++++++++++++++++++---------- scrapy/spidermiddlewares/referer.py | 6 ++- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 12aa53012..349dbb3a1 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -341,43 +341,65 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` `Referrer Policy`_ to apply when populating Request "Referer" header. -This setting accepts: +.. note:: + You can also set the Referrer Policy per request, + using the special ``"referrer_policy"`` :ref:`Request.meta ` key, + with the same acceptable values as for the ``REFERER_POLICY`` setting. -- a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass, - either a custom one or one of the built-in ones - (see ``scrapy.spidermiddlewares.referer``), -- or one of the standard W3C-defined string values +Acceptable values for REFERER_POLICY +************************************ -======================================= ======================================================================== ======================================================= -String value Class name -======================================= ======================================================================== ======================================================= -`"no-referrer"`_ ``'scrapy.spidermiddlewares.referer.NoReferrerPolicy'`` -`"no-referrer-when-downgrade"`_ ``'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'`` the W3C-recommended default, used by major web browsers -`"same-origin"`_ ``'scrapy.spidermiddlewares.referer.SameOriginPolicy'`` -`"origin"`_ ``'scrapy.spidermiddlewares.referer.OriginPolicy'`` -`"strict-origin"`_ ``'scrapy.spidermiddlewares.referer.StrictOriginPolicy'`` -`"origin-when-cross-origin"`_ ``'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'`` -`"strict-origin-when-cross-origin"`_ ``'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'`` -`"unsafe-url"`_ ``'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'`` NOT recommended -``"scrapy-default"`` ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` Scrapy's default policy (see below) -======================================= ======================================================================== ======================================================= +- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` + subclass — a custom policy or one of the built-in ones (see classes below), +- or one of the standard W3C-defined string values, +- or the special ``"scrapy-default"``. -Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, -with the addition that "Referer" is not sent if the parent request was -using ``file://`` or ``s3://`` scheme. +======================================= ======================================================================== +String value Class name (as a string) +======================================= ======================================================================== +``"scrapy-default"`` (default) :class:`scrapy.spidermiddlewares.referer.DefaultReferrerPolicy` +`"no-referrer"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerPolicy` +`"no-referrer-when-downgrade"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy` +`"same-origin"`_ :class:`scrapy.spidermiddlewares.referer.SameOriginPolicy` +`"origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginPolicy` +`"strict-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginPolicy` +`"origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy` +`"strict-origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy` +`"unsafe-url"`_ :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy` +======================================= ======================================================================== +.. autoclass:: DefaultReferrerPolicy .. warning:: Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_, the W3C-recommended value for browsers — will send a non-empty "Referer" header from any ``http(s)://`` to any ``https://`` URL, even if the domain is different. + `"same-origin"`_ may be a better choice if you want to remove referrer information for cross-domain requests. +.. autoclass:: NoReferrerPolicy + +.. autoclass:: NoReferrerWhenDowngradePolicy .. note:: - You can also override the Referrer Policy per request, - using the special ``"referrer_policy"`` :ref:`Request.meta ` key, - with the same acceptable values as for the ``REFERER_POLICY`` setting. + "no-referrer-when-downgrade" policy is the W3C-recommended default, + and is used by major web browsers. + + However, it is NOT Scrapy's default referrer policy (see :class:`DefaultReferrerPolicy`). + +.. autoclass:: SameOriginPolicy + +.. autoclass:: OriginPolicy + +.. autoclass:: StrictOriginPolicy + +.. autoclass:: OriginWhenCrossOriginPolicy + +.. autoclass:: StrictOriginWhenCrossOriginPolicy + +.. autoclass:: UnsafeUrlPolicy +.. warning:: + "unsafe-url" policy is NOT recommended. .. _Referrer Policy: https://www.w3.org/TR/referrer-policy .. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 4f50db689..c015e13c8 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -240,7 +240,11 @@ class LegacyPolicy(ReferrerPolicy): class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): - + """ + A variant of "no-referrer-when-downgrade", + with the addition that "Referer" is not sent if the parent request was + using ``file://`` or ``s3://`` scheme. + """ NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') name = POLICY_SCRAPY_DEFAULT From bc200d1155e0f93f37897ac206aa32945daf89bd Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 21 Feb 2017 17:08:39 +0100 Subject: [PATCH 046/109] Rename setting to REFERRER_POLICY (with 2 Rs) --- docs/topics/spider-middleware.rst | 12 ++++----- scrapy/settings/default_settings.py | 2 +- scrapy/spidermiddlewares/referer.py | 2 +- tests/test_spidermiddleware_referer.py | 36 +++++++++++++------------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 349dbb3a1..9a0ccd0c1 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -328,10 +328,10 @@ Default: ``True`` Whether to enable referer middleware. -.. setting:: REFERER_POLICY +.. setting:: REFERRER_POLICY -REFERER_POLICY -^^^^^^^^^^^^^^ +REFERRER_POLICY +^^^^^^^^^^^^^^^ .. versionadded:: 1.4 @@ -344,10 +344,10 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` .. note:: You can also set the Referrer Policy per request, using the special ``"referrer_policy"`` :ref:`Request.meta ` key, - with the same acceptable values as for the ``REFERER_POLICY`` setting. + with the same acceptable values as for the ``REFERRER_POLICY`` setting. -Acceptable values for REFERER_POLICY -************************************ +Acceptable values for REFERRER_POLICY +************************************* - either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass — a custom policy or one of the built-in ones (see classes below), diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 15a134dd3..35d9844a7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -234,7 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True -REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' +REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index c015e13c8..24c163089 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -266,7 +266,7 @@ class RefererMiddleware(object): def __init__(self, settings=None): self.default_policy = DefaultReferrerPolicy if settings is not None: - policy = settings.get('REFERER_POLICY') + policy = settings.get('REFERRER_POLICY') if policy is not None: # expect a string for the path to the policy class try: diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 4779b0ed1..81868efab 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -289,35 +289,35 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware): # --- Tests using settings to set policy using class path class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} class CustomPythonOrgPolicy(ReferrerPolicy): @@ -336,7 +336,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware): - settings = {'REFERER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} + settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} scenarii = [ ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), @@ -385,17 +385,17 @@ class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} req_meta = {'referrer_policy': POLICY_NO_REFERRER} class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} @@ -411,7 +411,7 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: - settings = Settings({'REFERER_POLICY': s}) + settings = Settings({'REFERRER_POLICY': s}) mw = RefererMiddleware(settings) self.assertEquals(mw.default_policy, p) @@ -425,24 +425,24 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: - settings = Settings({'REFERER_POLICY': s.upper()}) + settings = Settings({'REFERRER_POLICY': s.upper()}) mw = RefererMiddleware(settings) self.assertEquals(mw.default_policy, p) def test_invalid_name(self): - settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) + settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'}) with self.assertRaises(NotConfigured): mw = RefererMiddleware(settings) class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} From 04e4d08612364480c4d72d2b55e1cac161397136 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 27 Feb 2017 16:04:37 +0100 Subject: [PATCH 047/109] Pass URLs around instead of Request/Responses --- scrapy/spidermiddlewares/referer.py | 53 ++++++++++++++++------------- scrapy/utils/url.py | 7 +--- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 24c163089..f2910a0c8 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,6 +2,8 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ +from six.moves.urllib.parse import urlparse + from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals @@ -32,12 +34,14 @@ class ReferrerPolicy(object): raise NotImplementedError() def stripped_referrer(self, r): - return self.strip_url(r) + if urlparse(r).scheme not in self.NOREFERRER_SCHEMES: + return self.strip_url(r) def origin_referrer(self, r): - return self.strip_url(r, origin_only=True) + if urlparse(r).scheme not in self.NOREFERRER_SCHEMES: + return self.origin(r) - def strip_url(self, r, origin_only=False): + def strip_url(self, url, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -51,29 +55,27 @@ class ReferrerPolicy(object): Set url's query to null. Return url. """ - if r is None or not r.url: + if not url: return None - parsed_url = urlparse_cached(r) - if parsed_url.scheme not in self.NOREFERRER_SCHEMES: - return strip_url(parsed_url, - strip_credentials=True, - strip_fragment=True, - strip_default_port=True, - origin_only=origin_only) + return strip_url(url, + strip_credentials=True, + strip_fragment=True, + strip_default_port=True, + origin_only=origin_only) def origin(self, r): """Return serialized origin (scheme, host, path) for a request or response URL.""" return self.strip_url(r, origin_only=True) - def potentially_trustworthy(self, r): + def potentially_trustworthy(self, url): # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy - parsed_url = urlparse_cached(r) + parsed_url = urlparse(url) if parsed_url.scheme in ('data',): return False - return self.tls_protected(r) + return self.tls_protected(url) - def tls_protected(self, r): - return urlparse_cached(r).scheme in ('https', 'ftps') + def tls_protected(self, url): + return urlparse(url).scheme in ('https', 'ftps') class NoReferrerPolicy(ReferrerPolicy): @@ -287,12 +289,17 @@ class RefererMiddleware(object): crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled) return mw - def policy(self, response, request): + def policy(self, resp_or_url, request): + """ + Determine Referrer-Policy to use from a parent Response (or URL), + and a Request to be sent. + """ # policy set in request's meta dict takes precedence over default policy policy_name = request.meta.get('referrer_policy') if policy_name is None: - policy_name = to_native_str( - response.headers.get('Referrer-Policy', '').decode('latin1')) + if isinstance(resp_or_url, Response): + policy_name = to_native_str( + resp_or_url.headers.get('Referrer-Policy', '').decode('latin1')) cls = _policy_classes.get(policy_name.lower(), self.default_policy) return cls() @@ -300,7 +307,7 @@ class RefererMiddleware(object): def process_spider_output(self, response, result, spider): def _set_referer(r): if isinstance(r, Request): - referrer = self.policy(response, r).referrer(response, r) + referrer = self.policy(response, r).referrer(response.url, r.url) if referrer is not None: r.headers.setdefault('Referer', referrer) return r @@ -313,9 +320,9 @@ class RefererMiddleware(object): request_referrer = request.headers.get('Referer') # we don't patch the referrer value if there is none if request_referrer is not None: - faked_response = Response(redirected_urls[0]) - policy_referrer = self.policy(faked_response, - request).referrer(faked_response, request) + initial_url = redirected_urls[0] + policy_referrer = self.policy(initial_url, + request).referrer(orig_url, request.url) if policy_referrer != request_referrer: if policy_referrer is None: request.headers.pop('Referer') diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 9864f353d..8eed31060 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -117,12 +117,7 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only= - `strip_fragment` drops any #fragment component """ - if url is None: - return None - if not isinstance(url, ParseResult): - parsed_url = urlparse(url) - else: - parsed_url = url + parsed_url = urlparse(url) netloc = parsed_url.netloc if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password): netloc = netloc.split('@')[-1] From d2aa51c0fb81b4e264fc351b9184d9ce06855021 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 27 Feb 2017 16:05:22 +0100 Subject: [PATCH 048/109] Update tests --- tests/test_spidermiddleware_referer.py | 43 ++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 81868efab..39bbaab5d 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,3 +1,4 @@ +from six.moves.urllib.parse import urlparse from unittest import TestCase from scrapy.exceptions import NotConfigured @@ -326,9 +327,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): depending on the scheme of the target URL. """ def referrer(self, response, request): - from scrapy.utils.httpobj import urlparse_cached - - scheme = urlparse_cached(request).scheme + scheme = urlparse(request).scheme if scheme == 'https': return b'https://python.org/' elif scheme == 'http': @@ -446,3 +445,41 @@ class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefere class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} + + +class TestReferrerPolicyOnRedirect(TestRefererMiddleware): + + req_meta = {} + resp_headers = {} + #settings = {} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + scenarii = [ + ( # origin + 'http://scrapytest.org/1', + + # target + redirection + ['http://scrapytest.org/2', + 'http://scrapytest.org/3',], + + b'http://scrapytest.org/1', # expected initial referer + b'http://scrapytest.org/1', # expected referer for the redirection request + + ), + ] + + def setUp(self): + self.spider = Spider('foo') + settings = Settings(self.settings) + self.mw = RefererMiddleware(settings) + + def test_(self): + + for origin, target_chain, init_referrer, final_referrer in self.scenarii: + response = self.get_response(origin) + request = self.get_request(target_chain.pop()) + + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), init_referrer) + + request.meta['redirected_urls'] = target_chain From 8226e77010d79f64c7ec5ebdb9e7cee7803600ed Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 1 Mar 2017 12:41:33 +0100 Subject: [PATCH 049/109] Add test for Referer header on HTTP redirections --- scrapy/spidermiddlewares/referer.py | 16 +- tests/test_spidermiddleware_referer.py | 398 ++++++++++++++++++++++--- 2 files changed, 364 insertions(+), 50 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index f2910a0c8..bb184cc12 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -4,6 +4,8 @@ originated it. """ from six.moves.urllib.parse import urlparse +from w3lib.url import safe_url_string + from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals @@ -300,8 +302,7 @@ class RefererMiddleware(object): if isinstance(resp_or_url, Response): policy_name = to_native_str( resp_or_url.headers.get('Referrer-Policy', '').decode('latin1')) - - cls = _policy_classes.get(policy_name.lower(), self.default_policy) + cls = _policy_classes.get(policy_name.lower()) if policy_name else self.default_policy return cls() def process_spider_output(self, response, result, spider): @@ -320,9 +321,14 @@ class RefererMiddleware(object): request_referrer = request.headers.get('Referer') # we don't patch the referrer value if there is none if request_referrer is not None: - initial_url = redirected_urls[0] - policy_referrer = self.policy(initial_url, - request).referrer(orig_url, request.url) + # the request's referrer header value acts as a surrogate + # for the parent response URL + # + # Note: if the 3xx response contained a Referrer-Policy header, + # the information is not available using this hook + parent_url = safe_url_string(request_referrer) + policy_referrer = self.policy(parent_url, request).referrer( + parent_url, request.url) if policy_referrer != request_referrer: if policy_referrer is None: request.headers.pop('Referer') diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 39bbaab5d..28c694169 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -5,6 +5,7 @@ from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.spidermiddlewares.referer import RefererMiddleware, \ POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ @@ -13,6 +14,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \ DefaultReferrerPolicy, \ NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ OriginWhenCrossOriginPolicy, OriginPolicy, \ + StrictOriginWhenCrossOriginPolicy, StrictOriginPolicy, \ SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy @@ -289,35 +291,35 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware): # --- Tests using settings to set policy using class path -class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): +class TestSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} -class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): +class TestSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} -class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): +class TestSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} -class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): +class TestSettingsOrigin(MixinOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} -class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): +class TestSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} -class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): +class TestSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} -class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): +class TestSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} -class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): +class TestSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} @@ -334,7 +336,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): return b'http://python.org/' -class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware): +class TestSettingsCustomPolicy(TestRefererMiddleware): settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} scenarii = [ ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), @@ -347,58 +349,58 @@ class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware): ] # --- Tests using Request meta dict to set policy -class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware): +class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} -class TestRefererMiddlewareNoReferrer(MixinNoReferrer, TestRefererMiddleware): +class TestRequestMetaNoReferrer(MixinNoReferrer, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_NO_REFERRER} -class TestRefererMiddlewareNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): +class TestRequestMetaNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE} -class TestRefererMiddlewareSameOrigin(MixinSameOrigin, TestRefererMiddleware): +class TestRequestMetaSameOrigin(MixinSameOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_SAME_ORIGIN} -class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware): +class TestRequestMetaOrigin(MixinOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_ORIGIN} -class TestRefererMiddlewareSrictOrigin(MixinStrictOrigin, TestRefererMiddleware): +class TestRequestMetaSrictOrigin(MixinStrictOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN} -class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): +class TestRequestMetaOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} -class TestRefererMiddlewareStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): +class TestRequestMetaStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} -class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): +class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_UNSAFE_URL} -class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): +class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} -class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): +class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} req_meta = {'referrer_policy': POLICY_NO_REFERRER} -class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): +class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} -class TestRefererMiddlewareSettingsPolicyByName(TestCase): +class TestSettingsPolicyByName(TestCase): def test_valid_name(self): for s, p in [ @@ -407,7 +409,9 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), (POLICY_SAME_ORIGIN, SameOriginPolicy), (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: settings = Settings({'REFERRER_POLICY': s}) @@ -421,7 +425,9 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), (POLICY_SAME_ORIGIN, SameOriginPolicy), (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: settings = Settings({'REFERRER_POLICY': s.upper()}) @@ -434,52 +440,354 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): mw = RefererMiddleware(settings) -class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): +class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} -class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): +class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} -class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): +class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} -class TestReferrerPolicyOnRedirect(TestRefererMiddleware): +class TestReferrerOnRedirect(TestRefererMiddleware): - req_meta = {} - resp_headers = {} - #settings = {} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} scenarii = [ - ( # origin - 'http://scrapytest.org/1', - - # target + redirection - ['http://scrapytest.org/2', - 'http://scrapytest.org/3',], - + ( 'http://scrapytest.org/1', # parent + 'http://scrapytest.org/2', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/3'), + (301, 'http://scrapytest.org/4'), + ), b'http://scrapytest.org/1', # expected initial referer b'http://scrapytest.org/1', # expected referer for the redirection request - + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.org/2', + ( + # redirecting to non-secure URL + (301, 'http://scrapytest.org/3'), + ), + b'https://scrapytest.org/1', + b'https://scrapytest.org/1', + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.com/2', + ( + # redirecting to non-secure URL: different origin + (301, 'http://scrapytest.com/3'), + ), + b'https://scrapytest.org/1', + b'https://scrapytest.org/1', ), ] def setUp(self): self.spider = Spider('foo') settings = Settings(self.settings) - self.mw = RefererMiddleware(settings) + self.referrermw = RefererMiddleware(settings) + self.redirectmw = RedirectMiddleware(settings) - def test_(self): + def test(self): - for origin, target_chain, init_referrer, final_referrer in self.scenarii: - response = self.get_response(origin) - request = self.get_request(target_chain.pop()) + for parent, target, redirections, init_referrer, final_referrer in self.scenarii: + response = self.get_response(parent) + request = self.get_request(target) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) + out = list(self.referrermw.process_spider_output(response, [request], self.spider)) self.assertEquals(out[0].headers.get('Referer'), init_referrer) - request.meta['redirected_urls'] = target_chain + for status, url in redirections: + response = Response(request.url, headers={'Location': url}, status=status) + request = self.redirectmw.process_response(request, response, self.spider) + self.referrermw.request_scheduled(request, self.spider) + + assert isinstance(request, Request) + self.assertEquals(request.headers.get('Referer'), final_referrer) + + +class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): + """ + No Referrer policy never sets the "Referer" header. + HTTP redirections should not change that. + """ + settings = {'REFERRER_POLICY': 'no-referrer'} + scenarii = [ + ( 'http://scrapytest.org/1', # parent + 'http://scrapytest.org/2', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/3'), + (301, 'http://scrapytest.org/4'), + ), + None, # expected initial "Referer" + None, # expected "Referer" for the redirection request + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.org/2', + ( + (301, 'http://scrapytest.org/3'), + ), + None, + None, + ), + ( 'https://scrapytest.org/1', + 'https://example.com/2', # different origin + ( + (301, 'http://scrapytest.com/3'), + ), + None, + None, + ), + ] + + +class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect): + """ + Same Origin policy sends the full URL as "Referer" if the target origin + is the same as the parent response (same protocol, same domain, same port). + + HTTP redirections to a different domain or a lower secure level + should have the "Referer" removed. + """ + settings = {'REFERRER_POLICY': 'same-origin'} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial "Referer" + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting from secure to non-secure URL == different origin + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # different domain == different origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + None, + ), + ] + + +class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect): + """ + Strict Origin policy will always send the "origin" as referrer + (think of it as the parent URL without the path part), + unless the security level is lower and no "Referer" is sent. + + Redirections from secure to non-secure URLs should have the + "Referrer" header removed if necessary. + """ + settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', + 'http://scrapytest.org/102', + ( + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/', # send origin + b'http://scrapytest.org/', # redirects to same origin: send origin + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: no referrer + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): no referrer + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/', + None, + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # HTTPS all along, so origin referrer is kept as-is + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/601', + 'http://scrapytest.org/602', # TLS to non-TLS: no referrer + ( + (301, 'https://scrapytest.org/603'), # TLS URL again: (still) no referrer + ), + None, + None, + ), + ] + + +class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect): + """ + Origin When Cross-Origin policy sends the full URL as "Referer", + unless the target's origin is different (different domain, different protocol) + in which case only the origin is sent. + + Redirections to a different origin should strip the "Referer" + to the parent origin. + """ + settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + redirection + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: send origin + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + b'https://scrapytest.org/', + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): send origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + b'https://scrapytest.org/', + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # all different domains: send origin + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/301', + 'http://scrapytest.org/302', # TLS to non-TLS: send origin + ( + (301, 'https://scrapytest.org/303'), # TLS URL again: send origin (also) + ), + b'https://scrapytest.org/', + b'https://scrapytest.org/', + ), + ] + + +class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect): + """ + Strict Origin When Cross-Origin policy sends the full URL as "Referer", + unless the target's origin is different (different domain, different protocol) + in which case only the origin is sent... + Unless there's also a downgrade in security and then the "Referer" header + is not sent. + + Redirections to a different origin should strip the "Referer" to the parent origin, + and from https:// to http:// will remove the "Referer" header. + """ + settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + redirection + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: do not send the "Referer" header + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): send origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + None, + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # all different domains: send origin + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/601', + 'http://scrapytest.org/602', # TLS to non-TLS: do not send "Referer" + ( + (301, 'https://scrapytest.org/603'), # TLS URL again: (still) send nothing + ), + None, + None, + ), + ] From 6916dd6240940cab06e2d02b3fceeb32c70691de Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 1 Mar 2017 17:42:46 +0100 Subject: [PATCH 050/109] Warn or fail with exception on unknown policies --- scrapy/spidermiddlewares/referer.py | 55 +++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index bb184cc12..5ed40791b 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -3,6 +3,7 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ from six.moves.urllib.parse import urlparse +import warnings from w3lib.url import safe_url_string @@ -265,45 +266,69 @@ _policy_classes = {p.name: p for p in ( DefaultReferrerPolicy, )} + +def _load_policy_class(policy, warning_only=False): + """ + Expect a string for the path to the policy class, + otherwise try to interpret the string as a standard value + from https://www.w3.org/TR/referrer-policy/#referrer-policies + """ + try: + return load_object(policy) + except ValueError: + try: + return _policy_classes[policy.lower()] + except KeyError: + msg = "Could not load referrer policy %r" % policy + if not warning_only: + raise RuntimeError(msg) + else: + warnings.warn(msg, RuntimeWarning) + return None + + class RefererMiddleware(object): def __init__(self, settings=None): self.default_policy = DefaultReferrerPolicy if settings is not None: - policy = settings.get('REFERRER_POLICY') - if policy is not None: - # expect a string for the path to the policy class - try: - self.default_policy = load_object(policy) - except ValueError: - # otherwise try to interpret the string as standard - # https://www.w3.org/TR/referrer-policy/#referrer-policies - try: - self.default_policy = _policy_classes[policy.lower()] - except: - raise NotConfigured("Unknown referrer policy name %r" % policy) + self.default_policy = _load_policy_class( + settings.get('REFERRER_POLICY')) @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured mw = cls(crawler.settings) + + # Note: this hook is a bit of a hack to intercept redirections crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled) + return mw def policy(self, resp_or_url, request): """ Determine Referrer-Policy to use from a parent Response (or URL), and a Request to be sent. + + - if a valid policy is set in Request meta, it is used. + - if the policy is set in meta but is wrong (e.g. a typo error), + the policy from settings is used + - if the policy is not set in Request meta, + but there is a Referrer-policy header in the parent response, + it is used if valid + - otherwise, the policy from settings is used. """ - # policy set in request's meta dict takes precedence over default policy policy_name = request.meta.get('referrer_policy') if policy_name is None: if isinstance(resp_or_url, Response): policy_name = to_native_str( resp_or_url.headers.get('Referrer-Policy', '').decode('latin1')) - cls = _policy_classes.get(policy_name.lower()) if policy_name else self.default_policy - return cls() + if policy_name is None: + return self.default_policy() + + cls = _load_policy_class(policy_name, warning_only=True) + return cls() if cls else self.default_policy() def process_spider_output(self, response, result, spider): def _set_referer(r): From efa50039ec4dc8d5cd5103e64c47ad8c86b9ed4e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 1 Mar 2017 17:43:47 +0100 Subject: [PATCH 051/109] Add tests for policy fallback on unknown policies from meta and headers --- tests/test_spidermiddleware_referer.py | 76 +++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 28c694169..b1c815876 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,5 +1,6 @@ from six.moves.urllib.parse import urlparse from unittest import TestCase +import warnings from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request @@ -400,6 +401,79 @@ class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_UNSAFE_URL} +class TestRequestMetaSettingFallback(TestCase): + + params = [ + ( + # When an unknown policy is referenced in Request.meta + # (here, a typo error), + # the policy defined in settings takes precedence + {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}, + {}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # same as above but with string value for settings policy + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # request meta references a wrong policy but it is set, + # so the Referrer-Policy header in response is not used, + # and the settings' policy is applied + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unsafe-url'}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # here, request meta does not set the policy + # so response headers take precedence + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unsafe-url'}, + {}, + UnsafeUrlPolicy, + False + ), + ( + # here, request meta does not set the policy, + # but response headers also use an unknown policy, + # so the settings' policy is used + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unknown'}, + {}, + OriginWhenCrossOriginPolicy, + True + ) + ] + + def test(self): + + origin = 'http://www.scrapy.org' + target = 'http://www.example.com' + + for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]: + spider = Spider('foo') + mw = RefererMiddleware(Settings(settings)) + + response = Response(origin, headers=response_headers) + request = Request(target, meta=request_meta) + + with warnings.catch_warnings(record=True) as w: + policy = mw.policy(response, request) + self.assertIsInstance(policy, policy_class) + + if check_warning: + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, RuntimeWarning, w[0].message) + + class TestSettingsPolicyByName(TestCase): def test_valid_name(self): @@ -436,7 +510,7 @@ class TestSettingsPolicyByName(TestCase): def test_invalid_name(self): settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'}) - with self.assertRaises(NotConfigured): + with self.assertRaises(RuntimeError): mw = RefererMiddleware(settings) From 2d55d838ca1e2936e5729e67a6ab34a4bb58ba3b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 1 Mar 2017 20:59:52 +0100 Subject: [PATCH 052/109] Fix strip_url() tests --- tests/test_utils_url.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 9182d0fda..c2b9fc176 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -274,7 +274,6 @@ class StripUrl(unittest.TestCase): 'http://www.example.com/'), ]: self.assertEqual(strip_url(input_url, origin_only=origin), output_url) - self.assertEqual(strip_url(urlparse(input_url), origin_only=origin), output_url) def test_credentials(self): for i, o in [ @@ -288,7 +287,6 @@ class StripUrl(unittest.TestCase): 'ftp://www.example.com/index.html?somekey=somevalue'), ]: self.assertEqual(strip_url(i, strip_credentials=True), o) - self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) def test_credentials_encoded_delims(self): for i, o in [ @@ -308,7 +306,6 @@ class StripUrl(unittest.TestCase): 'ftp://www.example.com/index.html?somekey=somevalue'), ]: self.assertEqual(strip_url(i, strip_credentials=True), o) - self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) def test_default_ports_creds_off(self): for i, o in [ @@ -337,7 +334,6 @@ class StripUrl(unittest.TestCase): 'ftp://www.example.com:221/file.txt'), ]: self.assertEqual(strip_url(i), o) - self.assertEqual(strip_url(urlparse(i)), o) def test_default_ports(self): for i, o in [ @@ -366,7 +362,6 @@ class StripUrl(unittest.TestCase): 'ftp://username:password@www.example.com:221/file.txt'), ]: self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o) - self.assertEqual(strip_url(urlparse(i), strip_default_port=True, strip_credentials=False), o) def test_default_ports_keep(self): for i, o in [ @@ -395,7 +390,6 @@ class StripUrl(unittest.TestCase): 'ftp://username:password@www.example.com:221/file.txt'), ]: self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o) - self.assertEqual(strip_url(urlparse(i), strip_default_port=False, strip_credentials=False), o) def test_origin_only(self): for i, o in [ @@ -412,7 +406,6 @@ class StripUrl(unittest.TestCase): 'https://www.example.com/'), ]: self.assertEqual(strip_url(i, origin_only=True), o) - self.assertEqual(strip_url(urlparse(i), origin_only=True), o) if __name__ == "__main__": From 97d84d920bf97f3ec1becd291596b3f0ee966328 Mon Sep 17 00:00:00 2001 From: jorenham Date: Thu, 2 Mar 2017 11:04:16 +0100 Subject: [PATCH 053/109] Logging the cache directory at HttpCacheMiddleware instantiation #2604 --- scrapy/downloadermiddlewares/httpcache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 30e49b886..6f1ccce68 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,3 +1,5 @@ +import logging + from email.utils import formatdate from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ @@ -9,6 +11,9 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.utils.misc import load_object +logger = logging.getLogger(__name__) + + class HttpCacheMiddleware(object): DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, @@ -24,6 +29,8 @@ class HttpCacheMiddleware(object): self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') self.stats = stats + logger.debug("Using cache directory %(cachedir)s" % {'cachedir': self.storage.cachedir}) + @classmethod def from_crawler(cls, crawler): o = cls(crawler.settings, crawler.stats) From f3b75c940d2fefc3ff5bb4435e507c5959ff903c Mon Sep 17 00:00:00 2001 From: MrMenezes Date: Fri, 21 Oct 2016 18:02:16 -0300 Subject: [PATCH 054/109] Fix warning to duplicated spider. Issue 2181 --- scrapy/spiderloader.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index d4f0f663f..e6c3e64a5 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -23,7 +23,12 @@ class SpiderLoader(object): def _load_spiders(self, module): for spcls in iter_spider_classes(module): - self._spiders[spcls.name] = spcls + if spcls.name in self._spiders.keys(): + import warnings + warnings.warn("There are several spiders with the same name (" + spcls.name + + "), this can cause unexpected behavior", UserWarning) + self._spiders[spcls.name] = spcls + def _load_all_spiders(self): for name in self.spider_modules: From 6abd9ba843e54ecb869af9571192c3f33375a2b6 Mon Sep 17 00:00:00 2001 From: Erick Date: Fri, 21 Oct 2016 18:24:59 -0300 Subject: [PATCH 055/109] Fix warning to duplicated spider. Issue 2181 --- scrapy/spiderloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index e6c3e64a5..6093c07c6 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -27,7 +27,7 @@ class SpiderLoader(object): import warnings warnings.warn("There are several spiders with the same name (" + spcls.name + "), this can cause unexpected behavior", UserWarning) - self._spiders[spcls.name] = spcls + self._spiders[spcls.name] = spcls def _load_all_spiders(self): From 5be5ef57f38b5d6f8ed01ccf9c9cac1e1e02fe03 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 23 Nov 2016 11:01:31 +0100 Subject: [PATCH 056/109] Remove extra blank line --- scrapy/spiderloader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 6093c07c6..1322c01d1 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -28,7 +28,6 @@ class SpiderLoader(object): warnings.warn("There are several spiders with the same name (" + spcls.name + "), this can cause unexpected behavior", UserWarning) self._spiders[spcls.name] = spcls - def _load_all_spiders(self): for name in self.spider_modules: From e71803c833edd67400848a3b0d4dbb01e0ec80f7 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 23 Nov 2016 11:52:02 +0100 Subject: [PATCH 057/109] Add tests for duplicate spider name warnings --- tests/test_spiderloader/__init__.py | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index b2ad93b3f..ac5f0ddab 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -101,3 +101,52 @@ class SpiderLoaderTest(unittest.TestCase): spiders = spider_loader.list() self.assertEqual(spiders, []) + + +class DuplicateSpiderNameLoaderTest(unittest.TestCase): + + def setUp(self): + orig_spiders_dir = os.path.join(module_dir, 'test_spiders') + self.tmpdir = self.mktemp() + os.mkdir(self.tmpdir) + self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') + shutil.copytree(orig_spiders_dir, self.spiders_dir) + sys.path.append(self.tmpdir) + self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) + + def tearDown(self): + del sys.modules['test_spiders_xxx'] + sys.path.remove(self.tmpdir) + + def test_dupename_warning(self): + # copy 1 spider module so as to have duplicate spider name + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider3.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx/spider3dupe.py')) + + with warnings.catch_warnings(record=True) as w: + spider_loader = SpiderLoader.from_settings(self.settings) + + self.assertEqual(len(w), 1) + self.assertIn("several spiders with the same name (spider3)", str(w[0].message)) + + spiders = set(spider_loader.list()) + self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3'])) + + def test_multiple_dupename_warning(self): + # copy 2 spider modules so as to have duplicate spider name + # This should issue 2 warning, 1 for each duplicate spider name + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider1.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx/spider1dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider2.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx/spider2dupe.py')) + + with warnings.catch_warnings(record=True) as w: + spider_loader = SpiderLoader.from_settings(self.settings) + + self.assertEqual(len(w), 2) + msgs = sorted(str(wrn.message) for wrn in w) + self.assertIn("several spiders with the same name (spider1)", msgs[0]) + self.assertIn("several spiders with the same name (spider2)", msgs[1]) + + spiders = set(spider_loader.list()) + self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3'])) From 12a8ddecab8c64923d1789571955699270255211 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Mar 2017 13:03:18 +0100 Subject: [PATCH 058/109] Fix tests --- tests/test_spiderloader/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index ac5f0ddab..302bf9a10 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -130,7 +130,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): self.assertIn("several spiders with the same name (spider3)", str(w[0].message)) spiders = set(spider_loader.list()) - self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3'])) + self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) def test_multiple_dupename_warning(self): # copy 2 spider modules so as to have duplicate spider name @@ -149,4 +149,4 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): self.assertIn("several spiders with the same name (spider2)", msgs[1]) spiders = set(spider_loader.list()) - self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3'])) + self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) From f96490df2ce532e3b8757b2464f5f3cc54181ce5 Mon Sep 17 00:00:00 2001 From: jorenham Date: Thu, 2 Mar 2017 16:17:51 +0100 Subject: [PATCH 059/109] Move cache storage logging to the individual storage classes --- scrapy/downloadermiddlewares/httpcache.py | 7 ------- scrapy/extensions/httpcache.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 6f1ccce68..30e49b886 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,5 +1,3 @@ -import logging - from email.utils import formatdate from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ @@ -11,9 +9,6 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.utils.misc import load_object -logger = logging.getLogger(__name__) - - class HttpCacheMiddleware(object): DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, @@ -29,8 +24,6 @@ class HttpCacheMiddleware(object): self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') self.stats = stats - logger.debug("Using cache directory %(cachedir)s" % {'cachedir': self.storage.cachedir}) - @classmethod def from_crawler(cls, crawler): o = cls(crawler.settings, crawler.stats) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 247cac64e..8025efe77 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,6 +1,7 @@ from __future__ import print_function import os import gzip +import logging from six.moves import cPickle as pickle from importlib import import_module from time import time @@ -15,6 +16,9 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode +logger = logging.getLogger(__name__) + + class DummyPolicy(object): def __init__(self, settings): @@ -216,6 +220,8 @@ class DbmCacheStorage(object): self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE']) self.db = None + logger.debug("Using DBM cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) + def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) self.db = self.dbmodule.open(dbpath, 'c') @@ -271,6 +277,8 @@ class FilesystemCacheStorage(object): self.use_gzip = settings.getbool('HTTPCACHE_GZIP') self._open = gzip.open if self.use_gzip else open + logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) + def open_spider(self, spider): pass @@ -344,6 +352,8 @@ class LeveldbCacheStorage(object): self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') self.db = None + logger.debug("Using LevelDB cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) + def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name) self.db = self._leveldb.LevelDB(dbpath) From b50d0370f4ebdb485f4625ef6d71398ac8538c79 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Thu, 2 Mar 2017 14:46:33 -0300 Subject: [PATCH 060/109] Test response attributes in data URI download handler. --- tests/test_downloader_handlers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index b27245a36..74203dbfe 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -838,6 +838,16 @@ class DataURITestCase(unittest.TestCase): self.download_request = self.download_handler.download_request self.spider = Spider('foo') + def test_response_attrs(self): + uri = "data:,A%20brief%20note" + + def _test(response): + self.assertEquals(response.url, uri) + self.assertFalse(response.headers) + + request = Request(uri) + return self.download_request(request, self.spider).addCallback(_test) + def test_default_mediatype_encoding(self): def _test(response): self.assertEquals(response.text, 'A brief note') From c2c503192f24a198b4515ab006e3174e28062731 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Mar 2017 22:53:27 +0100 Subject: [PATCH 061/109] Rename arguments --- scrapy/spidermiddlewares/referer.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 5ed40791b..2bc8b1782 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -36,13 +36,13 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() - def stripped_referrer(self, r): - if urlparse(r).scheme not in self.NOREFERRER_SCHEMES: - return self.strip_url(r) + def stripped_referrer(self, url): + if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: + return self.strip_url(url) - def origin_referrer(self, r): - if urlparse(r).scheme not in self.NOREFERRER_SCHEMES: - return self.origin(r) + def origin_referrer(self, url): + if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: + return self.origin(url) def strip_url(self, url, origin_only=False): """ @@ -66,9 +66,9 @@ class ReferrerPolicy(object): strip_default_port=True, origin_only=origin_only) - def origin(self, r): + def origin(self, url): """Return serialized origin (scheme, host, path) for a request or response URL.""" - return self.strip_url(r, origin_only=True) + return self.strip_url(url, origin_only=True) def potentially_trustworthy(self, url): # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy @@ -241,7 +241,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): class LegacyPolicy(ReferrerPolicy): def referrer(self, response, request): - return response.url + return response class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): From db176f872b91c4f51f10ea1e732875dc40a9e7bb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Mar 2017 22:56:23 +0100 Subject: [PATCH 062/109] Remove Legacy Policy which is equivalent to UnsafeUrl Policy --- scrapy/spidermiddlewares/referer.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 2bc8b1782..7b41bfaf7 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -239,11 +239,6 @@ class UnsafeUrlPolicy(ReferrerPolicy): return self.stripped_referrer(response) -class LegacyPolicy(ReferrerPolicy): - def referrer(self, response, request): - return response - - class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): """ A variant of "no-referrer-when-downgrade", From fad499ab602e7f12bf56a514f5aae1a2f80670f7 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Mar 2017 23:06:04 +0100 Subject: [PATCH 063/109] Rename arguments (bis) --- scrapy/spidermiddlewares/referer.py | 58 ++++++++++++++--------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 7b41bfaf7..b444e34bb 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -33,7 +33,7 @@ class ReferrerPolicy(object): NOREFERRER_SCHEMES = LOCAL_SCHEMES - def referrer(self, response, request): + def referrer(self, response_url, request_url): raise NotImplementedError() def stripped_referrer(self, url): @@ -91,7 +91,7 @@ class NoReferrerPolicy(ReferrerPolicy): """ name = POLICY_NO_REFERRER - def referrer(self, response, request): + def referrer(self, response_url, request_url): return None @@ -111,9 +111,9 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): """ name = POLICY_NO_REFERRER_WHEN_DOWNGRADE - def referrer(self, response, request): - if not self.tls_protected(response) or self.tls_protected(request): - return self.stripped_referrer(response) + def referrer(self, response_url, request_url): + if not self.tls_protected(response_url) or self.tls_protected(request_url): + return self.stripped_referrer(response_url) class SameOriginPolicy(ReferrerPolicy): @@ -128,9 +128,9 @@ class SameOriginPolicy(ReferrerPolicy): """ name = POLICY_SAME_ORIGIN - def referrer(self, response, request): - if self.origin(response) == self.origin(request): - return self.stripped_referrer(response) + def referrer(self, response_url, request_url): + if self.origin(response_url) == self.origin(request_url): + return self.stripped_referrer(response_url) class OriginPolicy(ReferrerPolicy): @@ -144,8 +144,8 @@ class OriginPolicy(ReferrerPolicy): """ name = POLICY_ORIGIN - def referrer(self, response, request): - return self.origin_referrer(response) + def referrer(self, response_url, request_url): + return self.origin_referrer(response_url) class StrictOriginPolicy(ReferrerPolicy): @@ -163,11 +163,11 @@ class StrictOriginPolicy(ReferrerPolicy): """ name = POLICY_STRICT_ORIGIN - def referrer(self, response, request): - if ((self.tls_protected(response) and - self.potentially_trustworthy(request)) - or not self.tls_protected(response)): - return self.origin_referrer(response) + def referrer(self, response_url, request_url): + if ((self.tls_protected(response_url) and + self.potentially_trustworthy(request_url)) + or not self.tls_protected(response_url)): + return self.origin_referrer(response_url) class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -183,10 +183,10 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): """ name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response, request): - origin = self.origin(response) - if origin == self.origin(request): - return self.stripped_referrer(response) + def referrer(self, response_url, request_url): + origin = self.origin(response_url) + if origin == self.origin(request_url): + return self.stripped_referrer(response_url) else: return origin @@ -210,14 +210,14 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): """ name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response, request): - origin = self.origin(response) - if origin == self.origin(request): - return self.stripped_referrer(response) - elif ((self.tls_protected(response) and - self.potentially_trustworthy(request)) - or not self.tls_protected(response)): - return self.origin_referrer(response) + def referrer(self, response_url, request_url): + origin = self.origin(response_url) + if origin == self.origin(request_url): + return self.stripped_referrer(response_url) + elif ((self.tls_protected(response_url) and + self.potentially_trustworthy(request_url)) + or not self.tls_protected(response_url)): + return self.origin_referrer(response_url) class UnsafeUrlPolicy(ReferrerPolicy): @@ -235,8 +235,8 @@ class UnsafeUrlPolicy(ReferrerPolicy): """ name = POLICY_UNSAFE_URL - def referrer(self, response, request): - return self.stripped_referrer(response) + def referrer(self, response_url, request_url): + return self.stripped_referrer(response_url) class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): From 42b429dc37e17a3a24d9f5f6d9365b6d78479630 Mon Sep 17 00:00:00 2001 From: jorenham Date: Fri, 3 Mar 2017 15:15:59 +0100 Subject: [PATCH 064/109] Log full cache file path instead of cache directory for the storages that cache to single files. --- scrapy/extensions/httpcache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 8025efe77..fe8c55c6b 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -220,12 +220,12 @@ class DbmCacheStorage(object): self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE']) self.db = None - logger.debug("Using DBM cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) - def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) self.db = self.dbmodule.open(dbpath, 'c') + logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) + def close_spider(self, spider): self.db.close() @@ -352,12 +352,12 @@ class LeveldbCacheStorage(object): self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') self.db = None - logger.debug("Using LevelDB cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) - def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name) self.db = self._leveldb.LevelDB(dbpath) + logger.debug("Using LevelDB cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) + def close_spider(self, spider): # Do compactation each time to save space and also recreate files to # avoid them being removed in storages with timestamp-based autoremoval. From 5e89db548419c4114ff71bd0504155c9a83868fa Mon Sep 17 00:00:00 2001 From: jorenham Date: Fri, 3 Mar 2017 15:32:20 +0100 Subject: [PATCH 065/109] Moved cache dir logging to `open_spider` in FilesystemCacheStorage for consistency --- scrapy/extensions/httpcache.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index fe8c55c6b..2fb4b6a15 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -277,10 +277,9 @@ class FilesystemCacheStorage(object): self.use_gzip = settings.getbool('HTTPCACHE_GZIP') self._open = gzip.open if self.use_gzip else open - logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) - def open_spider(self, spider): - pass + logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}, + extra={'spider': spider}) def close_spider(self, spider): pass From 5d0058492c07d3f89f46ccfd2b8e33849f8bd30e Mon Sep 17 00:00:00 2001 From: Bernardas Date: Tue, 23 Aug 2016 16:54:10 +0000 Subject: [PATCH 066/109] add media pipeline settings to enable redirection and handling of certain http statuses --- scrapy/pipelines/files.py | 2 +- scrapy/pipelines/media.py | 34 +++++++++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 843b4d3ec..4ae7e1d89 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -249,7 +249,7 @@ class FilesPipeline(MediaPipeline): resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD ) - super(FilesPipeline, self).__init__(download_func=download_func) + super(FilesPipeline, self).__init__(download_func=download_func, settings=settings) @classmethod def from_settings(cls, settings): diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 57f70499e..4177d294f 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,5 +1,6 @@ from __future__ import print_function +import functools import logging from collections import defaultdict from twisted.internet.defer import Deferred, DeferredList @@ -16,6 +17,7 @@ logger = logging.getLogger(__name__) class MediaPipeline(object): LOG_FAILED_RESULTS = True + ALLOW_REDIRECTS = False class SpiderInfo(object): def __init__(self, spider): @@ -24,9 +26,16 @@ class MediaPipeline(object): self.downloaded = {} self.waiting = defaultdict(list) - def __init__(self, download_func=None): + def __init__(self, download_func=None, settings=None): self.download_func = download_func - + resolve = functools.partial(self._key_for_pipe, + base_class_name="MediaPipeline") + self.allow_redirects = settings.getbool( + resolve('MEDIA_ALLOW_REDIRECTS'), self.ALLOW_REDIRECTS + ) + self.allow_httpstatus_list = settings.getlist( + resolve('MEDIA_HTTPSTATUS_LIST'), [] + ) def _key_for_pipe(self, key, base_class_name=None, settings=None): @@ -93,6 +102,25 @@ class MediaPipeline(object): ) return dfd.addBoth(lambda _: wad) # it must return wad at last + def _modify_media_request(self, request): + httpstatus_list = [] + if self.allow_httpstatus_list: + httpstatus_list = self.allow_httpstatus_list + elif self.allow_redirects: + if not httpstatus_list: + httpstatus_list = list(range(0, 300)) + list(range(400, 1000)) + else: + for i in range(300, 400): + try: + httpstatus_list.remove(i) + except ValueError: + pass + if httpstatus_list: + request.meta['handle_httpstatus_list'] = httpstatus_list + else: + request.meta['handle_httpstatus_all'] = True + return request + def _check_media_to_download(self, result, request, info): if result is not None: return result @@ -103,7 +131,7 @@ class MediaPipeline(object): callback=self.media_downloaded, callbackArgs=(request, info), errback=self.media_failed, errbackArgs=(request, info)) else: - request.meta['handle_httpstatus_all'] = True + request = self._modify_media_request(request) dfd = self.crawler.engine.download(request, info.spider) dfd.addCallbacks( callback=self.media_downloaded, callbackArgs=(request, info), From 25ed491219924b5fc98a67f64d355249a1b68f50 Mon Sep 17 00:00:00 2001 From: Bernardas Date: Tue, 23 Aug 2016 16:55:34 +0000 Subject: [PATCH 067/109] add description for media pipeline MEDIA_ALLOW_REDIRECTS and MEDIA_HTTPSTATUS_LIST settings --- docs/topics/media-pipeline.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 82c0aaa88..a86bab4bf 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -322,6 +322,22 @@ By default, there are no size constraints, so all images are processed. .. _topics-media-pipeline-override: +Allowing redirection and handling various http statuses +------------------------------------------------------- + +.. setting:: MEDIA_ALLOW_REDIRECTS +.. setting:: MEDIA_HTTPSTATUS_LIST + +By default media pipelines ignore redirects. To allow redirecting(all 300 codes) set: + + MEDIA_ALLOW_REDIRECTS = True + +To only allow specific codes through set: + + MEDIA_HTTpSTATUS_LIST = + # example: + MEDIA_HTTPSTATUS_LIST = [303, 404] + Extending the Media Pipelines ============================= From 6a4221471610b7f3a5d44d93306294c21d550406 Mon Sep 17 00:00:00 2001 From: Bernardas Date: Tue, 23 Aug 2016 16:56:31 +0000 Subject: [PATCH 068/109] add tests for media pipeline MEDIA_ALLOW_REDIRECTS and MEDIA_HTTPSTATUS_LIST settings --- tests/test_pipeline_media.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index f30b4fea3..41ee9962f 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -6,6 +6,7 @@ from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.http import Request, Response +from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.request import request_fingerprint from scrapy.pipelines.media import MediaPipeline @@ -25,7 +26,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase): def setUp(self): self.spider = Spider('media.com') - self.pipe = self.pipeline_class(download_func=_mocked_download_func) + self.pipe = self.pipeline_class(download_func=_mocked_download_func, + settings=Settings()) self.pipe.open_spider(self.spider) self.info = self.pipe.spiderinfo @@ -82,6 +84,21 @@ class BaseMediaPipelineTestCase(unittest.TestCase): new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item + def test_modify_media_request(self): + request = Request('http://url') + assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True} + + request = Request('http://url') + self.pipe.allow_httpstatus_list = list(range(100)) + assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_list': list(range(100))} + self.pipe.allow_httpstatus_list = None + + request = Request('http://url') + self.pipe.allow_redirects = True + correct = {'handle_httpstatus_list': list(range(300)) + list(range(400,1000))} + assert self.pipe._modify_media_request(request).meta == correct + self.pipe.allow_redirects = False + class MockedMediaPipeline(MediaPipeline): From 854278085494053361a8cf070237683557b642ae Mon Sep 17 00:00:00 2001 From: Bernardas Date: Tue, 23 Aug 2016 17:09:43 +0000 Subject: [PATCH 069/109] typo and clarify handling --- docs/topics/media-pipeline.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index a86bab4bf..bfc405d9e 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -332,11 +332,11 @@ By default media pipelines ignore redirects. To allow redirecting(all 300 codes) MEDIA_ALLOW_REDIRECTS = True -To only allow specific codes through set: +To only allow handling only specific codes set (default: any code): - MEDIA_HTTpSTATUS_LIST = + MEDIA_HTTPSTATUS_LIST = # example: - MEDIA_HTTPSTATUS_LIST = [303, 404] + MEDIA_HTTPSTATUS_LIST = [303, 404] # will not go through pipelines Extending the Media Pipelines ============================= From 2e052c86150502c7441c8f92cd1488de6232ec7f Mon Sep 17 00:00:00 2001 From: Bernardas Date: Tue, 23 Aug 2016 17:13:09 +0000 Subject: [PATCH 070/109] fix error when settings are not provided --- scrapy/pipelines/media.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 4177d294f..976b5032b 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -6,6 +6,7 @@ from collections import defaultdict from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure +from scrapy.settings import Settings from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter @@ -28,6 +29,8 @@ class MediaPipeline(object): def __init__(self, download_func=None, settings=None): self.download_func = download_func + if isinstance(settings, dict) or settings is None: + settings = Settings(settings) resolve = functools.partial(self._key_for_pipe, base_class_name="MediaPipeline") self.allow_redirects = settings.getbool( From f0b4077f812619640df078a2485f8a460fafa58a Mon Sep 17 00:00:00 2001 From: Bernardas Date: Wed, 24 Aug 2016 08:39:30 +0000 Subject: [PATCH 071/109] expose allowed_status tuple for media pipeline --- scrapy/downloadermiddlewares/redirect.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 26677e527..ae4ad8891 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -57,6 +57,8 @@ class RedirectMiddleware(BaseRedirectMiddleware): Handle redirection of requests based on response status and meta-refresh html tag. """ + allowed_status = (301, 302, 303, 307) + def process_response(self, request, response, spider): if (request.meta.get('dont_redirect', False) or response.status in getattr(spider, 'handle_httpstatus_list', []) or @@ -64,8 +66,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): request.meta.get('handle_httpstatus_all', False)): return response - allowed_status = (301, 302, 303, 307) - if 'Location' not in response.headers or response.status not in allowed_status: + if 'Location' not in response.headers or response.status not in self.allowed_status: return response location = safe_url_string(response.headers['location']) From 3cef1cd451c8f28df8075aa4e3b65cedf5ecedfa Mon Sep 17 00:00:00 2001 From: Bernardas Date: Wed, 24 Aug 2016 08:40:46 +0000 Subject: [PATCH 072/109] adjust variable wording and redirect logic --- scrapy/pipelines/media.py | 19 +++++++++---------- tests/test_pipeline_media.py | 4 ++-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 976b5032b..b11e7095b 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -6,6 +6,7 @@ from collections import defaultdict from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.settings import Settings from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy.utils.request import request_fingerprint @@ -36,7 +37,7 @@ class MediaPipeline(object): self.allow_redirects = settings.getbool( resolve('MEDIA_ALLOW_REDIRECTS'), self.ALLOW_REDIRECTS ) - self.allow_httpstatus_list = settings.getlist( + self.handle_httpstatus_list = settings.getlist( resolve('MEDIA_HTTPSTATUS_LIST'), [] ) @@ -107,17 +108,15 @@ class MediaPipeline(object): def _modify_media_request(self, request): httpstatus_list = [] - if self.allow_httpstatus_list: - httpstatus_list = self.allow_httpstatus_list - elif self.allow_redirects: + if self.handle_httpstatus_list: + httpstatus_list = self.handle_httpstatus_list + if self.allow_redirects: if not httpstatus_list: - httpstatus_list = list(range(0, 300)) + list(range(400, 1000)) + httpstatus_list = [i for i in range(1000) + if i not in RedirectMiddleware.allowed_status] else: - for i in range(300, 400): - try: - httpstatus_list.remove(i) - except ValueError: - pass + httpstatus_list = [i for i in httpstatus_list + if i not in RedirectMiddleware.allowed_status] if httpstatus_list: request.meta['handle_httpstatus_list'] = httpstatus_list else: diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 41ee9962f..66e98db29 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -89,9 +89,9 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True} request = Request('http://url') - self.pipe.allow_httpstatus_list = list(range(100)) + self.pipe.handle_httpstatus_list = list(range(100)) assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_list': list(range(100))} - self.pipe.allow_httpstatus_list = None + self.pipe.handle_httpstatus_list = None request = Request('http://url') self.pipe.allow_redirects = True From 11b31c9fbddd75fe7fec60152a1bec57acccc039 Mon Sep 17 00:00:00 2001 From: Bernardas Date: Wed, 24 Aug 2016 08:44:56 +0000 Subject: [PATCH 073/109] fix redirect change --- tests/test_pipeline_media.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 66e98db29..f1b8076fd 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -5,6 +5,7 @@ from twisted.python.failure import Failure from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider @@ -95,7 +96,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase): request = Request('http://url') self.pipe.allow_redirects = True - correct = {'handle_httpstatus_list': list(range(300)) + list(range(400,1000))} + correct = {'handle_httpstatus_list': [i for i in range(1000) + if i not in RedirectMiddleware.allowed_status]} assert self.pipe._modify_media_request(request).meta == correct self.pipe.allow_redirects = False From c64ebee06253ef385564c83806bd2422bff6f6d6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Mar 2017 22:40:10 +0100 Subject: [PATCH 074/109] Refactor (WIP) --- scrapy/pipelines/media.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index b11e7095b..0712101b0 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -19,7 +19,6 @@ logger = logging.getLogger(__name__) class MediaPipeline(object): LOG_FAILED_RESULTS = True - ALLOW_REDIRECTS = False class SpiderInfo(object): def __init__(self, spider): @@ -35,12 +34,23 @@ class MediaPipeline(object): resolve = functools.partial(self._key_for_pipe, base_class_name="MediaPipeline") self.allow_redirects = settings.getbool( - resolve('MEDIA_ALLOW_REDIRECTS'), self.ALLOW_REDIRECTS + resolve('MEDIA_ALLOW_REDIRECTS'), False ) self.handle_httpstatus_list = settings.getlist( resolve('MEDIA_HTTPSTATUS_LIST'), [] ) + self.httpstatus_list = [] + if self.handle_httpstatus_list: + self.httpstatus_list = self.handle_httpstatus_list + if self.allow_redirects: + if not self.httpstatus_list: + self.httpstatus_list = [i for i in range(1000) + if i not in RedirectMiddleware.allowed_status] + else: + self.httpstatus_list = [i for i in self.httpstatus_list + if i not in RedirectMiddleware.allowed_status] + def _key_for_pipe(self, key, base_class_name=None, settings=None): """ @@ -107,18 +117,8 @@ class MediaPipeline(object): return dfd.addBoth(lambda _: wad) # it must return wad at last def _modify_media_request(self, request): - httpstatus_list = [] - if self.handle_httpstatus_list: - httpstatus_list = self.handle_httpstatus_list - if self.allow_redirects: - if not httpstatus_list: - httpstatus_list = [i for i in range(1000) - if i not in RedirectMiddleware.allowed_status] - else: - httpstatus_list = [i for i in httpstatus_list - if i not in RedirectMiddleware.allowed_status] - if httpstatus_list: - request.meta['handle_httpstatus_list'] = httpstatus_list + if self.httpstatus_list: + request.meta['handle_httpstatus_list'] = self.httpstatus_list else: request.meta['handle_httpstatus_all'] = True return request From 72fbb687d7a35ddf52f82b165df0a842d0b653e0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 3 Mar 2017 12:31:05 +0100 Subject: [PATCH 075/109] Revert "expose allowed_status tuple for media pipeline" This reverts commit 052809c73ed20b9a728a8fd7df3de5f45f2dad8d. --- scrapy/downloadermiddlewares/redirect.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index ae4ad8891..26677e527 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -57,8 +57,6 @@ class RedirectMiddleware(BaseRedirectMiddleware): Handle redirection of requests based on response status and meta-refresh html tag. """ - allowed_status = (301, 302, 303, 307) - def process_response(self, request, response, spider): if (request.meta.get('dont_redirect', False) or response.status in getattr(spider, 'handle_httpstatus_list', []) or @@ -66,7 +64,8 @@ class RedirectMiddleware(BaseRedirectMiddleware): request.meta.get('handle_httpstatus_all', False)): return response - if 'Location' not in response.headers or response.status not in self.allowed_status: + allowed_status = (301, 302, 303, 307) + if 'Location' not in response.headers or response.status not in allowed_status: return response location = safe_url_string(response.headers['location']) From ecde166ee18935456bcf4c6fce89ab909bbffbaf Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 3 Mar 2017 15:50:34 +0100 Subject: [PATCH 076/109] Refactor without MEDIA_HTTPSTATUS_LIST setting --- docs/topics/media-pipeline.rst | 15 +++++-------- scrapy/pipelines/media.py | 25 ++++++++------------- tests/test_pipeline_media.py | 41 +++++++++++++++++++++++++--------- 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index bfc405d9e..f258ff748 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -322,21 +322,18 @@ By default, there are no size constraints, so all images are processed. .. _topics-media-pipeline-override: -Allowing redirection and handling various http statuses -------------------------------------------------------- +Allowing redirections +--------------------- .. setting:: MEDIA_ALLOW_REDIRECTS -.. setting:: MEDIA_HTTPSTATUS_LIST -By default media pipelines ignore redirects. To allow redirecting(all 300 codes) set: +By default media pipelines ignore redirects, i.e. an HTTP redirection +to a media file URL request will mean the media download is considered failed. + +To handle media redirections, set this settings to ``True``: MEDIA_ALLOW_REDIRECTS = True -To only allow handling only specific codes set (default: any code): - - MEDIA_HTTPSTATUS_LIST = - # example: - MEDIA_HTTPSTATUS_LIST = [303, 404] # will not go through pipelines Extending the Media Pipelines ============================= diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 0712101b0..02daf8d2c 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -6,8 +6,8 @@ from collections import defaultdict from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure -from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.settings import Settings +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter @@ -29,6 +29,7 @@ class MediaPipeline(object): def __init__(self, download_func=None, settings=None): self.download_func = download_func + if isinstance(settings, dict) or settings is None: settings = Settings(settings) resolve = functools.partial(self._key_for_pipe, @@ -36,20 +37,12 @@ class MediaPipeline(object): self.allow_redirects = settings.getbool( resolve('MEDIA_ALLOW_REDIRECTS'), False ) - self.handle_httpstatus_list = settings.getlist( - resolve('MEDIA_HTTPSTATUS_LIST'), [] - ) + self._handle_statuses(self.allow_redirects) - self.httpstatus_list = [] - if self.handle_httpstatus_list: - self.httpstatus_list = self.handle_httpstatus_list - if self.allow_redirects: - if not self.httpstatus_list: - self.httpstatus_list = [i for i in range(1000) - if i not in RedirectMiddleware.allowed_status] - else: - self.httpstatus_list = [i for i in self.httpstatus_list - if i not in RedirectMiddleware.allowed_status] + def _handle_statuses(self, allow_redirects): + self.handle_httpstatus_list = None + if allow_redirects: + self.handle_httpstatus_list = SequenceExclude(range(300, 400)) def _key_for_pipe(self, key, base_class_name=None, settings=None): @@ -117,8 +110,8 @@ class MediaPipeline(object): return dfd.addBoth(lambda _: wad) # it must return wad at last def _modify_media_request(self, request): - if self.httpstatus_list: - request.meta['handle_httpstatus_list'] = self.httpstatus_list + if self.handle_httpstatus_list: + request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list else: request.meta['handle_httpstatus_all'] = True return request diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index f1b8076fd..4797956a0 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -24,11 +24,12 @@ def _mocked_download_func(request, info): class BaseMediaPipelineTestCase(unittest.TestCase): pipeline_class = MediaPipeline + settings = None def setUp(self): self.spider = Spider('media.com') self.pipe = self.pipeline_class(download_func=_mocked_download_func, - settings=Settings()) + settings=Settings(self.settings)) self.pipe.open_spider(self.spider) self.info = self.pipe.spiderinfo @@ -89,17 +90,37 @@ class BaseMediaPipelineTestCase(unittest.TestCase): request = Request('http://url') assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True} - request = Request('http://url') - self.pipe.handle_httpstatus_list = list(range(100)) - assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_list': list(range(100))} - self.pipe.handle_httpstatus_list = None +class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase): + + pipeline_class = MediaPipeline + settings = { + 'MEDIA_ALLOW_REDIRECTS': True + } + + def test_modify_media_request(self): request = Request('http://url') - self.pipe.allow_redirects = True - correct = {'handle_httpstatus_list': [i for i in range(1000) - if i not in RedirectMiddleware.allowed_status]} - assert self.pipe._modify_media_request(request).meta == correct - self.pipe.allow_redirects = False + meta = self.pipe._modify_media_request(request).meta + self.assertIn('handle_httpstatus_list', meta) + for status, check in [ + (200, True), + + # These are the status codes we want + # the downloader to handle itself + (301, False), + (302, False), + (302, False), + (307, False), + (308, False), + + # we still want to get 4xx and 5xx + (400, True), + (404, True), + (500, True)]: + if check: + self.assertIn(status, meta['handle_httpstatus_list']) + else: + self.assertNotIn(status, meta['handle_httpstatus_list']) class MockedMediaPipeline(MediaPipeline): From f7e11b198efd0213bb51205b6829123476ccf2ba Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 3 Mar 2017 16:00:59 +0100 Subject: [PATCH 077/109] Cleanup --- scrapy/pipelines/media.py | 3 +-- tests/test_pipeline_media.py | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 02daf8d2c..921e9e1c9 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -114,7 +114,6 @@ class MediaPipeline(object): request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list else: request.meta['handle_httpstatus_all'] = True - return request def _check_media_to_download(self, result, request, info): if result is not None: @@ -126,7 +125,7 @@ class MediaPipeline(object): callback=self.media_downloaded, callbackArgs=(request, info), errback=self.media_failed, errbackArgs=(request, info)) else: - request = self._modify_media_request(request) + self._modify_media_request(request) dfd = self.crawler.engine.download(request, info.spider) dfd.addCallbacks( callback=self.media_downloaded, callbackArgs=(request, info), diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 4797956a0..cfa2fc42b 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -5,7 +5,6 @@ from twisted.python.failure import Failure from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks -from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider @@ -88,7 +87,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase): def test_modify_media_request(self): request = Request('http://url') - assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True} + self.pipe._modify_media_request(request) + assert request.meta == {'handle_httpstatus_all': True} class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase): @@ -100,8 +100,8 @@ class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase): def test_modify_media_request(self): request = Request('http://url') - meta = self.pipe._modify_media_request(request).meta - self.assertIn('handle_httpstatus_list', meta) + self.pipe._modify_media_request(request) + self.assertIn('handle_httpstatus_list', request.meta) for status, check in [ (200, True), @@ -118,9 +118,9 @@ class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase): (404, True), (500, True)]: if check: - self.assertIn(status, meta['handle_httpstatus_list']) + self.assertIn(status, request.meta['handle_httpstatus_list']) else: - self.assertNotIn(status, meta['handle_httpstatus_list']) + self.assertNotIn(status, request.meta['handle_httpstatus_list']) class MockedMediaPipeline(MediaPipeline): From ef04cfd237ec3d072a487f92e217bad68195f2d8 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 21 Feb 2017 19:55:52 +0300 Subject: [PATCH 078/109] Respect log settings in custom_settings: fixes GH-1612 A new root logger is installed when a crawler is created if one was already installed before. This allows to respect custom settings related to logging, such as LOG_LEVEL, LOG_FILE, etc. --- scrapy/crawler.py | 9 +++++++-- scrapy/utils/log.py | 22 ++++++++++++++++++--- tests/test_crawl.py | 4 ++-- tests/test_crawler.py | 46 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 7 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 443a9aa2f..7b8518832 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -16,7 +16,9 @@ from scrapy.signalmanager import SignalManager from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object -from scrapy.utils.log import LogCounterHandler, configure_logging, log_scrapy_info +from scrapy.utils.log import ( + LogCounterHandler, configure_logging, log_scrapy_info, + get_scrapy_root_handler, install_scrapy_root_handler) from scrapy import signals logger = logging.getLogger(__name__) @@ -35,8 +37,11 @@ class Crawler(object): self.signals = SignalManager(self) self.stats = load_object(self.settings['STATS_CLASS'])(self) - handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL')) + handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL')) logging.root.addHandler(handler) + if get_scrapy_root_handler() is not None: + # scrapy root handler alread installed: update it with new settings + install_scrapy_root_handler(self.settings) # lambda is assigned to Crawler attribute because this way it is not # garbage collected after leaving __init__ scope self.__remove_handler = lambda: logging.root.removeHandler(handler) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index f33ce7017..6ceb61a82 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -96,9 +96,25 @@ def configure_logging(settings=None, install_root_handler=True): sys.stdout = StreamLogger(logging.getLogger('stdout')) if install_root_handler: - logging.root.setLevel(logging.NOTSET) - handler = _get_handler(settings) - logging.root.addHandler(handler) + install_scrapy_root_handler(settings) + + +def install_scrapy_root_handler(settings): + global _scrapy_root_handler + + if (_scrapy_root_handler is not None + and _scrapy_root_handler in logging.root.handlers): + logging.root.removeHandler(_scrapy_root_handler) + logging.root.setLevel(logging.NOTSET) + _scrapy_root_handler = _get_handler(settings) + logging.root.addHandler(_scrapy_root_handler) + + +def get_scrapy_root_handler(): + return _scrapy_root_handler + + +_scrapy_root_handler = None def _get_handler(settings): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d5babdded..3c5d9b958 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -97,8 +97,8 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): + crawler = self.runner.create_crawler(BrokenStartRequestsSpider) with LogCapture('scrapy', level=logging.ERROR) as l: - crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_before_yield=1) self.assertEqual(len(l.records), 1) @@ -108,8 +108,8 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_yielding(self): + crawler = self.runner.create_crawler(BrokenStartRequestsSpider) with LogCapture('scrapy', level=logging.ERROR) as l: - crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_yielding=1) self.assertEqual(len(l.records), 1) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 53a1202e3..ba0d709ff 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,3 +1,6 @@ +import logging +import os +import tempfile import warnings import unittest @@ -5,6 +8,7 @@ import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader +from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object from scrapy.extensions.throttle import AutoThrottle @@ -74,6 +78,48 @@ class SpiderSettingsTestCase(unittest.TestCase): self.assertIn(AutoThrottle, enabled_exts) +class CrawlerLoggingTestCase(unittest.TestCase): + def test_no_root_handler_installed(self): + handler = get_scrapy_root_handler() + if handler is not None: + logging.root.removeHandler(handler) + + class MySpider(scrapy.Spider): + name = 'spider' + + crawler = Crawler(MySpider, {}) + assert get_scrapy_root_handler() is None + + def test_spider_custom_settings_log_level(self): + with tempfile.NamedTemporaryFile() as log_file: + class MySpider(scrapy.Spider): + name = 'spider' + custom_settings = { + 'LOG_LEVEL': 'INFO', + 'LOG_FILE': log_file.name, + } + + configure_logging() + self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) + crawler = Crawler(MySpider, {}) + self.assertEqual(get_scrapy_root_handler().level, logging.INFO) + info_count = crawler.stats.get_value('log_count/INFO') + logging.debug('debug message') + logging.info('info message') + logging.warning('warning message') + logging.error('error message') + logged = log_file.read().decode('utf8') + self.assertNotIn('debug message', logged) + self.assertIn('info message', logged) + self.assertIn('warning message', logged) + self.assertIn('error message', logged) + self.assertEqual(crawler.stats.get_value('log_count/ERROR'), 1) + self.assertEqual(crawler.stats.get_value('log_count/WARNING'), 1) + self.assertEqual( + crawler.stats.get_value('log_count/INFO') - info_count, 1) + self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0) + + class SpiderLoaderWithWrongInterface(object): def unneeded_method(self): From c3b6feca0e15309386c080d71b0b03a07d4c8635 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 3 Mar 2017 16:29:07 +0100 Subject: [PATCH 079/109] Fix setting lookup for MEDIA_ALLOWED_REDIRECTS --- scrapy/pipelines/media.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 921e9e1c9..404bbf5bf 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -33,7 +33,8 @@ class MediaPipeline(object): if isinstance(settings, dict) or settings is None: settings = Settings(settings) resolve = functools.partial(self._key_for_pipe, - base_class_name="MediaPipeline") + base_class_name="MediaPipeline", + settings=settings) self.allow_redirects = settings.getbool( resolve('MEDIA_ALLOW_REDIRECTS'), False ) From c68f99eed843bd35224d8bf0e22b0c66bdc2122b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 3 Mar 2017 17:03:25 +0100 Subject: [PATCH 080/109] Refactor settings tests --- tests/test_pipeline_media.py | 90 +++++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index cfa2fc42b..5f6a6d9e6 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -91,38 +91,6 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert request.meta == {'handle_httpstatus_all': True} -class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase): - - pipeline_class = MediaPipeline - settings = { - 'MEDIA_ALLOW_REDIRECTS': True - } - - def test_modify_media_request(self): - request = Request('http://url') - self.pipe._modify_media_request(request) - self.assertIn('handle_httpstatus_list', request.meta) - for status, check in [ - (200, True), - - # These are the status codes we want - # the downloader to handle itself - (301, False), - (302, False), - (302, False), - (307, False), - (308, False), - - # we still want to get 4xx and 5xx - (400, True), - (404, True), - (500, True)]: - if check: - self.assertIn(status, request.meta['handle_httpstatus_list']) - else: - self.assertNotIn(status, request.meta['handle_httpstatus_list']) - - class MockedMediaPipeline(MediaPipeline): def __init__(self, *args, **kwargs): @@ -289,3 +257,61 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): self.assertEqual(new_item['results'], [(True, 'ITSME')]) self.assertEqual(self.pipe._mockcalled, \ ['get_media_requests', 'media_to_download', 'item_completed']) + + +class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): + + def _assert_request_no3xx(self, pipeline_class, settings): + pipe = pipeline_class(settings=Settings(settings)) + request = Request('http://url') + pipe._modify_media_request(request) + + self.assertIn('handle_httpstatus_list', request.meta) + for status, check in [ + (200, True), + + # These are the status codes we want + # the downloader to handle itself + (301, False), + (302, False), + (302, False), + (307, False), + (308, False), + + # we still want to get 4xx and 5xx + (400, True), + (404, True), + (500, True)]: + if check: + self.assertIn(status, request.meta['handle_httpstatus_list']) + else: + self.assertNotIn(status, request.meta['handle_httpstatus_list']) + + def test_standard_setting(self): + self._assert_request_no3xx( + MediaPipeline, + { + 'MEDIA_ALLOW_REDIRECTS': True + }) + + def test_subclass_standard_setting(self): + + class UserDefinedPipeline(MediaPipeline): + pass + + self._assert_request_no3xx( + UserDefinedPipeline, + { + 'MEDIA_ALLOW_REDIRECTS': True + }) + + def test_subclass_specific_setting(self): + + class UserDefinedPipeline(MediaPipeline): + pass + + self._assert_request_no3xx( + UserDefinedPipeline, + { + 'USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS': True + }) From 30d812eea233914b3aead55b8053a9c804e594d3 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 3 Mar 2017 17:15:37 +0100 Subject: [PATCH 081/109] Remove redundant slot.add_request() call in ExecutionEngine --- scrapy/core/engine.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 2b5770138..37fe0a873 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -218,10 +218,8 @@ class ExecutionEngine(object): request=request, spider=spider) def download(self, request, spider): - slot = self.slot - slot.add_request(request) d = self._download(request, spider) - d.addBoth(self._downloaded, slot, request, spider) + d.addBoth(self._downloaded, self.slot, request, spider) return d def _downloaded(self, response, slot, request, spider): From a8b47d6c689cd8f221acb4caf3974ff0057b6a2d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 6 Mar 2017 14:25:52 +0100 Subject: [PATCH 082/109] Update release notes for 1.0.7, 1.1.4 and 1.2.3 --- docs/news.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index ff1e4ce03..31f4d3026 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -101,6 +101,12 @@ Dependencies & Cleanups downloader middlewares. +Scrapy 1.2.3 (2017-03-03) +------------------------- + +- Packaging fix: disallow unsupported Twisted versions in setup.py + + Scrapy 1.2.2 (2016-12-06) ------------------------- @@ -229,6 +235,12 @@ Documentation - Add StackOverflow as a support channel (:issue:`2257`). +Scrapy 1.1.4 (2017-03-03) +------------------------- + +- Packaging fix: disallow unsupported Twisted versions in setup.py + + Scrapy 1.1.3 (2016-09-22) ------------------------- @@ -501,6 +513,12 @@ Bugfixes to same remote host (:issue:`1912`). +Scrapy 1.0.7 (2017-03-03) +------------------------- + +- Packaging fix: disallow unsupported Twisted versions in setup.py + + Scrapy 1.0.6 (2016-05-04) ------------------------- From 768f3155e57ed54419f0c137fc066a95197d1b46 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 6 Mar 2017 16:20:37 +0100 Subject: [PATCH 083/109] Fix referrer policy from response headers and support explicit empty string --- scrapy/spidermiddlewares/referer.py | 8 ++++++-- tests/test_spidermiddleware_referer.py | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index b444e34bb..1ddfb37f4 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -261,6 +261,9 @@ _policy_classes = {p.name: p for p in ( DefaultReferrerPolicy, )} +# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string +_policy_classes[''] = NoReferrerWhenDowngradePolicy + def _load_policy_class(policy, warning_only=False): """ @@ -317,8 +320,9 @@ class RefererMiddleware(object): policy_name = request.meta.get('referrer_policy') if policy_name is None: if isinstance(resp_or_url, Response): - policy_name = to_native_str( - resp_or_url.headers.get('Referrer-Policy', '').decode('latin1')) + policy_header = resp_or_url.headers.get('Referrer-Policy') + if policy_header is not None: + policy_name = to_native_str(policy_header.decode('latin1')) if policy_name is None: return self.default_policy() diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index b1c815876..f27f31b74 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -526,6 +526,13 @@ class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMid settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} +class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + """ + The empty string means "no-referrer-when-downgrade" + """ + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + resp_headers = {'Referrer-Policy': ''} + class TestReferrerOnRedirect(TestRefererMiddleware): From 2a7d391e0b379143810fdcaed241664e159e0d1d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 6 Mar 2017 17:30:32 +0100 Subject: [PATCH 084/109] DOC Mention brotli support in HttpCompressionMiddleware section --- docs/topics/downloader-middleware.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0ef3fb071..c3a454279 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -645,6 +645,12 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. + This middleware also supports decoding `brotli-compressed`_ responses, + provided `brotlipy`_ is installed. + +.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt +.. _brotlipy: https://pypi.python.org/pypi/brotlipy + HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From e42b846a9fc68a624e94052aac2ba44224b05cad Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 10 Nov 2016 16:26:55 +0100 Subject: [PATCH 085/109] Use body to chose response type after decompression content --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- .../test_downloadermiddleware_httpcompression.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 19d6345e4..eb00d8923 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -38,7 +38,7 @@ class HttpCompressionMiddleware(object): encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) respcls = responsetypes.from_args(headers=response.headers, \ - url=response.url) + url=response.url, body=decoded_body) kwargs = dict(cls=respcls, body=decoded_body) if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 7924fb3b5..5403e8f52 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -7,6 +7,7 @@ from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \ ACCEPTED_ENCODINGS +from scrapy.responsetypes import responsetypes from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -152,6 +153,20 @@ class HttpCompressionTest(TestCase): self.assertEqual(newresponse.body, plainbody) self.assertEqual(newresponse.encoding, resolve_encoding('gb2312')) + def test_process_response_no_content_type_header(self): + headers = { + 'Content-Encoding': 'identity', + } + plainbody = b"""Some page""" + respcls = responsetypes.from_args(url="http://www.example.com/index", headers=headers, body=plainbody) + response = respcls("http://www.example.com/index", headers=headers, body=plainbody) + request = Request("http://www.example.com/index") + + newresponse = self.mw.process_response(request, response, self.spider) + assert isinstance(newresponse, respcls) + self.assertEqual(newresponse.body, plainbody) + self.assertEqual(newresponse.encoding, resolve_encoding('gb2312')) + def test_process_response_gzipped_contenttype(self): response = self._getresponse('gzip') response.headers['Content-Type'] = 'application/gzip' From 11cdf58abe26a9f65d8c1aefa717b6735e5734c9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 9 Nov 2016 23:08:23 +0100 Subject: [PATCH 086/109] Always decompress Content-Encoding: gzip at HttpCompression stage Let SitemapSpider handle decoding of .xml.gz files if necessary --- .../downloadermiddlewares/httpcompression.py | 2 +- scrapy/spiders/sitemap.py | 23 +++++-- scrapy/utils/gz.py | 4 ++ ...st_downloadermiddleware_httpcompression.py | 61 ++++++++++++++++--- tests/test_spider.py | 4 ++ 5 files changed, 78 insertions(+), 16 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index eb00d8923..dd32c62de 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -34,7 +34,7 @@ class HttpCompressionMiddleware(object): return response if isinstance(response, Response): content_encoding = response.headers.getlist('Content-Encoding') - if content_encoding and not is_gzipped(response): + if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) respcls = responsetypes.from_args(headers=response.headers, \ diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 9e45637c3..10af90259 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -5,7 +5,8 @@ import six from scrapy.spiders import Spider from scrapy.http import Request, XmlResponse from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots -from scrapy.utils.gz import gunzip, is_gzipped +from scrapy.utils.gz import gunzip, gzip_magic_number + logger = logging.getLogger(__name__) @@ -59,12 +60,22 @@ class SitemapSpider(Spider): """ if isinstance(response, XmlResponse): return response.body - elif is_gzipped(response): - return gunzip(response.body) - elif response.url.endswith('.xml'): + elif gzip_magic_number(response): + try: + return gunzip(response.body) + except: + pass + # actual gzipped sitemap files are decompressed above ; + # if we are here (response body is not gzipped) + # and have a response for .xml.gz, + # it usually means that it was already gunzipped + # by HttpCompression middleware, + # the HTTP response being sent with "Content-Encoding: gzip" + # without actually being a .xml.gz file in the first place, + # merely XML gzip-compressed on the fly, + # in other word, here, we have plain XML + elif response.url.endswith('.xml') or response.url.endswith('.xml.gz'): return response.body - elif response.url.endswith('.xml.gz'): - return gunzip(response.body) def regex(x): diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 73c2eb73b..22cf58986 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -59,3 +59,7 @@ def is_gzipped(response): cenc = response.headers.get('Content-Encoding', b'').lower() return (_is_gzipped(ctype) or (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip'))) + + +def gzip_magic_number(response): + return response.body[:2] == b'\x1f\x8b' diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 5403e8f52..5f56c99ec 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -8,6 +8,7 @@ from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \ ACCEPTED_ENCODINGS from scrapy.responsetypes import responsetypes +from scrapy.utils.gz import gunzip from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -173,9 +174,9 @@ class HttpCompressionTest(TestCase): request = response.request newresponse = self.mw.process_response(request, response, self.spider) - self.assertIs(newresponse, response) - self.assertEqual(response.headers['Content-Encoding'], b'gzip') - self.assertEqual(response.headers['Content-Type'], b'application/gzip') + assert newresponse is not response + assert newresponse.body.startswith(b' + + + http://www.example.com/ + 2009-08-16 + daily + 1 + + + http://www.example.com/Special-Offers.html + 2009-08-16 + weekly + 0.8 + +""" + gz_file = GzipFile(fileobj=f, mode='wb') + gz_file.write(plainbody) + gz_file.close() + + # build a gzipped response body containing this gzipped file + r = BytesIO() + gz_resp = GzipFile(fileobj=r, mode='wb') + gz_resp.write(f.getvalue()) + gz_resp.close() + + response = Response("http;//www.example.com/", headers=headers, body=r.getvalue()) + request = Request("http://www.example.com/") + + newresponse = self.mw.process_response(request, response, self.spider) + self.assertEqual(gunzip(newresponse.body), plainbody) def test_process_response_head_request_no_decode_required(self): response = self._getresponse('gzip') diff --git a/tests/test_spider.py b/tests/test_spider.py index 371b8c1ac..e55f0fa6d 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -328,6 +328,10 @@ class SitemapSpiderTest(SpiderTest): r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY) self.assertSitemapBody(r, self.BODY) + # .xml.gz but body decoded by HttpCompression middleware already + r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.BODY) + self.assertSitemapBody(r, self.BODY) + def test_get_sitemap_urls_from_robotstxt(self): robots = b"""# Sitemap files Sitemap: http://example.com/sitemap.xml From b174744b80fd12efc6e4008ea5eb38256f21038a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 10 Nov 2016 10:50:34 +0100 Subject: [PATCH 087/109] Do not silently fail on gzip unzipping --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- scrapy/spiders/sitemap.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index dd32c62de..203dee42d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,6 @@ import zlib -from scrapy.utils.gz import gunzip, is_gzipped +from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.exceptions import NotConfigured diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 10af90259..e54001d88 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -61,10 +61,7 @@ class SitemapSpider(Spider): if isinstance(response, XmlResponse): return response.body elif gzip_magic_number(response): - try: - return gunzip(response.body) - except: - pass + return gunzip(response.body) # actual gzipped sitemap files are decompressed above ; # if we are here (response body is not gzipped) # and have a response for .xml.gz, From 4caceccd594f2563a3d32ea708579ce8e70132f3 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 7 Mar 2017 10:51:34 +0100 Subject: [PATCH 088/109] Use 3-bytes for gzip archive type sniffing --- scrapy/utils/gz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 22cf58986..16c9ce539 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -62,4 +62,4 @@ def is_gzipped(response): def gzip_magic_number(response): - return response.body[:2] == b'\x1f\x8b' + return response.body[:3] == b'\x1f\x8b\x08' From 6f55ca4643dfff7a163714d696eef66d06cb81a8 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 7 Mar 2017 14:20:52 +0300 Subject: [PATCH 089/109] Revert unneeded test_crawl changes --- tests/test_crawl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3c5d9b958..d5babdded 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -97,8 +97,8 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): - crawler = self.runner.create_crawler(BrokenStartRequestsSpider) with LogCapture('scrapy', level=logging.ERROR) as l: + crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_before_yield=1) self.assertEqual(len(l.records), 1) @@ -108,8 +108,8 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_yielding(self): - crawler = self.runner.create_crawler(BrokenStartRequestsSpider) with LogCapture('scrapy', level=logging.ERROR) as l: + crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_yielding=1) self.assertEqual(len(l.records), 1) From b6378c7ef6393412165239fd6bf489d45a1c5196 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 7 Mar 2017 12:28:24 +0100 Subject: [PATCH 090/109] Revert to using self.assert methods --- ...est_downloadermiddleware_httpcompression.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 5f56c99ec..0678fcb14 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -174,9 +174,9 @@ class HttpCompressionTest(TestCase): request = response.request newresponse = self.mw.process_response(request, response, self.spider) - assert newresponse is not response - assert newresponse.body.startswith(b' Date: Tue, 7 Mar 2017 14:44:27 +0100 Subject: [PATCH 091/109] Warn about modules where duplicate spider names were found --- scrapy/spiderloader.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 1322c01d1..27a909c9f 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +from collections import defaultdict import traceback import warnings @@ -19,14 +20,21 @@ class SpiderLoader(object): def __init__(self, settings): self.spider_modules = settings.getlist('SPIDER_MODULES') self._spiders = {} + self._found = defaultdict(list) self._load_all_spiders() def _load_spiders(self, module): for spcls in iter_spider_classes(module): + self._found[spcls.name].append((module.__name__, spcls.__name__)) if spcls.name in self._spiders.keys(): import warnings - warnings.warn("There are several spiders with the same name (" + spcls.name + - "), this can cause unexpected behavior", UserWarning) + msg = ("There are several spiders with the same name {!r}:\n" + "{}\n This can cause unexpected behavior.".format( + spcls.name, + "\n".join( + " {1} (in {0})".format(mod, cls) + for (mod, cls) in self._found[spcls.name]))) + warnings.warn(msg, UserWarning) self._spiders[spcls.name] = spcls def _load_all_spiders(self): From 978306a2236403f6275bad11e85d1e42ca37d6f1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 7 Mar 2017 14:48:16 +0100 Subject: [PATCH 092/109] Fix dupe spider name warning string tests --- tests/test_spiderloader/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 302bf9a10..4600e53dc 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -127,7 +127,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): spider_loader = SpiderLoader.from_settings(self.settings) self.assertEqual(len(w), 1) - self.assertIn("several spiders with the same name (spider3)", str(w[0].message)) + self.assertIn("several spiders with the same name 'spider3'", str(w[0].message)) spiders = set(spider_loader.list()) self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) @@ -145,8 +145,8 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): self.assertEqual(len(w), 2) msgs = sorted(str(wrn.message) for wrn in w) - self.assertIn("several spiders with the same name (spider1)", msgs[0]) - self.assertIn("several spiders with the same name (spider2)", msgs[1]) + self.assertIn("several spiders with the same name 'spider1'", msgs[0]) + self.assertIn("several spiders with the same name 'spider2'", msgs[1]) spiders = set(spider_loader.list()) self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) From 0b9a18e1a120751b8a2b2f1cc04c62b97967d612 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 7 Mar 2017 15:41:17 +0100 Subject: [PATCH 093/109] Warn only once for all spiders --- scrapy/spiderloader.py | 22 +++++++++++++--------- tests/test_spiderloader/__init__.py | 13 ++++++++----- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 27a909c9f..486a4637e 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -23,18 +23,21 @@ class SpiderLoader(object): self._found = defaultdict(list) self._load_all_spiders() + def _check_name_duplicates(self): + dupes = ["\n".join(" {cls} named {name!r} (in {module})".format( + module=mod, cls=cls, name=name) + for (mod, cls) in locations) + for name, locations in self._found.items() + if len(locations)>1] + if dupes: + msg = ("There are several spiders with the same name:\n\n" + "{}\n\n This can cause unexpected behavior.".format( + "\n\n".join(dupes))) + warnings.warn(msg, UserWarning) + def _load_spiders(self, module): for spcls in iter_spider_classes(module): self._found[spcls.name].append((module.__name__, spcls.__name__)) - if spcls.name in self._spiders.keys(): - import warnings - msg = ("There are several spiders with the same name {!r}:\n" - "{}\n This can cause unexpected behavior.".format( - spcls.name, - "\n".join( - " {1} (in {0})".format(mod, cls) - for (mod, cls) in self._found[spcls.name]))) - warnings.warn(msg, UserWarning) self._spiders[spcls.name] = spcls def _load_all_spiders(self): @@ -47,6 +50,7 @@ class SpiderLoader(object): "Check SPIDER_MODULES setting".format( modname=name, tb=traceback.format_exc())) warnings.warn(msg, RuntimeWarning) + self._check_name_duplicates() @classmethod def from_settings(cls, settings): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 4600e53dc..673a2d302 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -127,7 +127,9 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): spider_loader = SpiderLoader.from_settings(self.settings) self.assertEqual(len(w), 1) - self.assertIn("several spiders with the same name 'spider3'", str(w[0].message)) + msg = str(w[0].message) + self.assertIn("several spiders with the same name", msg) + self.assertIn("'spider3'", msg) spiders = set(spider_loader.list()) self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) @@ -143,10 +145,11 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) - self.assertEqual(len(w), 2) - msgs = sorted(str(wrn.message) for wrn in w) - self.assertIn("several spiders with the same name 'spider1'", msgs[0]) - self.assertIn("several spiders with the same name 'spider2'", msgs[1]) + self.assertEqual(len(w), 1) + msg = str(w[0].message) + self.assertIn("several spiders with the same name", msg) + self.assertIn("'spider1'", msg) + self.assertIn("'spider2'", msg) spiders = set(spider_loader.list()) self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) From 7be773e14ae27501914c3f3922f5504a7ac72f48 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 7 Mar 2017 17:40:40 +0100 Subject: [PATCH 094/109] Add SPIDER_LOADER_WARN_ONLY to toggle between spiderloader failure and warning --- scrapy/commands/list.py | 3 ++- scrapy/commands/runspider.py | 1 + scrapy/commands/settings.py | 3 ++- scrapy/commands/startproject.py | 5 +++-- scrapy/commands/version.py | 3 ++- scrapy/settings/default_settings.py | 1 + scrapy/spiderloader.py | 12 ++++++++---- tests/test_spiderloader/__init__.py | 11 +++++++++-- 8 files changed, 28 insertions(+), 11 deletions(-) diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index a255b3b94..185a77a40 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -4,7 +4,8 @@ from scrapy.commands import ScrapyCommand class Command(ScrapyCommand): requires_project = True - default_settings = {'LOG_ENABLED': False} + default_settings = {'LOG_ENABLED': False, + 'SPIDER_LOADER_WARN_ONLY': True} def short_desc(self): return "List available spiders" diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 1da09e4da..a98033dd1 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -28,6 +28,7 @@ def _import_file(filepath): class Command(ScrapyCommand): requires_project = False + default_settings = {'SPIDER_LOADER_WARN_ONLY': True} def syntax(self): return "[options] " diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index bce4e6086..bee52f06a 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -7,7 +7,8 @@ from scrapy.settings import BaseSettings class Command(ScrapyCommand): requires_project = False - default_settings = {'LOG_ENABLED': False} + default_settings = {'LOG_ENABLED': False, + 'SPIDER_LOADER_WARN_ONLY': True} def syntax(self): return "[options]" diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 594106632..c17aaf442 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -26,7 +26,8 @@ IGNORE = ignore_patterns('*.pyc', '.svn') class Command(ScrapyCommand): requires_project = False - default_settings = {'LOG_ENABLED': False} + default_settings = {'LOG_ENABLED': False, + 'SPIDER_LOADER_WARN_ONLY': True} def syntax(self): return " [project_dir]" @@ -118,4 +119,4 @@ class Command(ScrapyCommand): _templates_base_dir = self.settings['TEMPLATES_DIR'] or \ join(scrapy.__path__[0], 'templates') return join(_templates_base_dir, 'project') - + diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index a9954edb0..e22f98f5a 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -11,7 +11,8 @@ from scrapy.commands import ScrapyCommand class Command(ScrapyCommand): - default_settings = {'LOG_ENABLED': False} + default_settings = {'LOG_ENABLED': False, + 'SPIDER_LOADER_WARN_ONLY': True} def syntax(self): return "[-v]" diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d73c595d2..854cefc9c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -250,6 +250,7 @@ SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue' SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue' SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' +SPIDER_LOADER_WARN_ONLY = False SPIDER_MIDDLEWARES = {} diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 486a4637e..7478faa78 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -19,6 +19,7 @@ class SpiderLoader(object): """ def __init__(self, settings): self.spider_modules = settings.getlist('SPIDER_MODULES') + self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY') self._spiders = {} self._found = defaultdict(list) self._load_all_spiders() @@ -46,10 +47,13 @@ class SpiderLoader(object): for module in walk_modules(name): self._load_spiders(module) except ImportError as e: - msg = ("\n{tb}Could not load spiders from module '{modname}'. " - "Check SPIDER_MODULES setting".format( - modname=name, tb=traceback.format_exc())) - warnings.warn(msg, RuntimeWarning) + if self.warn_only: + msg = ("\n{tb}Could not load spiders from module '{modname}'. " + "See above traceback for details.".format( + modname=name, tb=traceback.format_exc())) + warnings.warn(msg, RuntimeWarning) + else: + raise self._check_name_duplicates() @classmethod diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 673a2d302..99a61daea 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -91,18 +91,25 @@ class SpiderLoaderTest(unittest.TestCase): self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) self.assertEqual(crawler.spidercls.name, 'spider1') + def test_bad_spider_modules_exception(self): + + module = 'tests.test_spiderloader.test_spiders.doesnotexist' + settings = Settings({'SPIDER_MODULES': [module]}) + with self.assertRaises(ImportError): + SpiderLoader.from_settings(settings) + def test_bad_spider_modules_warning(self): with warnings.catch_warnings(record=True) as w: module = 'tests.test_spiderloader.test_spiders.doesnotexist' - settings = Settings({'SPIDER_MODULES': [module]}) + settings = Settings({'SPIDER_MODULES': [module], + 'SPIDER_LOADER_WARN_ONLY': True}) spider_loader = SpiderLoader.from_settings(settings) self.assertIn("Could not load spiders from module", str(w[0].message)) spiders = spider_loader.list() self.assertEqual(spiders, []) - class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self): From ac63d3a3cf0a45b9ae66c9847d07d287154078e6 Mon Sep 17 00:00:00 2001 From: jorenham Date: Thu, 9 Mar 2017 10:56:23 +0100 Subject: [PATCH 095/109] Removed contrib section; contrib is deprecated --- docs/contributing.rst | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index b0a435ad2..ab3779395 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -124,15 +124,6 @@ Scrapy: * Don't put your name in the code you contribute. Our policy is to keep the contributor's name in the `AUTHORS`_ file distributed with Scrapy. -Scrapy Contrib -============== - -Scrapy contrib shares a similar rationale as Django contrib, which is explained -in `this post `_. If you -are working on a new functionality, please follow that rationale to decide -whether it should be a Scrapy contrib. If unsure, you can ask in -`scrapy-users`_. - Documentation policies ====================== From 9cfe9ae0989069b1a7a2ae0af916ffccb5a8bcee Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 9 Mar 2017 12:21:03 +0100 Subject: [PATCH 096/109] Do not use self.assertRaises() as context manager --- tests/test_spiderloader/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 99a61daea..1cd59b99a 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -95,8 +95,7 @@ class SpiderLoaderTest(unittest.TestCase): module = 'tests.test_spiderloader.test_spiders.doesnotexist' settings = Settings({'SPIDER_MODULES': [module]}) - with self.assertRaises(ImportError): - SpiderLoader.from_settings(settings) + self.assertRaises(ImportError, SpiderLoader.from_settings, settings) def test_bad_spider_modules_warning(self): From f2ac24eb7b353d1140729ba7d8c81ad9635d5899 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 9 Mar 2017 17:38:15 +0100 Subject: [PATCH 097/109] Do not only warn on wrong spider modules for "scrapy list" --- scrapy/commands/list.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 185a77a40..a255b3b94 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -4,8 +4,7 @@ from scrapy.commands import ScrapyCommand class Command(ScrapyCommand): requires_project = True - default_settings = {'LOG_ENABLED': False, - 'SPIDER_LOADER_WARN_ONLY': True} + default_settings = {'LOG_ENABLED': False} def short_desc(self): return "List available spiders" From 9628a739723983f2d3c4079d228dcaa79f558e73 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 9 Mar 2017 17:40:34 +0100 Subject: [PATCH 098/109] Update settings docs for new SPIDER_LOADER_WARN_ONLY --- docs/topics/settings.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ccdd02c4e..569b71518 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1180,6 +1180,29 @@ Default: ``'scrapy.spiderloader.SpiderLoader'`` The class that will be used for loading spiders, which must implement the :ref:`topics-api-spiderloader`. +.. setting:: SPIDER_LOADER_WARN_ONLY + +SPIDER_LOADER_WARN_ONLY +----------------------- + +.. versionadded:: 1.4 + +Default: ``False`` + +By default, when scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, +it will fail loudly if there is any ``ImportError`` exception. +But you can choose to silence this exception and turn it into a simple +warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. + +.. note:: + Some :ref:`scrapy commands ` run with this setting to ``True`` + already (i.e. they will only issue a warning and will not fail) + since they do not actually need to load spider classes to work: + :command:`scrapy runspider `, + :command:`scrapy settings `, + :command:`scrapy startproject `, + :command:`scrapy version `. + .. setting:: SPIDER_MIDDLEWARES SPIDER_MIDDLEWARES From d8865b33045c239e18704d7ab5012e334b128a32 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 10 Mar 2017 14:02:00 +0100 Subject: [PATCH 099/109] Update changelog for upcoming 1.3.3 --- docs/news.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 31f4d3026..f9a900771 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,13 +3,25 @@ Release notes ============= +Scrapy 1.3.3 (2017-03-XX) +------------------------- + +Bug fixes +~~~~~~~~~ + +- Make ``SpiderLoader`` raise ``ImportError`` again by default for missing + dependencies and wrong :setting:`SPIDER_MODULES`. + These exceptions were silenced as warnings since 1.3.0. + A new setting is introduced to toggle between warning or exception if needed ; + see :setting:`SPIDER_LOADER_WARN_ONLY` for details. + Scrapy 1.3.2 (2017-02-13) ------------------------- Bug fixes ~~~~~~~~~ -- Preserve crequest class when converting to/from dicts (utils.reqser) (:issue:`2510`). +- Preserve request class when converting to/from dicts (utils.reqser) (:issue:`2510`). - Use consistent selectors for author field in tutorial (:issue:`2551`). - Fix TLS compatibility in Twisted 17+ (:issue:`2558`) From b2c505d8ec71fd1226782c65edd06d901c3ef211 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 10 Mar 2017 16:29:44 +0100 Subject: [PATCH 100/109] Set release date in changelog for v1.3.3 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index f9a900771..da856d883 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,7 +3,7 @@ Release notes ============= -Scrapy 1.3.3 (2017-03-XX) +Scrapy 1.3.3 (2017-03-10) ------------------------- Bug fixes From a7f5207e9f2837040a3a08c3a8f4e76265b68bb2 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 10 Mar 2017 12:25:58 +0100 Subject: [PATCH 101/109] Update version added for SPIDER_LOADER_WARN_ONLY --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 569b71518..8367b1092 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1185,7 +1185,7 @@ The class that will be used for loading spiders, which must implement the SPIDER_LOADER_WARN_ONLY ----------------------- -.. versionadded:: 1.4 +.. versionadded:: 1.3.3 Default: ``False`` From 7dcc86e61adf37fdfb77b00375040fe25e7ad832 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 10 Mar 2017 21:35:25 +0100 Subject: [PATCH 102/109] Add file listing resource + redirecting resource to MockServer --- tests/mockserver.py | 16 ++++++++++++++++ .../python-logo-master-v3-TM-flattened.png | Bin 0 -> 11155 bytes .../files/images/python-powered-h-50x65.png | Bin 0 -> 3243 bytes .../test_site/files/images/scrapy.png | Bin 0 -> 2710 bytes 4 files changed, 16 insertions(+) create mode 100644 tests/sample_data/test_site/files/images/python-logo-master-v3-TM-flattened.png create mode 100644 tests/sample_data/test_site/files/images/python-powered-h-50x65.png create mode 100644 tests/sample_data/test_site/files/images/scrapy.png diff --git a/tests/mockserver.py b/tests/mockserver.py index e611cc3ec..26ab51183 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -5,13 +5,17 @@ from subprocess import Popen, PIPE from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource +from twisted.web.static import File from twisted.web.test.test_webclient import PayloadResource from twisted.web.server import GzipEncoderFactory from twisted.web.resource import EncodingResourceWrapper +from twisted.web.util import redirectTo from twisted.internet import reactor, ssl from twisted.internet.task import deferLater + from scrapy.utils.python import to_bytes, to_unicode +from tests import tests_datadir def getarg(request, name, default=None, type=None): @@ -120,6 +124,16 @@ class Echo(LeafResource): return to_bytes(json.dumps(output)) +class RedirectTo(LeafResource): + + def render(self, request): + goto = getarg(request, b'goto', b'/') + # we force the body content, otherwise Twisted redirectTo() + # returns HTML with )mfY@00XQM~#e_i5LqDi%dgZMIQ?b+Z22sAjAis zmi_Vf!5>^-B@F{Y@Cqe#NCdx$ywpv7v9O2_{=Fee<>J}kB;6}jlUK?0TYDc*cSnbpw(L({`Zxx7`ndYC zYn!l(3JZ%T_?Hi3VX(%->L@DUDIuIY)C`>CCbyk_-T<}6^)I*C_H@QqdIY1*i>j@X(n9|*#px% zxcYZq+^^6(T%6O_J8ke+2j`4c&-7?jhPr8u#C44H+_G1AD~D4&%N_=2YL#@cHE7OK zpyP7k(<5fmZeax#@G6*@+Q+A)qM{xhK7TKgZV~f=$6MSsIlVkN0Et% zT?Hs9z6@e$he33Oa=hAq-Vxh^)RKw0WgQ9I&5eHcPSa z2n^f&Ex<7Up(}Uv?o?(o_s;+zEAc6Yn)s0qzWkEcM?pE^Ycjuc%sa<=g3WEPM~DsL zn}#<-$ug0df`6)CExWk39F2J*f0a+R3HXsLNCZo>w(frpJPX)YT+hAxu2mj9E-srs zraMl*E0(s{ZkRHR1#PyJa5f2b1tA06p-rs&qqFs6jc?mAIMdd&0JN=s3MJZ%5=r(#P zs14&N0d&CYtM&Z9aW6V<`Ij+I!6=n~qpBjRrm0uy?4AV7#U~|%6MPumooWuE&J=uj zaN;Ypa-!9&9F!I57(sTB`<~WINrn{1`g&6L@L+$hYxAx-PBH=C6l;zcYD}nnodc#+Y2{I zi)?l+dgfhM8&mAb$=Qc~xX>~+%&)USwNpvbLsPiGFb*CRM-hh(E9dHv#6&8h=Te)R zJEsx_<_x?#?5&2uM)ub!Rb9H{$=q*;hKBrha3Y(U9{)#^d$W1Z3^_N)vcA49Ge9bJ z>cHLCl!9D(NQ!aZvqDBXe(YOJEuk~o_oj1J3Q0Y#XBl|JmKW9zgtxdd?NRvkI}-KG zFyoPXv7eKhTe*9SgL7y z8+TP=+-z+jMfypQ6!kRdKpGd($6~+M;!-~1I3+eSTEX~ouER#{-+#v(3;6higzmLQ zI!gYBUM9SLCj%e2xzS)lUYh(;m)UlgdScT|YU%0uWDb7s^7oT(CS>t-a>FMueeX+# zIZY6vEWcWwCSr(9SY4e3+B4^f-l(Yj`D7tTU}&gUg*3l3FhsX*3u`dcr1#mpkk2`s9_OmVhs%LJ~HlzRG={VIOP~j&< zF9x=8(crl#Cv!X~h815%PqH)-N3_BU8~ky!Bb0Rj-Y|57Hp4u}^57Wvt9aI^NZMba z0{bguPoSKIenRj2v780vg17%XV37-L92Ov^P&WF@klFfoVc=DfH;xli7K^p3TwrYe z2zF*jwmkxbi18>K06-^jV^O|XI&LGa!L`90WaP~M+7nK z3KxW(l2EzSwY9YZlZMV@SyQ6F%&;qL9dpK#BS<`Ti_C(T3jr8Z#FH1Q?;OFG;R8ka zhw2MEa3>*~4RCxuu~e~&Y()0JKp5&y;^0qeCX@^#G0CiNotu3EoNXwOd2tm)YxV3j z(lIczeo`Bpv;Zo(D#q5o1l~VNkzNR&oQO631~ek_-)t!yDLFxC8zLc5 zzA5u31+Ya2zcit^Nxv}OJ*_AGdSw$c9`klgjEsz>d8(Gk>ekk|>WH)mHd3*8=+W}S z&V}WA4{SOyBldbTGH^Hz=c-m1#Z;n3{BAGhi^pH~)sU*{r46iI92MyQx55{c@`-C} zT^TB)l4KD$G1mp|3m;X->n1xGC~LJiwfC;AIXZvL{I-aP>c{V1 zsz32=_SwbP=?G;_*$}kfQu?w>C;6mTtb{}AvcWhtZTPESx6$O$RUA^tuIQ$CpebjY zj>zP9@9pN|?fd%qEFCPQ^~T(-i{wF1-!(k z`-`a-g7lhU9)mW9|81MIRw-f*z@rpc2!>=bva$6}O=Wvpj*pMOEx7kIxzNVPNB>*7 zddORr$75!Ovs!Ido+LaeDJku}eKoJ2-j4~pJVndC`d+$yRy+9y=s>6(g{q#Cv+4fe zzBY)>aizk%$z#D)G;*g7J*y6D5;B(qi%Zkirt6P`<`RTYG*I4)_jyJR>5vg*t>+}> zh?)u8FgKZwhBh=NN?H!jt+}>#A$C-pth6+qseA|R`Vn7MPgq2R>LS5Do6Vm_po$&Z zwe!Ht#3Ztimw}l%MK=3eX|hifi`juN%D}MRbHASkxgYJ*^xCdkQn#--cQ6dGBXtd!;Q{7Z#=SG+NOv)u}7RIr5kUmrY_hhczpV+bhX^-HpX+upA9rm9`+ z_gv)u?CrH@e)9SU?+lr^1P8NSV0x2V{SiC`-S!fa}4 zs!R>|7`fxmok4%BoTca*I5a$L@8KbsX`^2@AuKCKq4fB{H_$u1?(f( z{O8ZAMfyac<>uziWG;H)`;*M^DBAUp!^N9joe@QNNv^s#>Q8#`fgNli2%_RmL7|3Y~F($YK0$_>JM_V5$!&47OT*GWU= zkHrNBd_>x$f>{H74x%|&C_6j5F7J*$+Siw-yC|(lS~BjfSIvGeJD>x1*_P_fDkL~8 zPMnO##i3BB&F#&?h6X;7Ns|TnC1xLU^;oB5ydvTrI`}RL2}yQd9(}MUP>wXmrV@p6 zRIux}g;XvLUK(O>s2AyXZGansg8`WN)m7{M@VAW8(!Lk9@2ulw2Eo3>FT1T*P6pN! zOiUmHU*KgMk+j0NMDq!-Pr^tmZD$Axz8BgvN^2Mk&bBFIoQF$favsVGG9TG0$1l9Z z#l^+o!3+HHHR8(m8mL!lr8mtK5fO_fnG#`< zde!&H#d7Fl97^-AhZf4@cn=0!u zrM=~{%x9PM7wAUHcOVbEbTcco@grS@u#i?i@+r^bETDAvnxZw*xj+av*`(jOx(fY< zA|G9PMuVsJmSyYl5Be0i-4@%-i=7gK7mIIG&j2h%Et|xDl;CBK>o8Be-1_7|6hZ^QY0%k>W2r15CN)sSO|r zG<_?7tDQhyp}Y0UEYRwiP#cAKAeV+xwaJgw)vqJO2b{@0g# z$t%Qi-#lSPKl8alO$V|;w88(?i9>~lwH%(=b6%R*8zm#dj5bqcpjpdP)%FF8a@-l3 zO;)5f$e|-#&STis{l);TbgpI0LuqN|i}Nq{FBTYTntRh|qu*y3f6|S`&a6JoGbVf~ z+Zf9b#UZ!ftcsnf97yC9I_ulpz|^Vjnb&qJ-g(LdNss8{dH;vxc;5VcL#)mnp0ZK6?0&wiA0N{x)J z>Q3Y+Q!}odH?gd-6`;gzrznt(#X8vGCU#V6ZQ7-UE%~-0&*xDM;*{n^L(a#iw6E}? zuF~a@fdZYXAoHCemwPi`sMdZh(WFXx%t58}WYmYh^?9L{DP(ER4Pk{Gf=To#2fsR< zMI6wZAhfwdNTNX~iN)IGC{E#!+i0F^kWQuV(0;2tJJZnCPOgoatFCZoMq2w5nl!MM zf%FE;VtG}YUn4!Xl2?pgg8PV5%7TZfMqHbkFJwvKWT5uz_{)L_{z7+FE(;1FGg4Do zsoGaT=nqToKCMcLrax(>X%ny^19B`|0ih#J4rQk=_AF<;$V}u?Cyzs9t*%Xb&ymuj z*0A}(q5?E`-yKR$mEdQe>7Kqxb{_T|i${K8QYyY5U(WhD{9I-dJCl2*!q~qkQFjDlMf{X*VdnFNCEjvo>D@MGW zpG$Ss=W&5;|JQ5tBzxWa4lWxCeI%r;tzvoMz8cg<_UPu) z8}8TW!D5>$c{~_m|MK`)KcNEYj?(*zm0Vq}JKHYxOt%%!NR*8z zK*{-0H#Q`HiCRfT>Gkt-c1T5SMJ~MGyNPcE?vWRXheaTNbW{HZz5uhhrq@H*>1N&= zUBI&XsqIa*OP1V1A1~WMLjxxI4Yho75E)k=7KLL(0jUrWmdJbL>MNCHKF_&7QPeur zncB)0f|;vrSh=#Un7B9genJ}|ie+K#<31kDJ#z`b8*i#MTc0`1Bn_G06LwYn98V?l(+Liy>2KV`-1>fAPS}9mjZr$ahJ2L@%t8nnEBKM zm$iW)j^4ZaR_82~Jx91mbWeH|+?7_5#Q*`=w<-55-1cJWdHzzE)3J5o6N=ORC?+R8IT=}^ZmrK(L9fB^a4pJ zcQev*xVhg}(uN^UWVbn=un)5Mj%D6$De2v|L_qK^+m1RIo&l6w!0y@q#~wn&W>e90%rI>trS>cphP^w%6^PSM@YwmOM$xBW1rW#EqezS>??OK=FYUQ-aQ$p-?# z%qZrLEdd5RBD37ygtMD?tc2Ud#KO+B3F>!|T%xNReyz`gaAx)4f5q4?Te1VsWcP+b zP*58SKO)-rjp7x>_+ixg9y}apE)eJ2En(oX>a7i}Kd=NN_Ewj)L|!IR)3)s(os8!} z`z!DwkDkMBww~M&|Ct$mg32(qz~f8Ao7zpO$XGfD+_(E$2)mJ*ziaC%%k~r^5a5OS z=i26%iM!b${DBI-q$W@9ev6=PJdIgD$Z2c;hMkT)v_(?;d)+IH;tD2+R=izd&(K7- z5Q$?k)C9vM{06qg;E@jJ3ZUMTEnh-0 z7+|Zsh(hDF!s6B1*hzFkOBBUKMgN)?v&qAoD@ii!`#As2B>F{~lGmm_qaF|1 z{eZWQuZv#pkYnIQ+jI=AA%eI}4$kf{zaQAjF-7j`T*F3z35IVdxT=KI(5NOga;s<; z%DZ|KRDc4sz~_GqP|>%M(uE4N9QCaSvig-OR?=`Q}IiswKD>u5B~)1baHyi zj#ZbCkc(qNQvNH zx}BztxAiVn*Kf4ct3NB(mGmtLR#~wl4^|9*vhL-)^?sR9AC|j3V)pZ%gINcKK7L)X zQ5lgg`)`M?>ltrBhM@kns|554xhdy{y%3!#uq|HS;oTvg1`at)BwoUi2aropUIixq z@VWsxn@h0wpr(F6qHk&&P3$t+3&YmWKJN9GBQ?8s!NIMezj^#Lu=eE<=h&wrRIr<^ zCDN}Tw_K@!QrVUF0gz1{sBrIMr>_NpaP;zz6n&ZBCzcW{yId`>><2bYvQl`^sz@%g zssV3B)S>jkm6SW{S`G+KY1~qw)Ulgfc4m1h!8KS*f4)Dbf`9$4|NSgeqhvhiM8_RW z{-?Ezl9I%v0frrj4`sj?#hAqtFIt@vvMYupLsP>QdDYp8@4%vQzgo@WVM((=Dn|Y8 zPPiC;+X0{@_JqKZA~oh21-BMohLT(!MxdnBg|vc7dKiF0cFsiHM9M~~dkYVy<4N1F zfj5NzGPkP5gSxt7MZ&}1^kJ*ZV;h<5Qj@vtv*1)v?LyQ`#-ldulga}Hm=2t;nb_Dc z0^`o&oD^msi_gimG}F0gh+l?*v{3OfjbldstomY;9_H3&8D9?_F@3m$_192W9tTV~ zD?{uyjpI^^ME$FC_-2Q!H#m)rCs63AU=Dn|1oarOd!_DpkZz8JFlPP(W#g2hF#KQ1 zNliHdPdYCrAMSKV008Op>%FnrH5S-N&{Gr04^-f2KHWc)3Hy>)7r}w7*ZIren8;&MRmC4z?$sRwM5WBBG{JH-E?^~&h!!l@TEEhBB4M+Rh@7b=h`%(<&BQ|=Y0^C&70UFZF79n%8TxV7h54L7t zl(|;9k)5bxhW~NjuufZXv-z$5_t>qiEt9S{P*6aRZlk3(3jJ)|+@5aQCG+G4a@NV~ zfOEwqC1D|opu8%A9xk{@RS#ANh~T1#;1`XN9o@@}s-L>@gtW;I!H zYFRpej0cZ|!A#7;!IX~p24FGPH%D15R-7v1;s8LnJeAZ;kF~4r>)%^_`MiT!_^A)Y zT#}l6puEn_YR~#x&@mZLEAZ=Zg}tPGMIm~6ZH3OBmA$(^NuGe&W>{xAm7cVpUZ^8$ z^t{tuIO=TU;X&@}>zn4t_~9u&0OK%w^X|sS6DYqwE)wAQR>~Ck4VpKc;Q!!aYKj45 z>^F!qB&q_FJ@f*AeGaDnEXm{IWX+YzT7>uN;J+j(8uB-GunubX2q5f5rUvJG75 z4yLsCW@;3_1TzVM>X1O+llKZ2-RgX*dUn=SvdNKn;yj&fk9}}~ZK3QkFa~^apppmo zAwgGuFP(W90R60U@6*sxvIGTKPde8>*e2}F&?WgP)65Wu7C!V&&tp@nso+@w$?!8Q za=EKq8^}PU@)XQFXLyxH!49MZ%F;PA^u~zh{X7VUCnVb|G?eA}^XFYCZS?k1XUscX z=zB()zwJSlO}~-V_PutpRnP$-?VVC5O{8Dz#daB0ol7IYqMmAN_s|7`hDZ6vmt{n5 zd7@j}uF=)FZYX7HtLLNKD}UOOVQA-TlOY0g&kPBbDP`49tT^gv3e;a06u)hp5f?!tnqxQo$>h$Y_Yd!!E>Ra;(`;-kG z5N$sx5?bCW$+HUJpY7%TtOFnq;?hz(fB%Q+K|$WP%mvF`X|&aII04IN%c;wA=lo|K z^71Kp?XcL43%{Ho?Ox3lpozlRMpKISOPA=@4`(rza&m-ohqg9auuBxTD$upnRW49e zt#527e^{;H)%-?17Xu(FN&g)N1H;`5O{JDsQe>)n`MjFQPQK*xXWtEod+ezGr`?7l zrWi4`dVD^KM*xcRg7TEi;f-p~#dZW{m#b#RhQULu{N<9xjVEHQ<1LTP?v1ZPuyx(O z)8~)5xu?4sQa-DR8$uYD`k7`I-4h(u%rzs5VfJf7k2YfcuW+4zK>eSOeb9lLX9~eP z)x9#-n9Gj|1y-o`48Sv({Ko|tb*^i7`dDlm6f`a3z?U4|uTjn6`A%cD2HHFJkg?ytyVyh~-vOPTAG9!ry7V7c+>fCh(1f+Hk)@?!8Z?fwQQ;3^ zo&hN4Qx$~yYyFWsTC%Rho%uJq%fHohvE|wmp-CfAIj5~+Yu~4tE}1~gEOcU}o9|DB zMzWJFQjUTkp!snmI zUqF*PkB(|KP&+-Gg}k8mqX|Z^wnwx6bAHS6y@pG7PpMT0_wR%+eEpDEh)B!ey4$8$ zc1yxMV4b{Qh ziUs%fkiuHl<1=%$o`pwx^xNoWPhBIytd#E(Ds>A$GW|aVhSa_Q2z=pv#>~p%UVxkH zsq^D{)Wdja^6w;S9 zHXR-wGMFi=plFqi#@uE*F1WttLWhG%Z!dliUBH93wtfc>t&FqL1&*uy{k2x(g|HK% zqf$-kj?<`Jp@`HPF846$O14)JP4Q!fEm6W=Pu?fRX$&V1+~P-ETfg$ypHN+zxZRwq zljwvEZ7PmQVrm^q1tx02N zwy`Fxkl~of5w)I6d0`Z7Htr2wdkR842O(8*TNeAp?ufnN(~}~0?2pSX9v;5~I`!T| z2P8w!yna-0a@zH27N5vtSKg2Y9v3ZO>l;G`tcJHw*T#o#=eDcm7v%e`pxYmdUw$*- z`({+u4j-&ecVqzy=fT^%y6uhcHErX`vt*Qu$snB>HZXbx9~xx)m+E*swOy99z)+P>B% zknhO+hM>B~ouAu`x!7M{5`t6(EKaEa!OfKTtqL$&dZZh$FszWhtCbfcW;gqYS^iOs zL(_!sWej4oq>(%`MyUWxBR$mko^>kdet`_gEWT4G(~*W44}fl`7F44OfXDI>J~u&X z7H}itbxz~8MKE-mgH2EAfoQ%4k1OF*XRXRflYBI@oe)=LW5b?1x5 znf6DO0iyqTW$B40Xo>V*?W1xKMfu*C)tmJU4nCc22j21BJ1Wqv1fLF~jD0n8C3JVF zl5G3TsutRb#ea*_leM+~A=TwCF>f01dBBdHrGaX2DjszP z9HjbqasZqp|1VBetD^Y-!J;Z7LB>o_EjRYYJ;Fl7lZoVI0ADB!?D?~8Vq8gf%p`uN zuSg=`1i*kM5?2XxQ`Y@$Z|_97L>|E)k=3>_j^7DqQKZ&@>taJ{it|f?rU{o1eq^=G zH@LJ$llW1~g|V`#WpY9$|4Y~o`X6g(`3`wZ*8>?Me7+LTf=#)qYtGiL_v5aoVRvVT zKDS?9{RWNlKc1wf*IW$ctx6fzOD?mgj6S$OHr&q9d_i#!7V{kiP+qLPVWbwIqi(}c zbCJKIwMSj9qHKg}YLlC7FrJ^^lv2P8TbNsJ-fc)tO#H^iP-C~sT#4J2=J~BgCP3m+Qyf&L&|26GI@^z zBpadt^~i(NRPT7J!e=|EMvk7y!o%^pVYgS(y@hHws%GDJwgLYuyW-%Dg~KN>7F2Om z)SnYsxt>mAlH$wQ3kGkRXEUNbAnM{4)9`E7>r=?iD3D1f!mp-qRhXd6ef*FTgRb6j zyu!{yg>&&cQt=nfBi=dJ7NjPCjR#yH?=;Rk_t^)p9^(my1eN4Fu)2DfhkVY7 z-z(n`n{N$z9=Ml28w!OZV>c{_2K#-acx0lPR%ofH&^SEpYD3`?9MRh0aJl_#m`9_~ zJiCvCE!>zPS+Ig~)kXPT5liE196*-?wfxho=-18&pScanKhd$@t-^7yGW8E;dG1tN z3`jhhJiT!-QN{-Ep}gP-jQdxWbq19ylM6a+l{a=eUej z`3{TZ`gC4IM%eEE_=(KQlUUwYMuf3~aQhhia&~#*aM>sA#yHJ8zR>Z|&Hy%GM4*wx zWT3PB%T<0(Ieif>S)R17_^UCFtLorKfdZhQ-T6lhA~eaZ!h1k{rz5v`lHVih*G3>v z5yjp#2~w5r_d!LrvD$|ux_8$YzXD5D2k4$H%r@qrbbWq-SvWCY5l|-pP)&ayO?{nYtKJPj2y@Vu5-S{Jb>W5EBoYbo#EBCnf*=rg@;-N* zh}Y|-PoF+r&hz};MI;uBk;jf5tKvBBjv77}1t$^=2IUtoUNnoM_@xq|vR#NqqhxPy zuh!@D(GG_rR$E&e2Y^Tv2WRk3}ua8~1azzgS=g*%ns;#Y! z&75<3tkda?YBU;Qx;+pG$h*6{4XIR0=5#tEcDp?}eN7ZaT)uo+?{>SD6h%p!Hf@?P z7!3Sedoz0Ewbx!-_R1@-*h8TZ#q&Jz^2;x;?C9ty27s=vF8%4#r^{yc@9gX}wzs!i z0f1#$82}`cNiv;If9{o+UV3T8n{U2Z)z#HyeD1mDoC5;`N&x8V>r=h(!V7gnLqjTx zqNG<}eYN_`nKLE;xOwxY>ctmdbbS2r$9lb9&)v9jL-+ji&##WfVq{((xq9`g@tJ3y zxmj0N7Y6{Erp0sT&RMo^-yX=+e`R z05mo>h7KJ%)Ntj>75&DI8z*ymWG-wrTf*n_DZ}A#)}@#W#l^+k^#6*AibNm~pgE2s zdV6~fD_5?JPG769ua89{5rxa;QqP@3tJMkygCRZB(~BZfR8+(R!0_-evospBK@d}R z#N%<2Wm$6OnpY?kB1KW+~fcWN>g$Ns^>kQ&W=wfOI-t5VbQH417gJMSNgj zKsDXwcDort5M&Jv4UxsnmDM1Z%cV^u5@a|WraC)2P1V)aQJSVjr_&h?g+hw1t}Z<= zr8r|_W6J4qMMXs%MNy*1<(s6h(ME9;MZ4WtU{ZOk4GpQbZ{I$)Zr!>?Da@Tf#u&L9vE-$d4S`6K z>b~XGjD_R!iD*(DiY6&aMo0!V!yB~>Z#AgXTh=U#P&s06u_zdkes75W@$rkcUiY{b zk_1Q+fX{wE|1ScP1dBnH`sNm=?~w=VCkjQVAWKUWfro#3s@@ZbF{^DR#FHru_(I4= zWV(IV`j!44d~3__@67@QI#>;ig=o)2oLOVH;O7Tg@b00nqgb!XIqp{E5>rNAOG_5a#2E#;LK+D zBJ-aAqy<7MZz0H=$jM8CMwSx_qWLqSQPKj$47r34%Q1e!lM@*jLR^d}~X?NOMC; z#H1LfAo0XJvJwFBK;pI%;eRn>EI{jQ#mR3!pWP6S;wS3lktoNIxQpTe{o>5 ztAhJ?3p~HIK@x~*56sRlGo?9?>`xVO`L6!FW2D5ON@csY|7-UQSES#uBJx`cP{7v- zOj+Iqjl7AJWg$Wkkajc_M-lm*Ip4YR22)jdBFzou!A$LDy)v!B5S=ScE(ylwyel(# z6Zwk68go_|=T%Ecf{>;HyTDM@MN#xXGn?A79>7CUW`$)UsAeAHbx!g|gil<2!w}n8L<$f4BQ1BCCM)=>U%$P?c z2~gG~3qj<||AeDx8({uZfGF%nx-cFg^L!~omjhLYrv9frKWR^x?)41r%-5p1_Wj8hqj5DX7v%Q&oi$3W=|mXq$>YlW0gf%G5^c7y?;5J8ux zz*MCmH|Ji0OTbWpa+TAT)eK~F$&?9~5SgVC7$ zpa1ro{UiRM;^(K%SuF-#dhd?)0i$jzKeheOMbn3$^y#WfOl&Y3&scHE=#g*z@D0bN zwYAZ|-~X^X9B0X+@3xl$!1rJM&#HmZapu{*+r96czF>LnKR&SNwMzc&Uv-v>qKE;{ zIDPy~cge5Me_R9r|9#<_vDBpF{_gdYmG*KA`|X`y@gF?ewn7jD;?~d@bKuB3)jJ<> z246b3cR(hS;n|&pufbV=l7E%j!AM-b-nk;Cph)TdRm)=~Mm^_pduJ6gT!UVQQBdOg)%F+w+*h}}kEBS+7YNZ0 zuB{2@vNLC}U%x$~TxBndnT6cdxF2addnb_$s~rB zOHHd+gpd4Nd*zn<>cSc&BkDCuE|ubB4qGYfu$8iUt!iE(bl6H+|728AWi_U2mzA=$ z%Su^VPD$$=Rnf~XwNz;}0V^6vl1+ci6yoD=7- z_301)<(?q`C^H*4vq76)ZgE_1@axcrdEi z+p@tw_kN|tdhRd2v1#=2AHMF`d~a=Vcx;mX-Vb){@kMM6wO~|A>q+qEXU!zQ9mmWb_Ef^MstD3i^q~okm8F+-)R) dKcc`unPw&?X(v;o(_+&^9j6^TNhdm!bdrCJ#A%1=M8%mVjhQAz zMQEpt+<>-36!8aE0rdjN9X}3^-*9-^_j&i-^S$4D+}_^O=9$^qTlQ{upAX;Xd7t;) zcNgF*^J)>{zJPyy_?nKa@;vNnUA^s~_5TJA0Z~L&xO|H&Sze&HK_ZKhESJb)Aixa& zGjIryQnK1a7F_FG_s1lpKult)+wW$^EHE=Zhd{ExsHqc1%^h^x1yng%4um2EBSHa1qm6y*u5HSp4E@R? zNXh&r{T!^Jnufl-45q6FURfc56d@xRRLWbTQj+vJur$Afsw7l~YU;jNOV%Acqa7#soyp&%Tpl&n=hR$X;#Z2FukiCHB9 zv?b(lcrd7zujM?g+VX#(;|r>yxdMs`5X@dyzo;IJP*-KZg2WWk^k2cvysaCbjyQDU zoI!Tw@Hu8afbpRq$vG%eA{gugy^o)lk``$`fkDIT#p`DFKJ!N&IAU{+Rc=Op$-EqX z!0X2AXGQ7}S>UidlZVapBP5*Wvpw>l+{j~o%{}L^>i0H&&fW03qE&AwtLe6y55&!E zUsZIZgm9vAoG)M|to>lTcpBR3kAc;6+igGk!NLptg;#{1fvI}@T(EhsDymIlTStou4{wTZTp;HX+fYdeA>C5ShQb zq=r?gkcb~wB^Xo+D#jIzTj#UO3>ja~hvdbchhsVaN+pFAs~)qEESJ?)lV8q#evsxE zwDrH-58r)hkK6a`_InhJR;P!T-#F;?c-+u!mZz%OQ&<&=V>D{ZFK0;@@y&BxQ%P+3YcCXn2c1;4j@az4MyK1eZ zW22u84*Gq-ADlcjDF^clhEZ(z| z!1UBKOc59p6BBfea2y@Pm~>ch?-oQy-gip22xW2hO4w5N9xQwODOs)s{bd94{*|MN zi46%1gfLDPTw?Vsb((CIG)*Y-#<0~2gBFWlbx2CgsViXbyYB^@XjKIV z!=|slpDIPRX1u)}^1i(~EMTMzc1eQLedi8bztJu$7_{bd&G7xg?}e?L2ue8^KD{Qh zQ!68vUAcS(($mu*Gcz-6aGaD=V;L0sG0qc&yxPdK z(~OO-#feA$Y-xHrL`6l(ez9j7&p=Mja`?uI6^g(S{T2Q92=%lKkLa)Hzqf>JG}&eX3azB1M7qdf*md~qub;v0-Mb(zE>6*qVtyNa z!nrs$CeSo@NIrpF;dQF|6l~h~tYYEB`^o(F4W~tfW9pQA0y)B~y1E+lOO`-M$uo+! zNTQ!De&JL!85^;ECWbRMr6%TJ!>hTu8B%n~P*(PHMZutd=qGI*(Qo3v&mwTlez zIxN&N?1!NYFRW)~Nh_f=7~Vhh7yXt}As=rL#KTV;3rTn(m@!f-g#!%lAM^LP6gY;O z!9f@&#T{nyKyptY;Z^s~I!H;@g$)e)h5n(R=r0J$$O5ulsdO92a9PkCWh8I@8moHs{T|0M#wKK{4gZ_;ReZxxWaD*nU3aMh{ zh=b2PL4?<}wl>g4N5i(QuP9dedNcF`{XxH&YN}kDa4F+C_k;e>vmpygg)7~#o`B9qOQl@O(Yl?e&7E_i6g8ihil%*%#&bLvo$+1X-31+!lk^1FYCZ_4a#n-! z*NuLlKfHf|D@Ra|m8b`&#ua@*inSjuS_Bc(<1jta4~v!+%o{iz^+#z)VHo|Y{e~D$ z`-p0)<_<>yWFUY_$pM!%jgG*&lvJ>f4$!2A_n89;v2ZBn*4a`dG*G1_)@q< z)Jnj-=308fN76Vx1HVkqHtieT{ifjx&MsX>u+rt>{!q8==IN{41= zK~xZi82(pKLE5+^)qufn0fLz#;qL+lZ97rT-{pfN0BR+GM#!gUL_jO!6JX)}yA)8Q zCK?hKXN4UMb|!DseF%1zeD~b!&AAT4%q%LxN z1tH;NXekfZ&qXO1LLwA2yzg|Nt4nOEA2cQvO_g&wM{ti$m#uO<9W4Gy9}@#x)6$$E zi8E6XRxqFt2*SXjP5xC52<1?3FPLU#_^(?T80DW1<^W{@_gpy->G{PE(ZVr-8{5HY zW^t(*u#mp8A%(uS0WFR|!+qEUPqN!3_?iZUFhUBrNlTAY*?{UeH8=>L_V>9$5eKfe zcAKr#BMthB>J)SqJ%~pQYEv%)KQdJ!!m!(lBgw%_+QEu<}*M8wp$hd3?c# ztD^K;EdiB874@i(A&hp?1fxPOj*Sv9HctxSq0o0(5HRX{mqnnEWU`Pz2`wD=r0we8}@g0u@Ln5&aO~|%2y_n{a1hi00-6?7X^q( Q0RR9107*qoM6N<$g8M56QUCw| literal 0 HcmV?d00001 From 708f1b009b9b23971d73bfc7bc09163969ab6e00 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 10 Mar 2017 21:36:33 +0100 Subject: [PATCH 103/109] Add integration tests for MEDIA_ALLOW_REDIRECTS --- tests/test_pipeline_crawl.py | 163 +++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 tests/test_pipeline_crawl.py diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py new file mode 100644 index 000000000..1f5b80954 --- /dev/null +++ b/tests/test_pipeline_crawl.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +import os +import shutil + +from testfixtures import LogCapture +from twisted.internet import defer +from twisted.trial.unittest import TestCase +from w3lib.url import add_or_replace_parameter + +from scrapy.crawler import CrawlerRunner +from scrapy import signals +from tests.mockserver import MockServer +from tests.spiders import SimpleSpider + + +class MediaDownloadSpider(SimpleSpider): + name = 'mediadownload' + + def _process_url(self, url): + return url + + def parse(self, response): + self.logger.info(response.headers) + self.logger.info(response.text) + item = { + 'images': [], + 'image_urls': [ + self._process_url(response.urljoin(href)) + for href in response.xpath(''' + //table[thead/tr/th="Filename"] + /tbody//a/@href + ''').extract()], + } + yield item + + +class BrokenLinksMediaDownloadSpider(MediaDownloadSpider): + name = 'brokenmedia' + + def _process_url(self, url): + return url + '.foo' + + +class RedirectedMediaDownloadSpider(MediaDownloadSpider): + name = 'redirectedmedia' + + def _process_url(self, url): + return add_or_replace_parameter( + 'http://localhost:8998/redirect-to', + 'goto', url) + + +class MediaDownloadCrawlTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + # prepare a directory for storing files + self.tmpmediastore = self.mktemp() + os.mkdir(self.tmpmediastore) + self.settings = { + 'ITEM_PIPELINES': {'scrapy.pipelines.images.ImagesPipeline': 1}, + 'IMAGES_STORE': self.tmpmediastore, + } + self.runner = CrawlerRunner(self.settings) + self.items = [] + # these are the checksums for images in test_site/files/images + # - scrapy.png + # - python-powered-h-50x65.png + # - python-logo-master-v3-TM-flattened.png + self.expected_checksums = set([ + 'a7020c30837f971084834e603625af58', + 'acac52d42b63cf2c3b05832641f3a53c', + '195672ac5888feb400fbf7b352553afe']) + + def tearDown(self): + shutil.rmtree(self.tmpmediastore) + self.items = [] + self.mockserver.__exit__(None, None, None) + + def _on_item_scraped(self, item): + self.items.append(item) + + def _create_crawler(self, spider_class): + crawler = self.runner.create_crawler(spider_class) + crawler.signals.connect(self._on_item_scraped, signals.item_scraped) + return crawler + + def _assert_files_downloaded(self, items, logs): + self.assertEqual(len(items), 1) + self.assertIn('images', items[0]) + + # check that logs show the expected number of successful file downloads + file_dl_success = 'File (downloaded): Downloaded file from' + self.assertEqual(logs.count(file_dl_success), 3) + + # check that the images checksums are what we know they should be + checksums = set( + i['checksum'] + for item in items + for i in item['images']) + self.assertEqual(checksums, self.expected_checksums) + + # check that the image files where actually written to the media store + for item in items: + for i in item['images']: + self.assertTrue( + os.path.exists( + os.path.join(self.tmpmediastore, i['path']))) + + def _assert_files_download_failure(self, crawler, items, code, logs): + + # check that the item does NOT have the "images" field populated + self.assertEqual(len(items), 1) + self.assertIn('images', items[0]) + self.assertFalse(items[0]['images']) + + # check that there was 1 successful fetch and 3 other responses with non-200 code + self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4) + self.assertEqual(crawler.stats.get_value('downloader/response_count'), 4) + self.assertEqual(crawler.stats.get_value('downloader/response_status_count/200'), 1) + self.assertEqual(crawler.stats.get_value('downloader/response_status_count/%d' % code), 3) + + # check that logs do show the failure on the file downloads + file_dl_failure = 'File (code: %d): Error downloading file from' % code + self.assertEqual(logs.count(file_dl_failure), 3) + + # check that no files were written to the media store + self.assertEqual(os.listdir(self.tmpmediastore), []) + + @defer.inlineCallbacks + def test_download_media(self): + crawler = self._create_crawler(MediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl("http://localhost:8998/files/images/") + self._assert_files_downloaded(self.items, str(log)) + + @defer.inlineCallbacks + def test_download_media_wrong_urls(self): + crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl("http://localhost:8998/files/images/") + self._assert_files_download_failure(crawler, self.items, 404, str(log)) + + @defer.inlineCallbacks + def test_download_media_redirected_default_failure(self): + crawler = self._create_crawler(RedirectedMediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl("http://localhost:8998/files/images/") + self._assert_files_download_failure(crawler, self.items, 302, str(log)) + + @defer.inlineCallbacks + def test_download_media_redirected_allowed(self): + settings = dict(self.settings) + settings.update({'MEDIA_ALLOW_REDIRECTS': True}) + self.runner = CrawlerRunner(settings) + + crawler = self._create_crawler(RedirectedMediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl("http://localhost:8998/files/images/") + self._assert_files_downloaded(self.items, str(log)) + self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3) From 871134ee22653f7f074fbfb8f3c393ba3555a6d4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Sun, 12 Mar 2017 17:30:24 +0100 Subject: [PATCH 104/109] Refactor to also test FilesPipeline --- tests/test_pipeline_crawl.py | 79 ++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 1f5b80954..9b81f827d 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -23,8 +23,8 @@ class MediaDownloadSpider(SimpleSpider): self.logger.info(response.headers) self.logger.info(response.text) item = { - 'images': [], - 'image_urls': [ + self.media_key: [], + self.media_urls_key: [ self._process_url(response.urljoin(href)) for href in response.xpath(''' //table[thead/tr/th="Filename"] @@ -50,7 +50,15 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): 'goto', url) -class MediaDownloadCrawlTestCase(TestCase): +class FileDownloadCrawlTestCase(TestCase): + pipeline_class = 'scrapy.pipelines.files.FilesPipeline' + store_setting_key = 'FILES_STORE' + media_key = 'files' + media_urls_key = 'file_urls' + expected_checksums = set([ + '5547178b89448faf0015a13f904c936e', + 'c2281c83670e31d8aaab7cb642b824db', + 'ed3f6538dc15d4d9179dae57319edc5f']) def setUp(self): self.mockserver = MockServer() @@ -60,19 +68,11 @@ class MediaDownloadCrawlTestCase(TestCase): self.tmpmediastore = self.mktemp() os.mkdir(self.tmpmediastore) self.settings = { - 'ITEM_PIPELINES': {'scrapy.pipelines.images.ImagesPipeline': 1}, - 'IMAGES_STORE': self.tmpmediastore, + 'ITEM_PIPELINES': {self.pipeline_class: 1}, + self.store_setting_key: self.tmpmediastore, } self.runner = CrawlerRunner(self.settings) self.items = [] - # these are the checksums for images in test_site/files/images - # - scrapy.png - # - python-powered-h-50x65.png - # - python-logo-master-v3-TM-flattened.png - self.expected_checksums = set([ - 'a7020c30837f971084834e603625af58', - 'acac52d42b63cf2c3b05832641f3a53c', - '195672ac5888feb400fbf7b352553afe']) def tearDown(self): shutil.rmtree(self.tmpmediastore) @@ -82,39 +82,40 @@ class MediaDownloadCrawlTestCase(TestCase): def _on_item_scraped(self, item): self.items.append(item) - def _create_crawler(self, spider_class): - crawler = self.runner.create_crawler(spider_class) + def _create_crawler(self, spider_class, **kwargs): + crawler = self.runner.create_crawler(spider_class, **kwargs) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) return crawler def _assert_files_downloaded(self, items, logs): self.assertEqual(len(items), 1) - self.assertIn('images', items[0]) + self.assertIn(self.media_key, items[0]) # check that logs show the expected number of successful file downloads file_dl_success = 'File (downloaded): Downloaded file from' self.assertEqual(logs.count(file_dl_success), 3) - # check that the images checksums are what we know they should be - checksums = set( - i['checksum'] - for item in items - for i in item['images']) - self.assertEqual(checksums, self.expected_checksums) + # check that the images/files checksums are what we know they should be + if self.expected_checksums is not None: + checksums = set( + i['checksum'] + for item in items + for i in item[self.media_key]) + self.assertEqual(checksums, self.expected_checksums) # check that the image files where actually written to the media store for item in items: - for i in item['images']: + for i in item[self.media_key]: self.assertTrue( os.path.exists( os.path.join(self.tmpmediastore, i['path']))) def _assert_files_download_failure(self, crawler, items, code, logs): - # check that the item does NOT have the "images" field populated + # check that the item does NOT have the "images/files" field populated self.assertEqual(len(items), 1) - self.assertIn('images', items[0]) - self.assertFalse(items[0]['images']) + self.assertIn(self.media_key, items[0]) + self.assertFalse(items[0][self.media_key]) # check that there was 1 successful fetch and 3 other responses with non-200 code self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4) @@ -133,21 +134,27 @@ class MediaDownloadCrawlTestCase(TestCase): def test_download_media(self): crawler = self._create_crawler(MediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/") + yield crawler.crawl("http://localhost:8998/files/images/", + media_key=self.media_key, + media_urls_key=self.media_urls_key) self._assert_files_downloaded(self.items, str(log)) @defer.inlineCallbacks def test_download_media_wrong_urls(self): crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/") + yield crawler.crawl("http://localhost:8998/files/images/", + media_key=self.media_key, + media_urls_key=self.media_urls_key) self._assert_files_download_failure(crawler, self.items, 404, str(log)) @defer.inlineCallbacks def test_download_media_redirected_default_failure(self): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/") + yield crawler.crawl("http://localhost:8998/files/images/", + media_key=self.media_key, + media_urls_key=self.media_urls_key) self._assert_files_download_failure(crawler, self.items, 302, str(log)) @defer.inlineCallbacks @@ -158,6 +165,18 @@ class MediaDownloadCrawlTestCase(TestCase): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/") + yield crawler.crawl("http://localhost:8998/files/images/", + media_key=self.media_key, + media_urls_key=self.media_urls_key) self._assert_files_downloaded(self.items, str(log)) self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3) + + +class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase): + pipeline_class = 'scrapy.pipelines.images.ImagesPipeline' + store_setting_key = 'IMAGES_STORE' + media_key = 'images' + media_urls_key = 'image_urls' + + # somehow checksums for images are different for Python 3.3 + expected_checksums = None From 3cd9185aa12430708b17eb8c02a6bdc3709ed5f9 Mon Sep 17 00:00:00 2001 From: jorenham Date: Wed, 8 Mar 2017 20:44:39 +0100 Subject: [PATCH 105/109] Fixed the FIXME; more specific exception catching --- scrapy/pipelines/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 843b4d3ec..bdc0f24e5 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -58,7 +58,7 @@ class FSFilesStore(object): absolute_path = self._get_filesystem_path(path) try: last_modified = os.path.getmtime(absolute_path) - except: # FIXME: catching everything! + except os.error: return {} with open(absolute_path, 'rb') as f: From fbb411a805724fec50b786f369be79dc221c798e Mon Sep 17 00:00:00 2001 From: woxcab Date: Mon, 13 Mar 2017 14:16:39 +0300 Subject: [PATCH 106/109] Allowed passing objects of Mapping class or its subclass to the CaselessDict initializer --- scrapy/utils/datatypes.py | 4 ++-- tests/test_utils_datatypes.py | 43 +++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index e516185bd..eb373c501 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -8,7 +8,7 @@ This module must not depend on any module outside the Standard Library. import copy import six import warnings -from collections import OrderedDict +from collections import OrderedDict, Mapping from scrapy.exceptions import ScrapyDeprecationWarning @@ -224,7 +224,7 @@ class CaselessDict(dict): return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) def update(self, seq): - seq = seq.items() if isinstance(seq, dict) else seq + seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super(CaselessDict, self).update(iseq) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 80f797227..3a4137942 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,5 +1,6 @@ import copy import unittest +from collections import Mapping, MutableMapping from scrapy.utils.datatypes import CaselessDict, SequenceExclude @@ -18,6 +19,48 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) + class MyMapping(Mapping): + def __init__(self, **kwargs): + self._d = kwargs + + def __getitem__(self, key): + return self._d[key] + + def __iter__(self): + return iter(self._d) + + def __len__(self): + return len(self._d) + + seq = MyMapping(red=1, black=3) + d = CaselessDict(seq) + self.assertEqual(d['red'], 1) + self.assertEqual(d['black'], 3) + + class MyMutableMapping(MutableMapping): + def __init__(self, **kwargs): + self._d = kwargs + + def __getitem__(self, key): + return self._d[key] + + def __setitem__(self, key, value): + self._d[key] = value + + def __delitem__(self, key): + del self._d[key] + + def __iter__(self): + return iter(self._d) + + def __len__(self): + return len(self._d) + + seq = MyMutableMapping(red=1, black=3) + d = CaselessDict(seq) + self.assertEqual(d['red'], 1) + self.assertEqual(d['black'], 3) + def test_caseless(self): d = CaselessDict() d['key_Lower'] = 1 From 0f2a5cdb8edb3d4d6ce542f13b5d7e5858905bae Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Mon, 13 Mar 2017 15:13:24 +0100 Subject: [PATCH 107/109] [logformatter] 'flags' format spec backward compatibility pass 'flags' kwarg to logger so that it is compatible with old format of CRAWLEDMSG. --- scrapy/logformatter.py | 1 + tests/test_logformatter.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index e7bf7942e..2a89c00c5 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -43,6 +43,7 @@ class LogFormatter(object): 'request_flags' : request_flags, 'referer': referer_str(request), 'response_flags': response_flags, + 'flags': response_flags } } diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 11fe7b653..52646ec1b 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -64,5 +64,32 @@ class LoggingContribTest(unittest.TestCase): assert all(isinstance(x, six.text_type) for x in lines) self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3']) + +class LogFormatterSubclass(LogFormatter): + # Formatter with format spec that is same as in Scrapy before 1.3 version. + def crawled(self, request, response, spider): + kwargs = super(LogFormatterSubclass, self).crawled( + request, response, spider) + CRAWLEDMSG = ( + u"Crawled (%(status)s) %(request)s (referer: " + u"%(referer)s)%(flags)s" + ) + return { + 'level': kwargs['level'], + 'msg': CRAWLEDMSG, + 'args': kwargs['args'] + } + + +class LogformatterSubclassTest(LoggingContribTest): + # Test if old crawledmsg format string still works fine + def setUp(self): + self.formatter = LogFormatterSubclass() + self.spider = Spider('default') + + def test_flags_in_request(self): + pass + + if __name__ == "__main__": unittest.main() From a84652e775fda1135fe959465b27cf6ed2c25e1d Mon Sep 17 00:00:00 2001 From: woxcab Date: Wed, 15 Mar 2017 12:39:48 +0300 Subject: [PATCH 108/109] Init tests are split by initializer' input --- tests/test_utils_datatypes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 3a4137942..49323f0ff 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -8,17 +8,19 @@ __doctests__ = ['scrapy.utils.datatypes'] class CaselessDictTest(unittest.TestCase): - def test_init(self): + def test_init_dict(self): seq = {'red': 1, 'black': 3} d = CaselessDict(seq) self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) + def test_init_pair_sequence(self): seq = (('red', 1), ('black', 3)) d = CaselessDict(seq) self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) + def test_init_mapping(self): class MyMapping(Mapping): def __init__(self, **kwargs): self._d = kwargs @@ -37,6 +39,7 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) + def test_init_mutable_mapping(self): class MyMutableMapping(MutableMapping): def __init__(self, **kwargs): self._d = kwargs From 4345eaf1b640bfecb2340a0e27649b1ab1079da6 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Fri, 17 Mar 2017 08:11:20 +0100 Subject: [PATCH 109/109] [logformatter] backward compat comments --- scrapy/logformatter.py | 1 + tests/test_logformatter.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 2a89c00c5..075a6d862 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -43,6 +43,7 @@ class LogFormatter(object): 'request_flags' : request_flags, 'referer': referer_str(request), 'response_flags': response_flags, + # backward compatibility with Scrapy logformatter below 1.4 version 'flags': response_flags } } diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 52646ec1b..94e6c9fde 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -66,7 +66,6 @@ class LoggingContribTest(unittest.TestCase): class LogFormatterSubclass(LogFormatter): - # Formatter with format spec that is same as in Scrapy before 1.3 version. def crawled(self, request, response, spider): kwargs = super(LogFormatterSubclass, self).crawled( request, response, spider) @@ -82,7 +81,6 @@ class LogFormatterSubclass(LogFormatter): class LogformatterSubclassTest(LoggingContribTest): - # Test if old crawledmsg format string still works fine def setUp(self): self.formatter = LogFormatterSubclass() self.spider = Spider('default')