From 2240f00a136e264cc1856860fddba85ec97432c1 Mon Sep 17 00:00:00 2001 From: nyov Date: Tue, 1 Mar 2016 07:12:19 +0000 Subject: [PATCH 001/362] Remove dependency on os.environ from default settings Avoid loading settings from environment in scrapy core. Instead it's better to populate them from the starting shell or an embedding script. --- docs/topics/commands.rst | 6 +++--- docs/topics/settings.rst | 8 ++++---- scrapy/cmdline.py | 8 +++++++- scrapy/commands/edit.py | 4 +++- scrapy/settings/default_settings.py | 11 +++-------- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 6636c30cb..935e3281e 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -291,12 +291,12 @@ edit * Syntax: ``scrapy edit `` * Requires project: *yes* -Edit the given spider using the editor defined in the :setting:`EDITOR` -setting. +Edit the given spider using the editor defined in the ``EDITOR`` environment +variable or (if unset) the :setting:`EDITOR` setting. This command is provided only as a convenience shortcut for the most common case, the developer is of course free to choose any tool or IDE to write and -debug his spiders. +debug spiders. Usage example:: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0515a9e0d..1bb7a9d2b 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -635,11 +635,11 @@ Setting :setting:`DUPEFILTER_DEBUG` to ``True`` will make it log all duplicate r EDITOR ------ -Default: `depends on the environment` +Default: ``vi`` (on Unix systems) or the IDLE editor (on Windows) -The editor to use for editing spiders with the :command:`edit` command. It -defaults to the ``EDITOR`` environment variable, if set. Otherwise, it defaults -to ``vi`` (on Unix systems) or the IDLE editor (on Windows). +The editor to use for editing spiders with the :command:`edit` command. +Additionally, if the ``EDITOR`` environment variable is set, the :command:`edit` +command will prefer it over the default setting. .. setting:: EXTENSIONS diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index cb7bbd64d..dca931e99 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,5 +1,5 @@ from __future__ import print_function -import sys +import sys, os import optparse import cProfile import inspect @@ -106,6 +106,12 @@ def execute(argv=None, settings=None): if settings is None: settings = get_project_settings() + # set EDITOR from environment if available + try: + editor = os.environ['EDITOR'] + except KeyError: pass + else: + settings['EDITOR'] = editor check_deprecated_settings(settings) # --- backwards compatibility for scrapy.conf.settings singleton --- diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index 2df6a730c..a7f8983b4 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -3,6 +3,7 @@ import sys, os from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError + class Command(ScrapyCommand): requires_project = True @@ -15,7 +16,8 @@ class Command(ScrapyCommand): return "Edit spider" def long_desc(self): - return "Edit a spider using the editor defined in EDITOR setting" + return ("Edit a spider using the editor defined in the EDITOR environment" + " variable or else the EDITOR setting") def _err(self, msg): sys.stderr.write(msg + os.linesep) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 24714a7a8..f687ef6b1 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -13,7 +13,6 @@ Scrapy developers, if you add a setting here remember to: """ -import os import sys from importlib import import_module from os.path import join, abspath, dirname @@ -111,13 +110,9 @@ DOWNLOADER_STATS = True DUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter' -try: - EDITOR = os.environ['EDITOR'] -except KeyError: - if sys.platform == 'win32': - EDITOR = '%s -m idlelib.idle' - else: - EDITOR = 'vi' +EDITOR = 'vi' +if sys.platform == 'win32': + EDITOR = '%s -m idlelib.idle' EXTENSIONS = {} From b6ab1ae9c3ffa7aeee0e7ee77e2dd55d7b663a30 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Tue, 3 Jan 2017 15:14:59 -0200 Subject: [PATCH 002/362] docs: installation instructions, mention conda in the beginning (closes #2475) --- docs/intro/install.rst | 74 +++++++++++------------------------------- 1 file changed, 19 insertions(+), 55 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 767749ec5..86387ef5e 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,14 +7,25 @@ Installation guide Installing Scrapy ================= -Scrapy runs on Python 2.7 and Python 3.3 or above -(except on Windows where Python 3 is not supported yet). +Scrapy runs on Python 2.7 and Python 3.3 or above. -If you’re already familiar with installation of Python packages, +If you're using `Anaconda`_ or `Miniconda`_, you can install the package from +the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows +and OS X. + +To install Scrapy using ``conda``, run:: + + conda install -c conda-forge scrapy + +Alternatively, if you’re already familiar with installation of Python packages, you can install Scrapy and its dependencies from PyPI with:: pip install Scrapy +Note that sometimes this may require solving compilation issues for some Scrapy +dependencies depending on your operating system, so be sure to check the +:ref:`intro-install-platform-notes`. + We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, to avoid conflicting with your system packages. @@ -108,42 +119,14 @@ Platform specific installation notes Windows ------- -* Install Python 2.7 from https://www.python.org/downloads/ +Though it's possible to install Scrapy on Windows using pip, we recommend you +to install `Anaconda`_ or `Miniconda`_ and use the package from the +`conda-forge`_ channel, which will avoid most installation issues. - You need to adjust ``PATH`` environment variable to include paths to - the Python executable and additional scripts. The following paths need to be - added to ``PATH``:: +Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: - C:\Python27\;C:\Python27\Scripts\; + conda install -c conda-forge scrapy - To update the ``PATH`` open a Command prompt and run:: - - c:\python27\python.exe c:\python27\tools\scripts\win_add2path.py - - Close the command prompt window and reopen it so changes take effect, run the - following command and check it shows the expected Python version:: - - python --version - -* Install `pywin32` from http://sourceforge.net/projects/pywin32/ - - Be sure you download the architecture (win32 or amd64) that matches your system - -* *(Only required for Python<2.7.9)* Install `pip`_ from - https://pip.pypa.io/en/latest/installing/ - - Now open a Command prompt to check ``pip`` is installed correctly:: - - pip --version - -* At this point Python 2.7 and ``pip`` package manager must be working, let's - install Scrapy:: - - pip install Scrapy - -.. note:: - Python 3 is not supported on Windows. This is because Scrapy core requirement Twisted does not support - Python 3 on Windows. Ubuntu 12.04 or above --------------------- @@ -234,27 +217,8 @@ After any of these workarounds you should be able to install Scrapy:: pip install Scrapy -Anaconda --------- - - -Using Anaconda is an alternative to using a virtualenv and installing with ``pip``. - -.. note:: - - For Windows users, or if you have issues installing through ``pip``, this is - the recommended way to install Scrapy. - -If you already have `Anaconda`_ or `Miniconda`_ installed, the `conda-forge`_ -community have up-to-date packages for Linux, Windows and OS X. - -To install Scrapy using ``conda``, run:: - - conda install -c conda-forge scrapy - .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing/ -.. _Control Panel: https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx .. _lxml: http://lxml.de/ .. _parsel: https://pypi.python.org/pypi/parsel .. _w3lib: https://pypi.python.org/pypi/w3lib From e285b1d6c2aaa1fdfe788f1894b0196bc64d1be1 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 18:17:07 +0500 Subject: [PATCH 003/362] retry stats --- scrapy/downloadermiddlewares/retry.py | 14 ++++++++++++-- scrapy/downloadermiddlewares/stats.py | 4 +++- scrapy/utils/misc.py | 2 +- scrapy/utils/python.py | 11 +++++++++++ scrapy/utils/response.py | 3 ++- tests/test_downloadermiddleware_retry.py | 15 ++++++++++++--- tests/test_proxy_connect.py | 4 +++- 7 files changed, 44 insertions(+), 9 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index c9c512be8..d84697b14 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -22,6 +22,7 @@ from twisted.web.client import ResponseFailed from scrapy.exceptions import NotConfigured from scrapy.utils.response import response_status_message from scrapy.core.downloader.handlers.http11 import TunnelError +from scrapy.utils.python import global_object_name logger = logging.getLogger(__name__) @@ -35,16 +36,18 @@ class RetryMiddleware(object): ConnectionLost, TCPTimedOutError, ResponseFailed, IOError, TunnelError) - def __init__(self, settings): + def __init__(self, crawler): + settings = crawler.settings if not settings.getbool('RETRY_ENABLED'): raise NotConfigured self.max_retry_times = settings.getint('RETRY_TIMES') self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES')) self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') + self.stats = crawler.stats @classmethod def from_crawler(cls, crawler): - return cls(crawler.settings) + return cls(crawler) def process_response(self, request, response, spider): if request.meta.get('dont_retry', False): @@ -70,8 +73,15 @@ class RetryMiddleware(object): retryreq.meta['retry_times'] = retries retryreq.dont_filter = True retryreq.priority = request.priority + self.priority_adjust + + if isinstance(reason, Exception): + reason = global_object_name(reason.__class__) + + self.stats.inc_value('retry/count') + self.stats.inc_value('retry/reason_count/%s' % reason) return retryreq else: + self.stats.inc_value('retry/max_reached') logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s", {'request': request, 'retries': retries, 'reason': reason}, extra={'spider': spider}) diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index 9c0ad90a5..ef0aafce0 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -1,6 +1,8 @@ from scrapy.exceptions import NotConfigured from scrapy.utils.request import request_httprepr from scrapy.utils.response import response_httprepr +from scrapy.utils.python import global_object_name + class DownloaderStats(object): @@ -27,6 +29,6 @@ class DownloaderStats(object): return response def process_exception(self, request, exception, spider): - ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__) + ex_class = global_object_name(exception.__class__) self.stats.inc_value('downloader/exception_count', spider=spider) self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 30c9e5058..35f855007 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -113,7 +113,7 @@ def md5sum(file): m.update(d) return m.hexdigest() + def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" return True if rel is not None and 'nofollow' in rel.split() else False - diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 42fbbda7f..4c500abf4 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -344,3 +344,14 @@ def without_none_values(iterable): return {k: v for k, v in six.iteritems(iterable) if v is not None} except AttributeError: return type(iterable)((v for v in iterable if v is not None)) + + +def global_object_name(obj): + """ + Return full name of a global object. + + >>> from scrapy import Request + >>> global_object_name(Request) + 'scrapy.http.request.Request' + """ + return "%s.%s" % (obj.__module__, obj.__name__) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index deb5741be..bf276b5ca 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -43,7 +43,8 @@ def get_meta_refresh(response): def response_status_message(status): """Return status code plus status text descriptive message """ - return '%s %s' % (status, to_native_str(http.RESPONSES.get(int(status), "Unknown Status"))) + message = http.RESPONSES.get(int(status), "Unknown Status") + return '%s %s' % (status, to_native_str(message)) def response_httprepr(response): diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index e129b71f8..b833cb448 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -13,9 +13,9 @@ from scrapy.utils.test import get_crawler class RetryTest(unittest.TestCase): def setUp(self): - crawler = get_crawler(Spider) - self.spider = crawler._create_spider('foo') - self.mw = RetryMiddleware.from_crawler(crawler) + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('foo') + self.mw = RetryMiddleware.from_crawler(self.crawler) self.mw.max_retry_times = 2 def test_priority_adjust(self): @@ -70,6 +70,10 @@ class RetryTest(unittest.TestCase): # discard it assert self.mw.process_response(req, rsp, self.spider) is rsp + assert self.crawler.stats.get_value('retry/max_reached') == 1 + assert self.crawler.stats.get_value('retry/reason_count/503 Service Unavailable') == 2 + assert self.crawler.stats.get_value('retry/count') == 2 + def test_twistederrors(self): exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, @@ -79,6 +83,11 @@ class RetryTest(unittest.TestCase): req = Request('http://www.scrapytest.org/%s' % exc.__name__) self._test_retry_exception(req, exc('foo')) + stats = self.crawler.stats + assert stats.get_value('retry/max_reached') == len(exceptions) + assert stats.get_value('retry/count') == len(exceptions) * 2 + assert stats.get_value('retry/reason_count/twisted.internet.defer.TimeoutError') == 2 + def _test_retry_exception(self, req, exception): # first retry req = self.mw.process_exception(req, exception, self.spider) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 0f06fd53d..6213a51e8 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -101,7 +101,9 @@ class ProxyConnectTestCase(TestCase): self._assert_got_response_code(407, l) def _assert_got_response_code(self, code, log): + print(log) self.assertEqual(str(log).count('Crawled (%d)' % code), 1) def _assert_got_tunnel_error(self, log): - self.assertEqual(str(log).count('TunnelError'), 1) + print(log) + self.assertIn('TunnelError', str(log)) From 39df675f091cf904dc904acd9538a0bebe2e55cd Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 14 Feb 2017 23:28:50 +0500 Subject: [PATCH 004/362] make retry middleware changes backwards compatible --- scrapy/downloadermiddlewares/retry.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index d84697b14..549d74f46 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -36,18 +36,16 @@ class RetryMiddleware(object): ConnectionLost, TCPTimedOutError, ResponseFailed, IOError, TunnelError) - def __init__(self, crawler): - settings = crawler.settings + def __init__(self, settings): if not settings.getbool('RETRY_ENABLED'): raise NotConfigured self.max_retry_times = settings.getint('RETRY_TIMES') self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES')) self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') - self.stats = crawler.stats @classmethod def from_crawler(cls, crawler): - return cls(crawler) + return cls(crawler.settings) def process_response(self, request, response, spider): if request.meta.get('dont_retry', False): @@ -65,6 +63,7 @@ class RetryMiddleware(object): def _retry(self, request, reason, spider): retries = request.meta.get('retry_times', 0) + 1 + stats = spider.crawler.stats if retries <= self.max_retry_times: logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s", {'request': request, 'retries': retries, 'reason': reason}, @@ -77,11 +76,11 @@ class RetryMiddleware(object): if isinstance(reason, Exception): reason = global_object_name(reason.__class__) - self.stats.inc_value('retry/count') - self.stats.inc_value('retry/reason_count/%s' % reason) + stats.inc_value('retry/count') + stats.inc_value('retry/reason_count/%s' % reason) return retryreq else: - self.stats.inc_value('retry/max_reached') + stats.inc_value('retry/max_reached') logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s", {'request': request, 'retries': retries, 'reason': reason}, extra={'spider': spider}) From bb7d99ed81e70a5a59d47e69995ac5c025a6a8c0 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 01:30:45 +0500 Subject: [PATCH 005/362] drop unneeded urlparse call --- scrapy/linkextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index e5d21e174..8676c3b92 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -101,7 +101,7 @@ class FilteringLinkExtractor(object): links = [x for x in links if self._link_allowed(x)] if self.canonicalize: for link in links: - link.url = canonicalize_url(urlparse(link.url)) + link.url = canonicalize_url(link.url) links = self.link_extractor._process_links(links) return links From 47f7da8724b5a453979c2b0d10114c23ac1bd170 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 02:03:26 +0500 Subject: [PATCH 006/362] canonicalize=False by default for LinkExtractor. Fixes GH-1941. --- docs/topics/link-extractors.rst | 9 +++++-- scrapy/linkextractors/lxmlhtml.py | 22 +++++++++++---- .../link_extractor/sgml_linkextractor.html | 1 + tests/test_linkextractors.py | 27 +++++++++++++++++++ 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 75bdb4142..01d7f0b97 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -51,7 +51,7 @@ LxmlLinkExtractor :synopsis: lxml's HTMLParser-based link extractors -.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, strip=True) +.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, strip=True) LxmlLinkExtractor is the recommended link extractor with handy filtering options. It is implemented using lxml's robust HTMLParser. @@ -103,7 +103,12 @@ LxmlLinkExtractor :type attrs: list :param canonicalize: canonicalize each extracted url (using - w3lib.url.canonicalize_url). Defaults to ``True``. + w3lib.url.canonicalize_url). Defaults to ``False``. + Note that canonicalize_url is meant for duplicate checking; + it can change the URL visible at server side, so the response can be + different for requests with canonicalized and raw URLs. If you're + using LinkExtractor to follow links it is more robust to + keep the default ``canonicalize=False``. :type canonicalize: boolean :param unique: whether duplicate filtering should be applied to extracted diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index c284f1905..a7092f9b8 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -6,6 +6,7 @@ from six.moves.urllib.parse import urljoin import lxml.etree as etree from w3lib.html import strip_html5_whitespace +from w3lib.url import canonicalize_url from scrapy.link import Link from scrapy.utils.misc import arg_to_iter, rel_has_nofollow @@ -29,12 +30,17 @@ def _nons(tag): class LxmlParserLinkExtractor(object): def __init__(self, tag="a", attr="href", process=None, unique=False, - strip=True): + strip=True, canonicalized=False): self.scan_tag = tag if callable(tag) else lambda t: t == tag self.scan_attr = attr if callable(attr) else lambda a: a == attr self.process_attr = process if callable(process) else lambda v: v self.unique = unique self.strip = strip + if canonicalized: + self.link_key = lambda link: link.url + else: + self.link_key = lambda link: canonicalize_url(link.url, + keep_fragments=True) def _iter_links(self, document): for el in document.iter(etree.Element): @@ -82,21 +88,27 @@ class LxmlParserLinkExtractor(object): def _deduplicate_if_needed(self, links): if self.unique: - return unique_list(links, key=lambda link: link.url) + return unique_list(links, key=self.link_key) return links class LxmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=True, + tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, deny_extensions=None, restrict_css=(), strip=True): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) tag_func = lambda x: x in tags attr_func = lambda x: x in attrs - lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process=process_value, strip=strip) + lx = LxmlParserLinkExtractor( + tag=tag_func, + attr=attr_func, + unique=unique, + process=process_value, + strip=strip, + canonicalized=canonicalize + ) super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, allow_domains=allow_domains, deny_domains=deny_domains, diff --git a/tests/sample_data/link_extractor/sgml_linkextractor.html b/tests/sample_data/link_extractor/sgml_linkextractor.html index fbb803f2d..7d5db368a 100644 --- a/tests/sample_data/link_extractor/sgml_linkextractor.html +++ b/tests/sample_data/link_extractor/sgml_linkextractor.html @@ -11,6 +11,7 @@ sample 3 text sample 3 repetition +sample 3 repetition with fragment inner tag href with whitespaces diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 340c64f35..50484f060 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -30,6 +30,7 @@ class Base: Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), @@ -41,6 +42,7 @@ class Base: Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') ]) def test_extract_filter_allow_with_duplicates(self): @@ -50,6 +52,27 @@ class Base: Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment') + ]) + + def test_extract_filter_allow_with_duplicates_canonicalize(self): + lx = self.extractor_cls(allow=('sample', ), unique=False, + canonicalize=True) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition with fragment') + ]) + + def test_extract_filter_allow_no_duplicates_canonicalize(self): + lx = self.extractor_cls(allow=('sample',), unique=True, + canonicalize=True) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), ]) def test_extract_filter_allow_and_deny(self): @@ -73,6 +96,8 @@ class Base: Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', + text='sample 3 repetition with fragment') ]) lx = self.extractor_cls(allow='sample', deny='3') @@ -280,6 +305,7 @@ class Base: Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), @@ -291,6 +317,7 @@ class Base: Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample2.jpg', text=u''), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), From df446d167f6b539c4a79fb64d91cbc1607f30acb Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 03:11:59 +0500 Subject: [PATCH 007/362] fix deprecated link extractors --- scrapy/linkextractors/sgml.py | 18 +++++++++++------- tests/test_linkextractors_deprecated.py | 2 ++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 11ff7a261..f4ca4262a 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -6,7 +6,7 @@ from six.moves.urllib.parse import urljoin import warnings from sgmllib import SGMLParser -from w3lib.url import safe_url_string +from w3lib.url import safe_url_string, canonicalize_url from w3lib.html import strip_html5_whitespace from scrapy.link import Link @@ -20,7 +20,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning class BaseSgmlLinkExtractor(SGMLParser): def __init__(self, tag="a", attr="href", unique=False, process_value=None, - strip=True): + strip=True, canonicalized=False): warnings.warn( "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. " "Please use scrapy.linkextractors.LinkExtractor", @@ -33,6 +33,11 @@ class BaseSgmlLinkExtractor(SGMLParser): self.current_link = None self.unique = unique self.strip = strip + if canonicalized: + self.link_key = lambda link: link.url + else: + self.link_key = lambda link: canonicalize_url(link.url, + keep_fragments=True) def _extract_links(self, response_text, response_url, response_encoding, base_url=None): """ Do the real extraction work """ @@ -61,8 +66,7 @@ class BaseSgmlLinkExtractor(SGMLParser): The subclass should override it if necessary """ - links = unique_list(links, key=lambda link: link.url) if self.unique else links - return links + return unique_list(links, key=self.link_key) if self.unique else links def extract_links(self, response): # wrapper needed to allow to work directly with text @@ -107,10 +111,9 @@ class BaseSgmlLinkExtractor(SGMLParser): class SgmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, + tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, deny_extensions=None, restrict_css=(), strip=True): - warnings.warn( "SgmlLinkExtractor is deprecated and will be removed in future releases. " "Please use scrapy.linkextractors.LinkExtractor", @@ -124,7 +127,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor): with warnings.catch_warnings(): warnings.simplefilter('ignore', ScrapyDeprecationWarning) lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process_value=process_value, strip=strip) + unique=unique, process_value=process_value, strip=strip, + canonicalized=canonicalize) super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, allow_domains=allow_domains, deny_domains=deny_domains, diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py index fef227aa1..794f85e0f 100644 --- a/tests/test_linkextractors_deprecated.py +++ b/tests/test_linkextractors_deprecated.py @@ -121,6 +121,7 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), @@ -190,6 +191,7 @@ class RegexLinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_links(self.response), [Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'),]) From 2b4d46315f2ce7ce20e7355fe4ad184bb041d6f4 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 21 Feb 2017 00:05:40 +0500 Subject: [PATCH 008/362] TST fixed compatibility with new link extractor whitespace handling --- tests/test_http_response.py | 1 + tests/test_linkextractors.py | 17 ++++++++++++++--- tests/test_linkextractors_deprecated.py | 1 + 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 924bb7979..779f5a71c 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -392,6 +392,7 @@ class TextResponseTest(BaseResponseTest): 'http://example.com/sample2.html', 'http://example.com/sample3.html', 'http://example.com/sample3.html', + 'http://example.com/sample3.html#foo', 'http://www.google.com/something', 'http://example.com/innertag.html' ] diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 50484f060..1d7c4f311 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -13,6 +13,7 @@ from tests import get_testdata class Base: class LinkExtractorTestCase(unittest.TestCase): extractor_cls = None + escapes_whitespace = False def setUp(self): body = get_testdata('link_extractor', 'sgml_linkextractor.html') @@ -26,6 +27,11 @@ class Base: def test_extract_all_links(self): lx = self.extractor_cls() + if self.escapes_whitespace: + page4_url = 'http://example.com/page%204.html' + else: + page4_url = 'http://example.com/page 4.html' + self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), @@ -33,7 +39,7 @@ class Base: Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), + Link(url=page4_url, text=u'href with whitespaces'), ]) def test_extract_filter_allow(self): @@ -301,6 +307,11 @@ class Base: def test_attrs(self): lx = self.extractor_cls(attrs="href") + if self.escapes_whitespace: + page4_url = 'http://example.com/page%204.html' + else: + page4_url = 'http://example.com/page 4.html' + self.assertEqual(lx.extract_links(self.response), [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), @@ -308,7 +319,7 @@ class Base: Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), + Link(url=page4_url, text=u'href with whitespaces'), ]) lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=()) @@ -320,7 +331,7 @@ class Base: Link(url='http://example.com/sample3.html#foo', text='sample 3 repetition with fragment'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), + Link(url=page4_url, text=u'href with whitespaces'), ]) lx = self.extractor_cls(attrs=None) diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py index 794f85e0f..1366971be 100644 --- a/tests/test_linkextractors_deprecated.py +++ b/tests/test_linkextractors_deprecated.py @@ -143,6 +143,7 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase): class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = SgmlLinkExtractor + escapes_whitespace = True def test_deny_extensions(self): html = """asd and """ From d60642e1755b004e46b87e2011c1873f25895bb8 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Fri, 12 Aug 2016 00:45:42 -0300 Subject: [PATCH 009/362] data URI download handler. --- scrapy/core/downloader/handlers/data.py | 93 +++++++++++++++++++++++++ tests/test_downloader_handlers.py | 58 ++++++++++++++- 2 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 scrapy/core/downloader/handlers/data.py diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py new file mode 100644 index 000000000..637a6f9c0 --- /dev/null +++ b/scrapy/core/downloader/handlers/data.py @@ -0,0 +1,93 @@ +import base64 +import re +from six.moves.urllib.parse import unquote + +from scrapy.http import TextResponse +from scrapy.responsetypes import responsetypes +from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.decorators import defers + + +# ASCII characters. +_char = set(map(chr, range(127))) + +# RFC 2045 token. +_token = r'[{}]+'.format(re.escape(''.join(_char - + # Control characters. + set(map(chr, range(0, 32))) - + # tspecials and space. + set('()<>@,;:\\"/[]?= ')))) + +# RFC 822 quoted-string, without surrounding quotation marks. +_quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format( + re.escape(''.join(_char - {'"', '\\', '\r'})), + re.escape(''.join(_char)) +) + +# RFC 2397 mediatype. +_mediatype_pattern = re.compile(r'{token}/{token}'.format(token=_token)) + +_mediatype_parameter_pattern = re.compile( + r';({token})=(?:({token})|"({quoted})")'.format(token=_token, + quoted=_quoted_string) +) + + +class DataURIDownloadHandler(object): + def __init__(self, settings): + super(DataURIDownloadHandler, self).__init__() + + @defers + def download_request(self, request, spider): + url = request.url + + scheme, url = url.split(':', 1) + if scheme != 'data': + raise ValueError("not a data URI") + + # RFC 3986 section 2.1 allows percent encoding to escape characters + # that would be interpreted as delimiters, implying that actual + # delimiters should not be percent-encoded. + # Decoding before parsing will allow malformed URIs with + # percent-encoded delimiters, but it makes parsing easier and should + # not affect well-formed URIs, as the delimiters used in this URI + # scheme are not allowed, percent-encoded or not, in tokens. + url = unquote(url) + + media_type = "text/plain" + media_type_params = CaselessDict() + + m = _mediatype_pattern.match(url) + if m: + media_type = m.group() + url = url[m.end():] + else: + media_type_params['charset'] = "US-ASCII" + + while True: + m = _mediatype_parameter_pattern.match(url) + if m: + attribute, value, value_quoted = m.groups() + if value_quoted: + value = re.sub(r'\\(.)', '\1', value_quoted) + media_type_params[attribute] = value + url = url[m.end():] + else: + break + + is_base64, data = url.split(',', 1) + if is_base64: + if is_base64 != ";base64": + raise ValueError("invalid data URI") + data = base64.b64decode(data) + + respcls = responsetypes.from_mimetype(media_type) + + resp_kwargs = {} + + if media_type: + media_type = media_type.split('/') + if issubclass(respcls, TextResponse) and media_type[0] == 'text': + resp_kwargs['encoding'] = media_type_params.get('charset') + + return respcls(url=request.url, body=data, **resp_kwargs) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c1683fb3e..c21a1670f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -6,7 +6,6 @@ try: from unittest import mock except ImportError: import mock -import shutil from twisted.trial import unittest from twisted.protocols.policies import WrappingFactory @@ -20,6 +19,7 @@ from twisted.cred import portal, checkers, credentials from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers +from scrapy.core.downloader.handlers.data import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler @@ -29,6 +29,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spiders import Spider from scrapy.http import Request from scrapy.http.response.text import TextResponse +from scrapy.responsetypes import responsetypes from scrapy.settings import Settings from scrapy.utils.test import get_crawler, skip_if_no_boto from scrapy.utils.python import to_bytes @@ -828,3 +829,58 @@ class AnonymousFTPTestCase(BaseFTPTestCase): def tearDown(self): shutil.rmtree(self.directory) + + +class DataURITestCase(unittest.TestCase): + + def setUp(self): + self.download_handler = DataURIDownloadHandler(Settings()) + self.download_request = self.download_handler.download_request + self.spider = Spider('foo') + + def test_default_mediatype_encoding(self): + def _test(response): + self.assertEquals(response.text, 'A brief note') + self.assertEquals(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEquals(response.encoding, "US-ASCII") + + request = Request("data:,A%20brief%20note") + return self.download_request(request, self.spider).addCallback(_test) + + def test_default_mediatype(self): + def _test(response): + self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEquals(response.encoding, "iso-8859-7") + + request = Request("data:;charset=iso-8859-7,%be%d3%be") + return self.download_request(request, self.spider).addCallback(_test) + + def test_text_charset(self): + def _test(response): + self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(response.encoding, "iso-8859-7") + + request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") + return self.download_request(request, self.spider).addCallback(_test) + + def test_mediatype_parameters(self): + def _test(response): + self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(type(response), + responsetypes.from_mimetype("text/plain")) + self.assertEquals(response.encoding, "utf-8") + + request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;' + 'charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22' + ',%CE%8E%CE%A3%CE%8E') + return self.download_request(request, self.spider).addCallback(_test) + + def test_base64(self): + def _test(response): + self.assertEquals(response.text, 'Hello, world.') + + request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D') + return self.download_request(request, self.spider).addCallback(_test) From 7e9f2c31d78dcc798a281e5ba6ddf91714dc9cc7 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Fri, 12 Aug 2016 02:09:35 -0300 Subject: [PATCH 010/362] Ensure bytes objects when needed in data URI downloader. --- scrapy/core/downloader/handlers/data.py | 29 ++++++++++++++++++------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py index 637a6f9c0..30c39865f 100644 --- a/scrapy/core/downloader/handlers/data.py +++ b/scrapy/core/downloader/handlers/data.py @@ -1,6 +1,12 @@ import base64 import re -from six.moves.urllib.parse import unquote + +import six + +if six.PY2: + from urllib import unquote +else: + from urllib.parse import unquote_to_bytes as unquote from scrapy.http import TextResponse from scrapy.responsetypes import responsetypes @@ -24,12 +30,19 @@ _quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format( re.escape(''.join(_char)) ) +# Encode the regular expression strings to make them into bytes, as Python 3 +# bytes have no format() method, but bytes must be passed to re.compile() in +# order to make a pattern object that can be used to match on bytes. + # RFC 2397 mediatype. -_mediatype_pattern = re.compile(r'{token}/{token}'.format(token=_token)) +_mediatype_pattern = re.compile( + r'{token}/{token}'.format(token=_token).encode() +) _mediatype_parameter_pattern = re.compile( r';({token})=(?:({token})|"({quoted})")'.format(token=_token, - quoted=_quoted_string) + quoted=_quoted_string + ).encode() ) @@ -59,7 +72,7 @@ class DataURIDownloadHandler(object): m = _mediatype_pattern.match(url) if m: - media_type = m.group() + media_type = m.group().decode() url = url[m.end():] else: media_type_params['charset'] = "US-ASCII" @@ -69,15 +82,15 @@ class DataURIDownloadHandler(object): if m: attribute, value, value_quoted = m.groups() if value_quoted: - value = re.sub(r'\\(.)', '\1', value_quoted) - media_type_params[attribute] = value + value = re.sub(br'\\(.)', r'\1', value_quoted) + media_type_params[attribute.decode()] = value.decode() url = url[m.end():] else: break - is_base64, data = url.split(',', 1) + is_base64, data = url.split(b',', 1) if is_base64: - if is_base64 != ";base64": + if is_base64 != b";base64": raise ValueError("invalid data URI") data = base64.b64decode(data) From 3397d27574d03a222b6a790004fcaa1a03711d75 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 8 Feb 2017 12:32:00 -0200 Subject: [PATCH 011/362] Test for binary body content from data URI downloader. --- tests/test_downloader_handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c21a1670f..cfcdcd8f8 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -861,6 +861,7 @@ class DataURITestCase(unittest.TestCase): def test_text_charset(self): def _test(response): self.assertEquals(response.text, u'\u038e\u03a3\u038e') + self.assertEquals(response.body, b'\xbe\xd3\xbe') self.assertEquals(response.encoding, "iso-8859-7") request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") From c847e7d4d00647a953a3c5458d198439c30b87f7 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Wed, 8 Feb 2017 12:38:49 -0200 Subject: [PATCH 012/362] Use w3lib data URI parser. --- scrapy/core/downloader/handlers/data.py | 99 ++----------------------- 1 file changed, 8 insertions(+), 91 deletions(-) diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py index 30c39865f..d102f2b73 100644 --- a/scrapy/core/downloader/handlers/data.py +++ b/scrapy/core/downloader/handlers/data.py @@ -1,106 +1,23 @@ -import base64 -import re - -import six - -if six.PY2: - from urllib import unquote -else: - from urllib.parse import unquote_to_bytes as unquote +from w3lib.url import parse_data_uri from scrapy.http import TextResponse from scrapy.responsetypes import responsetypes -from scrapy.utils.datatypes import CaselessDict from scrapy.utils.decorators import defers -# ASCII characters. -_char = set(map(chr, range(127))) - -# RFC 2045 token. -_token = r'[{}]+'.format(re.escape(''.join(_char - - # Control characters. - set(map(chr, range(0, 32))) - - # tspecials and space. - set('()<>@,;:\\"/[]?= ')))) - -# RFC 822 quoted-string, without surrounding quotation marks. -_quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format( - re.escape(''.join(_char - {'"', '\\', '\r'})), - re.escape(''.join(_char)) -) - -# Encode the regular expression strings to make them into bytes, as Python 3 -# bytes have no format() method, but bytes must be passed to re.compile() in -# order to make a pattern object that can be used to match on bytes. - -# RFC 2397 mediatype. -_mediatype_pattern = re.compile( - r'{token}/{token}'.format(token=_token).encode() -) - -_mediatype_parameter_pattern = re.compile( - r';({token})=(?:({token})|"({quoted})")'.format(token=_token, - quoted=_quoted_string - ).encode() -) - - class DataURIDownloadHandler(object): def __init__(self, settings): super(DataURIDownloadHandler, self).__init__() @defers def download_request(self, request, spider): - url = request.url - - scheme, url = url.split(':', 1) - if scheme != 'data': - raise ValueError("not a data URI") - - # RFC 3986 section 2.1 allows percent encoding to escape characters - # that would be interpreted as delimiters, implying that actual - # delimiters should not be percent-encoded. - # Decoding before parsing will allow malformed URIs with - # percent-encoded delimiters, but it makes parsing easier and should - # not affect well-formed URIs, as the delimiters used in this URI - # scheme are not allowed, percent-encoded or not, in tokens. - url = unquote(url) - - media_type = "text/plain" - media_type_params = CaselessDict() - - m = _mediatype_pattern.match(url) - if m: - media_type = m.group().decode() - url = url[m.end():] - else: - media_type_params['charset'] = "US-ASCII" - - while True: - m = _mediatype_parameter_pattern.match(url) - if m: - attribute, value, value_quoted = m.groups() - if value_quoted: - value = re.sub(br'\\(.)', r'\1', value_quoted) - media_type_params[attribute.decode()] = value.decode() - url = url[m.end():] - else: - break - - is_base64, data = url.split(b',', 1) - if is_base64: - if is_base64 != b";base64": - raise ValueError("invalid data URI") - data = base64.b64decode(data) - - respcls = responsetypes.from_mimetype(media_type) + uri = parse_data_uri(request.url) + respcls = responsetypes.from_mimetype(uri.media_type) resp_kwargs = {} + if (issubclass(respcls, TextResponse) and + uri.media_type.split('/')[0] == 'text'): + charset = uri.media_type_parameters.get('charset') + resp_kwargs['encoding'] = charset - if media_type: - media_type = media_type.split('/') - if issubclass(respcls, TextResponse) and media_type[0] == 'text': - resp_kwargs['encoding'] = media_type_params.get('charset') - - return respcls(url=request.url, body=data, **resp_kwargs) + return respcls(url=request.url, body=uri.data, **resp_kwargs) From 121a668a479c3b681581a36c7da090518fbc2120 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Sun, 12 Feb 2017 11:23:21 -0200 Subject: [PATCH 013/362] Rename data URI downloader module. --- scrapy/core/downloader/handlers/{data.py => datauri.py} | 0 tests/test_downloader_handlers.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename scrapy/core/downloader/handlers/{data.py => datauri.py} (100%) diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/datauri.py similarity index 100% rename from scrapy/core/downloader/handlers/data.py rename to scrapy/core/downloader/handlers/datauri.py diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index cfcdcd8f8..b27245a36 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -19,7 +19,7 @@ from twisted.cred import portal, checkers, credentials from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers -from scrapy.core.downloader.handlers.data import DataURIDownloadHandler +from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler From 3139f4a5f700a100b6ab6890677dac808b01835f Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Sun, 12 Feb 2017 11:23:56 -0200 Subject: [PATCH 014/362] Add data URI download handler to settings. --- scrapy/settings/default_settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index e0e39120c..2251d3db5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -67,6 +67,7 @@ DOWNLOAD_DELAY = 0 DOWNLOAD_HANDLERS = {} DOWNLOAD_HANDLERS_BASE = { + 'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler', 'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler', 'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', 'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler', From 96a570a93a78c977f7554179face6123d338ade0 Mon Sep 17 00:00:00 2001 From: MikeinRealLife Date: Wed, 22 Feb 2017 21:17:34 -0800 Subject: [PATCH 015/362] fixed ticket #2574 --- docs/topics/request-response.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 9a6e0d1b6..018d14100 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -304,6 +304,8 @@ Those are: * :reqmeta:`download_maxsize` * :reqmeta:`download_latency` * :reqmeta:`proxy` +* ``ftp_user`` (See :setting:`FTP_USER` for more info) +* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) .. reqmeta:: bindaddress @@ -332,6 +334,8 @@ used to control Scrapy behavior, this one is supposed to be read-only. .. _topics-request-response-ref-request-subclasses: +.. _topics-request-response-ref-request-subclasses: + Request subclasses ================== From 441f25507ea37b098e14bf3a1f9de7ccbe5a1cc5 Mon Sep 17 00:00:00 2001 From: MikeinRealLife Date: Wed, 22 Feb 2017 21:23:27 -0800 Subject: [PATCH 016/362] fixed typo removed duplicate line --- docs/topics/request-response.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 018d14100..06e245b16 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -334,8 +334,6 @@ used to control Scrapy behavior, this one is supposed to be read-only. .. _topics-request-response-ref-request-subclasses: -.. _topics-request-response-ref-request-subclasses: - Request subclasses ================== From 4274f0d4d4b3a79ebb76f61f4f9d5d3fb379d42b Mon Sep 17 00:00:00 2001 From: mangogao Date: Sat, 25 Feb 2017 15:44:20 +0800 Subject: [PATCH 017/362] Add omitted "self" arguments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit “self” argument is omitted in some methods. --- scrapy/templates/project/module/middlewares.py.tmpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 42318fec2..292bf572e 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -20,14 +20,14 @@ class ${ProjectName}SpiderMiddleware(object): crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) return s - def process_spider_input(response, spider): + def process_spider_input(self, response, spider): # Called for each response that goes through the spider # middleware and into the spider. # Should return None or raise an exception. return None - def process_spider_output(response, result, spider): + def process_spider_output(self, response, result, spider): # Called with the results returned from the Spider, after # it has processed the response. @@ -35,7 +35,7 @@ class ${ProjectName}SpiderMiddleware(object): for i in result: yield i - def process_spider_exception(response, exception, spider): + def process_spider_exception(self, response, exception, spider): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. @@ -43,7 +43,7 @@ class ${ProjectName}SpiderMiddleware(object): # or Item objects. pass - def process_start_requests(start_requests, spider): + def process_start_requests(self, start_requests, spider): # Called with the start requests of the spider, and works # similarly to the process_spider_output() method, except # that it doesn’t have a response associated. From e85f0db1285d7338330e0a7575dcc59c76375d6a Mon Sep 17 00:00:00 2001 From: Arvind Chembarpu Date: Sat, 25 Feb 2017 16:48:17 +0530 Subject: [PATCH 018/362] Use single quotes uniformly The default spider template mixes single and double quotes. --- scrapy/templates/spiders/basic.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 99e5d43b2..1cfe9cc9d 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -3,8 +3,8 @@ import scrapy class $classname(scrapy.Spider): - name = "$name" - allowed_domains = ["$domain"] + name = '$name' + allowed_domains = ['$domain'] start_urls = ['http://$domain/'] def parse(self, response): From 0b90c3b43c4eedc891056fa433e0608afbe6cd32 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 27 Feb 2017 17:42:00 +0100 Subject: [PATCH 019/362] Re-enable FTP tests on Python 3 --- scrapy/core/downloader/handlers/ftp.py | 7 ++++--- tests/py3-ignores.txt | 7 ------- tests/test_downloader_handlers.py | 27 ++++++++++++-------------- 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 1398140b4..933bc7e8d 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -39,12 +39,13 @@ from twisted.internet.protocol import Protocol, ClientCreator from scrapy.http import Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_bytes class ReceivedDataProtocol(Protocol): def __init__(self, filename=None): self.__filename = filename - self.body = open(filename, "w") if filename else BytesIO() + self.body = open(filename, "wb") if filename else BytesIO() self.size = 0 def dataReceived(self, data): @@ -97,7 +98,7 @@ class FTPDownloadHandler(object): protocol.close() body = protocol.filename or protocol.body.read() headers = {"local filename": protocol.filename or '', "size": protocol.size} - return respcls(url=request.url, status=200, body=body, headers=headers) + return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers) def _failed(self, result, request): message = result.getErrorMessage() @@ -106,6 +107,6 @@ class FTPDownloadHandler(object): if m: ftpcode = m.group() httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"]) - return Response(url=request.url, status=httpcode, body=message) + return Response(url=request.url, status=httpcode, body=to_bytes(message)) raise result.type(result.value) diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index ec2947003..313e74ec9 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,13 +1,6 @@ tests/test_linkextractors_deprecated.py tests/test_proxy_connect.py -scrapy/xlib/tx/iweb.py -scrapy/xlib/tx/interfaces.py -scrapy/xlib/tx/endpoints.py -scrapy/xlib/tx/client.py -scrapy/xlib/tx/_newclient.py -scrapy/xlib/tx/__init__.py -scrapy/core/downloader/handlers/ftp.py scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py scrapy/linkextractors/htmlparser.py diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index c1683fb3e..e49a514b8 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -687,9 +687,6 @@ class BaseFTPTestCase(unittest.TestCase): password = "passwd" req_meta = {"ftp_user": username, "ftp_password": password} - if six.PY3: - skip = "Twisted missing ftp support for PY3" - def setUp(self): from twisted.protocols.ftp import FTPRealm, FTPFactory from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler @@ -700,8 +697,8 @@ class BaseFTPTestCase(unittest.TestCase): userdir = os.path.join(self.directory, self.username) os.mkdir(userdir) fp = FilePath(userdir) - fp.child('file.txt').setContent("I have the power!") - fp.child('file with spaces.txt').setContent("Moooooooooo power!") + fp.child('file.txt').setContent(b"I have the power!") + fp.child('file with spaces.txt').setContent(b"Moooooooooo power!") # setup server realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) @@ -736,8 +733,8 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.status, 200) - self.assertEqual(r.body, 'I have the power!') - self.assertEqual(r.headers, {'Local Filename': [''], 'Size': ['17']}) + self.assertEqual(r.body, b'I have the power!') + self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']}) return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): @@ -749,8 +746,8 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.status, 200) - self.assertEqual(r.body, 'Moooooooooo power!') - self.assertEqual(r.headers, {'Local Filename': [''], 'Size': ['18']}) + self.assertEqual(r.body, b'Moooooooooo power!') + self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']}) return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): @@ -763,7 +760,7 @@ class BaseFTPTestCase(unittest.TestCase): return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): - local_fname = "/tmp/file.txt" + local_fname = b"/tmp/file.txt" meta = {"ftp_local_filename": local_fname} meta.update(self.req_meta) request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, @@ -772,10 +769,10 @@ class BaseFTPTestCase(unittest.TestCase): def _test(r): self.assertEqual(r.body, local_fname) - self.assertEqual(r.headers, {'Local Filename': ['/tmp/file.txt'], 'Size': ['17']}) + self.assertEqual(r.headers, {b'Local Filename': [b'/tmp/file.txt'], b'Size': [b'17']}) self.assertTrue(os.path.exists(local_fname)) - with open(local_fname) as f: - self.assertEqual(f.read(), "I have the power!") + with open(local_fname, "rb") as f: + self.assertEqual(f.read(), b"I have the power!") os.remove(local_fname) return self._add_test_callbacks(d, _test) @@ -810,8 +807,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase): os.mkdir(self.directory) fp = FilePath(self.directory) - fp.child('file.txt').setContent("I have the power!") - fp.child('file with spaces.txt').setContent("Moooooooooo power!") + fp.child('file.txt').setContent(b"I have the power!") + fp.child('file with spaces.txt').setContent(b"Moooooooooo power!") # setup server for anonymous access realm = FTPRealm(anonymousRoot=self.directory) From f01ae6ffcd431b73f5358f9f876f8e9ee9be0113 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Thu, 23 Feb 2017 11:42:34 -0400 Subject: [PATCH 020/362] Handle data loss gracefully. Websites that return a wrong ``Content-Length`` header may cause a data loss error. Also when a chunked response is not finished properly. This change adds a new setting ``DOWNLOAD_FAIL_ON_DATALOSS`` (default: ``True``) and request.meta key ``download_fail_on_dataloss``. --- docs/topics/request-response.rst | 9 +++ docs/topics/settings.rst | 26 ++++++ scrapy/core/downloader/handlers/http11.py | 41 +++++++--- scrapy/settings/default_settings.py | 2 + tests/test_downloader_handlers.py | 99 ++++++++++++++++++++++- 5 files changed, 167 insertions(+), 10 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 9a6e0d1b6..214ac5640 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -303,6 +303,7 @@ Those are: * :reqmeta:`download_timeout` * :reqmeta:`download_maxsize` * :reqmeta:`download_latency` +* :reqmeta:`download_fail_on_dataloss` * :reqmeta:`proxy` .. reqmeta:: bindaddress @@ -330,6 +331,14 @@ started, i.e. HTTP message sent over the network. This meta key only becomes available when the response has been downloaded. While most other meta keys are used to control Scrapy behavior, this one is supposed to be read-only. +.. reqmeta:: download_fail_on_dataloss + +download_fail_on_dataloss +------------------------- + +Whether or not to fail on broken responses. See: +:setting:`DOWNLOAD_FAIL_ON_DATALOSS`. + .. _topics-request-response-ref-request-subclasses: Request subclasses diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f616742c4..ccdd02c4e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -604,6 +604,32 @@ If you want to disable it set to 0. This feature needs Twisted >= 11.1. +.. setting:: DOWNLOAD_FAIL_ON_DATALOSS + +DOWNLOAD_FAIL_ON_DATALOSS +------------------------- + +Default: ``True`` + +Whether or not to fail on broken responses, that is, declared +``Content-Length`` does not match content sent by the server or chunked +response was not properly finish. If ``True``, these responses raise a +``ResponseFailed([_DataLoss])`` error. If ``False``, these responses +are passed through and the flag ``dataloss`` is added to the response, i.e.: +``'dataloss' in response.flags`` is ``True``. + +Optionally, this can be set per-request basis by using the +:reqmeta:`download_fail_on_dataloss` Request.meta key to ``False``. + +.. note:: + + A broken response, or data loss error, may happen under several + circumstances, from server misconfiguration to network errors to data + corruption. It is up to the user to decide if it makes sense to process + broken responses considering they may contain partial or incomplete content. + If setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``, + the ``ResponseFailed([_DataLoss])`` failure will be retried as usual. + .. setting:: DUPEFILTER_CLASS DUPEFILTER_CLASS diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index b96c8c6fe..37e836809 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -12,9 +12,9 @@ from twisted.internet import defer, reactor, protocol from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from twisted.internet.error import TimeoutError -from twisted.web.http import PotentialDataLoss +from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool + HTTPConnectionPool, ResponseFailed from twisted.internet.endpoints import TCP4ClientEndpoint from scrapy.http import Headers @@ -51,13 +51,15 @@ class HTTP11DownloadHandler(object): warnings.warn(msg) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') + self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS') self._disconnect_timeout = 1 def download_request(self, request, spider): """Return a deferred for the HTTP download""" agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool, maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), - warnsize=getattr(spider, 'download_warnsize', self._default_warnsize)) + warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), + fail_on_dataloss=self._fail_on_dataloss) return agent.download_request(request) def close(self): @@ -233,13 +235,14 @@ class ScrapyAgent(object): _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, - maxsize=0, warnsize=0): + maxsize=0, warnsize=0, fail_on_dataloss=True): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress self._pool = pool self._maxsize = maxsize self._warnsize = warnsize + self._fail_on_dataloss = fail_on_dataloss self._txresponse = None def _get_agent(self, request, timeout): @@ -326,6 +329,7 @@ class ScrapyAgent(object): maxsize = request.meta.get('download_maxsize', self._maxsize) warnsize = request.meta.get('download_warnsize', self._warnsize) expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 + fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss) if maxsize and expected_size > maxsize: error_msg = ("Cancelling download of %(url)s: expected response " @@ -345,7 +349,8 @@ class ScrapyAgent(object): txresponse._transport._producer.loseConnection() d = defer.Deferred(_cancel) - txresponse.deliverBody(_ResponseReader(d, txresponse, request, maxsize, warnsize)) + txresponse.deliverBody(_ResponseReader( + d, txresponse, request, maxsize, warnsize, fail_on_dataloss)) # save response for timeouts self._txresponse = txresponse @@ -380,13 +385,16 @@ class _RequestBodyProducer(object): class _ResponseReader(protocol.Protocol): - def __init__(self, finished, txresponse, request, maxsize, warnsize): + def __init__(self, finished, txresponse, request, maxsize, warnsize, + fail_on_dataloss): self._finished = finished self._txresponse = txresponse self._request = request self._bodybuf = BytesIO() self._maxsize = maxsize self._warnsize = warnsize + self._fail_on_dataloss = fail_on_dataloss + self._fail_on_dataloss_warned = False self._reached_warnsize = False self._bytes_received = 0 @@ -415,7 +423,22 @@ class _ResponseReader(protocol.Protocol): body = self._bodybuf.getvalue() if reason.check(ResponseDone): self._finished.callback((self._txresponse, body, None)) - elif reason.check(PotentialDataLoss): + return + + if reason.check(PotentialDataLoss): self._finished.callback((self._txresponse, body, ['partial'])) - else: - self._finished.errback(reason) + return + + if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons): + if not self._fail_on_dataloss: + self._finished.callback((self._txresponse, body, ['dataloss'])) + return + + elif not self._fail_on_dataloss_warned: + logger.warn("Got data loss in %s. If you want to process broken " + "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" + " -- This message won't be shown in further requests", + self._txresponse.request.absoluteURI.decode()) + self._fail_on_dataloss_warned = True + + self._finished.errback(reason) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index e0e39120c..a5931a3d5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -79,6 +79,8 @@ DOWNLOAD_TIMEOUT = 180 # 3mins DOWNLOAD_MAXSIZE = 1024*1024*1024 # 1024m DOWNLOAD_WARNSIZE = 32*1024*1024 # 32m +DOWNLOAD_FAIL_ON_DATALOSS = True + DOWNLOADER = 'scrapy.core.downloader.Downloader' DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index e49a514b8..4e63b2038 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -13,9 +13,11 @@ from twisted.protocols.policies import WrappingFactory from twisted.python.filepath import FilePath from twisted.internet import reactor, defer, error from twisted.web import server, static, util, resource +from twisted.web._newclient import ResponseFailed +from twisted.web.http import _DataLoss from twisted.web.test.test_webclient import ForeverTakingResource, \ NoLengthResource, HostHeaderResource, \ - PayloadResource, BrokenDownloadResource + PayloadResource from twisted.cred import portal, checkers, credentials from w3lib.url import path_to_file_uri @@ -118,6 +120,52 @@ class ContentLengthHeaderResource(resource.Resource): return request.requestHeaders.getRawHeaders(b"content-length")[0] +class ChunkedResource(resource.Resource): + + def render(self, request): + def response(): + request.write(b"chunked ") + request.write(b"content\n") + request.finish() + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class BrokenChunkedResource(resource.Resource): + + def render(self, request): + def response(): + request.write(b"chunked ") + request.write(b"content\n") + # Disable terminating chunk on finish. + request.chunked = False + closeConnection(request) + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +class BrokenDownloadResource(resource.Resource): + + def render(self, request): + def response(): + request.setHeader(b"Content-Length", b"20") + request.write(b"partial") + closeConnection(request) + + reactor.callLater(0, response) + return server.NOT_DONE_YET + + +def closeConnection(request): + # We have to force a disconnection for HTTP/1.1 clients. Otherwise + # client keeps the connection open waiting for more data. + if hasattr(request.channel, 'loseConnection'): # twisted >=16.3.0 + request.channel.loseConnection() + else: + request.channel.transport.loseConnection() + request.finish() + + class EmptyContentTypeHeaderResource(resource.Resource): """ A testing resource which renders itself as the value of request body @@ -149,6 +197,8 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"host", HostHeaderResource()) r.putChild(b"payload", PayloadResource()) r.putChild(b"broken", BrokenDownloadResource()) + r.putChild(b"chunked", ChunkedResource()) + r.putChild(b"broken-chunked", BrokenChunkedResource()) r.putChild(b"contentlength", ContentLengthHeaderResource()) r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) self.site = server.Site(r, timeout=None) @@ -341,6 +391,53 @@ class Http11TestCase(HttpTestCase): d.addCallback(self.assertEquals, b"0123456789") return d + def test_download_chunked_content(self): + request = Request(self.getURL('chunked')) + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEquals, b"chunked content\n") + return d + + def test_download_broken_content_cause_data_loss(self, url='broken'): + request = Request(self.getURL(url)) + d = self.download_request(request, Spider('foo')) + + def checkDataLoss(failure): + if failure.check(ResponseFailed): + if any(r.check(_DataLoss) for r in failure.value.reasons): + return None + return failure + + d.addCallback(lambda _: self.fail("No DataLoss exception")) + d.addErrback(checkDataLoss) + return d + + def test_download_broken_chunked_content_cause_data_loss(self): + return self.test_download_broken_content_cause_data_loss('broken-chunked') + + def test_download_broken_content_allow_data_loss(self, url='broken'): + request = Request(self.getURL(url), meta={'download_fail_on_dataloss': False}) + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.flags) + d.addCallback(self.assertEqual, ['dataloss']) + return d + + def test_download_broken_chunked_content_allow_data_loss(self): + return self.test_download_broken_content_allow_data_loss('broken-chunked') + + def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'): + download_handler = self.download_handler_cls(Settings({ + 'DOWNLOAD_FAIL_ON_DATALOSS': False, + })) + request = Request(self.getURL(url)) + d = download_handler.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.flags) + d.addCallback(self.assertEqual, ['dataloss']) + return d + + def test_download_broken_chunked_content_allow_data_loss_via_setting(self): + return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked') + class Https11TestCase(Http11TestCase): scheme = 'https' From baed7c436f8c6e52a1aab62fbaf900c0a0f3bbda Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 11:18:26 +0200 Subject: [PATCH 021/362] WIP Add Referrer policies --- scrapy/spidermiddlewares/referer.py | 206 ++++++++++++++++++++++++- tests/test_spidermiddleware_referer.py | 74 +++++++++ 2 files changed, 279 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 6a8c46543..21b340f22 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,22 +2,226 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ +from six.moves.urllib.parse import urlsplit, urlunsplit from scrapy.http import Request from scrapy.exceptions import NotConfigured +from scrapy.utils.python import to_native_str + + +LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) + +class ReferrerPolicy(object): + + NOREFERRER_SCHEMES = LOCAL_SCHEMES + + def referrer(self, response, request): + raise NotImplementedError() + + def strip_url(self, url, origin_only=False): + """ + https://www.w3.org/TR/referrer-policy/#strip-url + + If url is null, return no referrer. + If url's scheme is a local scheme, then return no referrer. + Set url's username to the empty string. + Set url's password to null. + Set url's fragment to null. + If the origin-only flag is true, then: + Set url's path to null. + Set url's query to null. + Return url. + """ + if url is None or not url: + return None + parsed = urlsplit(url, allow_fragments=True) + + if parsed.scheme in self.NOREFERRER_SCHEMES: + return None + if parsed.username or parsed.password: + netloc = parsed.netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') + else: + netloc = parsed.netloc + return urlunsplit(( + parsed.scheme, + netloc, + '' if origin_only else parsed.path, + '' if origin_only else parsed.query, + '')) + + +class NoReferrerPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer + + The simplest policy is "no-referrer", which specifies that no referrer information + is to be sent along with requests made from a particular request client to any origin. + The header will be omitted entirely. + """ + name = "no-referrer" + + def referrer(self, response, request): + return None + + +class NoReferrerWhenDowngradePolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + + The "no-referrer-when-downgrade" policy sends a full URL + along with requests from a TLS-protected environment settings object + to a a priori authenticated URL, + and requests from request clients which are not TLS-protected to any origin. + + Requests from TLS-protected request clients to non-a priori authenticated URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + + This is a user agent's default behavior, if no policy is otherwise specified. + """ + name = "no-referrer-when-downgrade" + + def referrer(self, response, request): + target_url = request.url + + referrer_source = response.url + referrer_url = self.strip_url(referrer_source) + + # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer: + # + # If environment is TLS-protected + # and the origin of request's current URL is not an a priori authenticated URL, + # then return no referrer. + if urlsplit(referrer_source).scheme in ('https', 'ftps') and \ + urlsplit(target_url).scheme in ('http',): + return None + return referrer_url + + +class SameOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin + + The "same-origin" policy specifies that a full URL, stripped for use as a referrer, + is sent as referrer information when making same-origin requests from a particular request client. + + Cross-origin requests, on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = "same-origin" + + def referrer(self, response, request): + target_url = request.url + referrer_source = response.url + if urlsplit(referrer_source).netloc == urlsplit(target_url).netloc: + return self.strip_url(referrer_source) + else: + return None + + +class OriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-origin + + The "origin" policy specifies that only the ASCII serialization + of the origin of the request client is sent as referrer information + when making both same-origin requests and cross-origin requests + from a particular request client. + """ + name = "origin" + + def referrer(self, response, request): + return self.strip_url(referrer_source, origin_only=True) + + +class OriginWhenCrossOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin + + The "origin-when-cross-origin" policy specifies that a full URL, + stripped for use as a referrer, is sent as referrer information + when making same-origin requests from a particular request client, + and only the ASCII serialization of the origin of the request client + is sent as referrer information when making cross-origin requests + from a particular request client. + """ + name = "origin-when-cross-origin" + + def referrer(self, response, request): + target_url = request.url + referrer_source = response.url + + # same origin --> send full referrer + # different origin --> send only "origin" as referrer + if urlsplit(referrer_source).netloc != urlsplit(target_url).netloc: + origin_only = True + return self.strip_url(referrer_source, origin_only=origin_only) + + +class UnsafeUrlPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url + + The "unsafe-url" policy specifies that a full URL, stripped for use as a referrer, + is sent along with both cross-origin requests + and same-origin requests made from a particular request client. + + Note: The policy's name doesn't lie; it is unsafe. + This policy will leak origins and paths from TLS-protected resources + to insecure origins. + Carefully consider the impact of setting such a policy for potentially sensitive documents. + """ + name = "unsafe-url" + + def referrer(self, response, request): + referrer_source = response.url + return self.strip_url(referrer_source) + + +class LegacyPolicy(ReferrerPolicy): + def referrer(self, response, request): + return response.url + + +class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): + + NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') + + +_policies = {p.name: p for p in ( + NoReferrerPolicy, + NoReferrerWhenDowngradePolicy, + SameOriginPolicy, + OriginPolicy, + OriginWhenCrossOriginPolicy, + UnsafeUrlPolicy, +)} class RefererMiddleware(object): + def __init__(self, policy_class=DefaultReferrerPolicy): + self.default_policy = policy_class + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured return cls() + def policy(self, response, request): + policy_name = request.meta.get('referrer_policy') + if policy_name is None: + policy_name = to_native_str(response.headers.get('Referrer-Policy', '').decode('latin1')) + + policy_class = _policies.get(policy_name.lower(), self.default_policy) + return policy_class() + def process_spider_output(self, response, result, spider): def _set_referer(r): if isinstance(r, Request): - r.headers.setdefault('Referer', response.url) + referrer = self.policy(response, r).referrer(response, r) + if referrer is not None: + r.headers.setdefault('Referer', referrer) return r return (_set_referer(r) for r in result or ()) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index bd7673efb..f109bb248 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -19,3 +19,77 @@ class TestRefererMiddleware(TestCase): self.assertEquals(out[0].headers.get('Referer'), b'http://scrapytest.org') + def test_policy_default(self): + """ + Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + + with some additional filtering of s3:// + """ + # a) https:// --> https:// -- include Referer header + origin = Response('https://example.com/') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), + b'https://example.com/') + + # b.1) http:// --> http:// -- include Referer header + origin = Response('http://example.com/') + target = Request('http://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), + b'http://example.com/') + + # b.2) http:// --> https:// -- include Referer header + origin = Response('http://example.com/') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), + b'http://example.com/') + + # c) https:// --> http:// -- Referer header NOT sent + origin = Response('https://example.com/') + target = Request('http://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) + + def test_policy_default_no_credentials_leak(self): + origin = Response('http://user:password@example.com/') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), + b'http://example.com/') + + def test_policy_default_file_no_referrer_leak(self): + # file:// --> https:// -- Referrer NOT sent + origin = Response('file:///home/path/to/somefile.html') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) + + # file:// --> http:// -- Referrer NOT sent + origin = Response('file:///home/path/to/somefile.html') + target = Request('http://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) + + def test_policy_default_s3_no_referrer_leak(self): + # s3:// --> https:// -- Referrer NOT sent + origin = Response('s3://mybucket/path/to/data.csv') + target = Request('https://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) + + # s3:// --> http:// -- Referrer NOT sent + origin = Response('s3://mybucket/path/to/data.csv') + target = Request('http://scrapy.org/') + + out = list(self.mw.process_spider_output(origin, [target], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), None) From 7ec1b5f6c316f6c251821441f513e3f71cf63da0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 17:12:44 +0200 Subject: [PATCH 022/362] Add tests for the different referrer policies --- scrapy/spidermiddlewares/referer.py | 66 ++++--- tests/test_spidermiddleware_referer.py | 242 +++++++++++++++++++------ 2 files changed, 235 insertions(+), 73 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 21b340f22..1895aa95d 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -11,6 +11,15 @@ from scrapy.utils.python import to_native_str LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) +POLICY_NO_REFERRER = "no-referrer" +POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade" +POLICY_SAME_ORIGIN = "same-origin" +POLICY_ORIGIN = "origin" +POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin" +POLICY_UNSAFE_URL = "unsafe-url" +POLICY_SCRAPY_DEFAULT = "scrapy-default" + + class ReferrerPolicy(object): NOREFERRER_SCHEMES = LOCAL_SCHEMES @@ -38,17 +47,29 @@ class ReferrerPolicy(object): if parsed.scheme in self.NOREFERRER_SCHEMES: return None + + netloc = parsed.netloc + # strip username and password if present if parsed.username or parsed.password: - netloc = parsed.netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') - else: - netloc = parsed.netloc + netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') + + # strip standard protocol numbers + # Note: strictly speaking, standard port numbers should only be + # stripped when comparing origins + if parsed.port: + if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): + netloc = netloc.replace(':{p.port}'.format(p=parsed), '') + return urlunsplit(( parsed.scheme, netloc, - '' if origin_only else parsed.path, + '/' if origin_only else parsed.path, '' if origin_only else parsed.query, '')) + def origin(self, url): + return self.strip_url(url, origin_only=True) + class NoReferrerPolicy(ReferrerPolicy): """ @@ -58,7 +79,7 @@ class NoReferrerPolicy(ReferrerPolicy): is to be sent along with requests made from a particular request client to any origin. The header will be omitted entirely. """ - name = "no-referrer" + name = POLICY_NO_REFERRER def referrer(self, response, request): return None @@ -79,7 +100,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): This is a user agent's default behavior, if no policy is otherwise specified. """ - name = "no-referrer-when-downgrade" + name = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response, request): target_url = request.url @@ -108,12 +129,12 @@ class SameOriginPolicy(ReferrerPolicy): Cross-origin requests, on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. """ - name = "same-origin" + name = POLICY_SAME_ORIGIN def referrer(self, response, request): target_url = request.url referrer_source = response.url - if urlsplit(referrer_source).netloc == urlsplit(target_url).netloc: + if self.origin(referrer_source) == self.origin(target_url): return self.strip_url(referrer_source) else: return None @@ -128,10 +149,10 @@ class OriginPolicy(ReferrerPolicy): when making both same-origin requests and cross-origin requests from a particular request client. """ - name = "origin" + name = POLICY_ORIGIN def referrer(self, response, request): - return self.strip_url(referrer_source, origin_only=True) + return self.strip_url(response.url, origin_only=True) class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -145,17 +166,17 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): is sent as referrer information when making cross-origin requests from a particular request client. """ - name = "origin-when-cross-origin" + name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response, request): target_url = request.url referrer_source = response.url + source_origin = self.origin(referrer_source) + if source_origin == self.origin(target_url): + return self.strip_url(referrer_source, origin_only=False) + else: + return source_origin - # same origin --> send full referrer - # different origin --> send only "origin" as referrer - if urlsplit(referrer_source).netloc != urlsplit(target_url).netloc: - origin_only = True - return self.strip_url(referrer_source, origin_only=origin_only) class UnsafeUrlPolicy(ReferrerPolicy): @@ -171,7 +192,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): to insecure origins. Carefully consider the impact of setting such a policy for potentially sensitive documents. """ - name = "unsafe-url" + name = POLICY_UNSAFE_URL def referrer(self, response, request): referrer_source = response.url @@ -186,15 +207,17 @@ class LegacyPolicy(ReferrerPolicy): class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') + name = POLICY_SCRAPY_DEFAULT -_policies = {p.name: p for p in ( +_policy_classes = {p.name: p for p in ( NoReferrerPolicy, NoReferrerWhenDowngradePolicy, SameOriginPolicy, OriginPolicy, OriginWhenCrossOriginPolicy, UnsafeUrlPolicy, + DefaultReferrerPolicy, )} class RefererMiddleware(object): @@ -211,10 +234,11 @@ class RefererMiddleware(object): def policy(self, response, request): policy_name = request.meta.get('referrer_policy') if policy_name is None: - policy_name = to_native_str(response.headers.get('Referrer-Policy', '').decode('latin1')) + policy_name = to_native_str( + response.headers.get('Referrer-Policy', '').decode('latin1')) - policy_class = _policies.get(policy_name.lower(), self.default_policy) - return policy_class() + cls = _policy_classes.get(policy_name.lower(), self.default_policy) + return cls() def process_spider_output(self, response, result, spider): def _set_referer(r): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index f109bb248..8458fe90b 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -2,7 +2,10 @@ from unittest import TestCase from scrapy.http import Response, Request from scrapy.spiders import Spider -from scrapy.spidermiddlewares.referer import RefererMiddleware +from scrapy.spidermiddlewares.referer import RefererMiddleware, \ + POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ + POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ + POLICY_UNSAFE_URL class TestRefererMiddleware(TestCase): @@ -25,71 +28,206 @@ class TestRefererMiddleware(TestCase): with some additional filtering of s3:// """ - # a) https:// --> https:// -- include Referer header - origin = Response('https://example.com/') - target = Request('https://scrapy.org/') + for origin, target, referrer in [ + ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), + ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), + ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), + ('https://example.com/', 'http://scrapy.org/', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'https://example.com/') + # no credentials leak + ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), - # b.1) http:// --> http:// -- include Referer header - origin = Response('http://example.com/') - target = Request('http://scrapy.org/') + # no referrer leak for local schemes + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://example.com/') + # no referrer leak for s3 origins + ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), + ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), + ]: + response = Response(origin) + request = Request(target) - # b.2) http:// --> https:// -- include Referer header - origin = Response('http://example.com/') - target = Request('https://scrapy.org/') + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://example.com/') + def test_policy_no_referrer(self): - # c) https:// --> http:// -- Referer header NOT sent - origin = Response('https://example.com/') - target = Request('http://scrapy.org/') + for origin, target, referrer in [ + ('https://example.com/page.html', 'https://example.com/', None), + ('http://www.example.com/', 'https://scrapy.org/', None), + ('http://www.example.com/', 'http://scrapy.org/', None), + ('https://www.example.com/', 'http://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER}) - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) - def test_policy_default_no_credentials_leak(self): - origin = Response('http://user:password@example.com/') - target = Request('https://scrapy.org/') + def test_policy_no_referrer_when_downgrade(self): - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://example.com/') + for origin, target, referrer in [ + # TLS to TLS: send non-empty referrer + ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), + ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), + ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), - def test_policy_default_file_no_referrer_leak(self): - # file:// --> https:// -- Referrer NOT sent - origin = Response('file:///home/path/to/somefile.html') - target = Request('https://scrapy.org/') + # TLS to non-TLS: do not send referrer + ('https://example.com/page.html', 'http://not.example.com/', None), + ('https://example.com/page.html', 'http://scrapy.org/', None), + ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + # non-TLS to TLS or non-TLS: send referrer + ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), + ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), + ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), + ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), - # file:// --> http:// -- Referrer NOT sent - origin = Response('file:///home/path/to/somefile.html') - target = Request('http://scrapy.org/') + # test for user/password stripping + ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}) - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) - def test_policy_default_s3_no_referrer_leak(self): - # s3:// --> https:// -- Referrer NOT sent - origin = Response('s3://mybucket/path/to/data.csv') - target = Request('https://scrapy.org/') + def test_policy_same_origin(self): - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + for origin, target, referrer in [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), - # s3:// --> http:// -- Referrer NOT sent - origin = Response('s3://mybucket/path/to/data.csv') - target = Request('http://scrapy.org/') + # Different host: do NOT send referrer + ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), - out = list(self.mw.process_spider_output(origin, [target], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), None) + # Different port: do NOT send referrer + ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), + ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), + ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), + + # Different protocols: do NOT send refferer + ('https://example.com/page.html', 'http://example.com/not-page.html', None), + ('https://example.com/page.html', 'http://not.example.com/', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_SAME_ORIGIN}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) + + def test_policy_origin(self): + + for origin, target, referrer in [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_ORIGIN}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) + + def test_policy_origin_when_cross_origin(self): + + for origin, target, referrer in [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # Different protocols: send origin as referrer + ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), + ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + # TLS to non-TLS downgrade: send origin + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) + + def test_policy_unsafe_url(self): + + for origin, target, referrer in [ + # TLS to TLS: send referrer + ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), + ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), + ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), + ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), + + # TLS to non-TLS: send referrer (yes, it's unsafe) + ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), + ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), + ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), + + # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) + ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), + ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), + ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), + ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), + ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), + ]: + response = Response(origin) + request = Request(target, meta={'referrer_policy': POLICY_UNSAFE_URL}) + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), referrer) From 3af88a2877f947c74c6d9003da620629b27d5a17 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 18:27:18 +0200 Subject: [PATCH 023/362] Use urlparse_cached() on request and responses --- scrapy/spidermiddlewares/referer.py | 79 ++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 19 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 1895aa95d..cbbcd3e97 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,11 +2,12 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ -from six.moves.urllib.parse import urlsplit, urlunsplit +from six.moves.urllib.parse import urlparse, urlunparse, ParseResult from scrapy.http import Request from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_native_str +from scrapy.utils.httpobj import urlparse_cached LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) @@ -27,7 +28,7 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() - def strip_url(self, url, origin_only=False): + def strip_url_parsed(self, req_or_resp, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -41,9 +42,9 @@ class ReferrerPolicy(object): Set url's query to null. Return url. """ - if url is None or not url: + if req_or_resp.url is None or not req_or_resp.url: return None - parsed = urlsplit(url, allow_fragments=True) + parsed = urlparse_cached(req_or_resp) if parsed.scheme in self.NOREFERRER_SCHEMES: return None @@ -60,16 +61,61 @@ class ReferrerPolicy(object): if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): netloc = netloc.replace(':{p.port}'.format(p=parsed), '') - return urlunsplit(( + return ParseResult(parsed.scheme, + netloc, + '/' if origin_only else parsed.path, + '' if origin_only else parsed.params, + '' if origin_only else parsed.query, + '') + + def strip_url(self, url, origin_only=False): + """ + https://www.w3.org/TR/referrer-policy/#strip-url + + If url is null, return no referrer. + If url's scheme is a local scheme, then return no referrer. + Set url's username to the empty string. + Set url's password to null. + Set url's fragment to null. + If the origin-only flag is true, then: + Set url's path to null. + Set url's query to null. + Return url. + """ + if url is None or not url: + return None + parsed = urlparse(url, allow_fragments=True) + + if parsed.scheme in self.NOREFERRER_SCHEMES: + return None + + netloc = parsed.netloc + # strip username and password if present + if parsed.username or parsed.password: + netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') + + # strip standard protocol numbers + # Note: strictly speaking, standard port numbers should only be + # stripped when comparing origins + if parsed.port: + if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): + netloc = netloc.replace(':{p.port}'.format(p=parsed), '') + + return urlunparse(( parsed.scheme, netloc, '/' if origin_only else parsed.path, + '' if origin_only else parsed.params, '' if origin_only else parsed.query, '')) def origin(self, url): return self.strip_url(url, origin_only=True) + def origin_parsed(self, req_or_resp): + """Return (scheme, host, path) tuple for a request or response URL.""" + return tuple(self.strip_url_parsed(req_or_resp, origin_only=True)[:3]) + class NoReferrerPolicy(ReferrerPolicy): """ @@ -103,20 +149,17 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): name = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response, request): - target_url = request.url - - referrer_source = response.url - referrer_url = self.strip_url(referrer_source) - # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer: # # If environment is TLS-protected # and the origin of request's current URL is not an a priori authenticated URL, # then return no referrer. - if urlsplit(referrer_source).scheme in ('https', 'ftps') and \ - urlsplit(target_url).scheme in ('http',): + if urlparse_cached(response).scheme in ('https', 'ftps') and \ + urlparse_cached(request).scheme in ('http',): return None - return referrer_url + stripped = self.strip_url_parsed(response) + if stripped is not None: + return urlunparse(stripped) class SameOriginPolicy(ReferrerPolicy): @@ -132,12 +175,10 @@ class SameOriginPolicy(ReferrerPolicy): name = POLICY_SAME_ORIGIN def referrer(self, response, request): - target_url = request.url - referrer_source = response.url - if self.origin(referrer_source) == self.origin(target_url): - return self.strip_url(referrer_source) - else: - return None + if self.origin_parsed(response) == self.origin_parsed(request): + stripped = self.strip_url_parsed(response) + if stripped is not None: + return urlunparse(stripped) class OriginPolicy(ReferrerPolicy): From f2ee6be3bb311bb8f89ccd929bc0ebd8412d0183 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 18:33:31 +0200 Subject: [PATCH 024/362] Use urlparse_cached() for OriginPolicy --- scrapy/spidermiddlewares/referer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index cbbcd3e97..bf2a3c037 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -193,7 +193,9 @@ class OriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN def referrer(self, response, request): - return self.strip_url(response.url, origin_only=True) + stripped = self.strip_url_parsed(response, origin_only=True) + if stripped is not None: + return urlunparse(stripped) class OriginWhenCrossOriginPolicy(ReferrerPolicy): From 59cb884ace1cb1c3339f1d0f05895501dd4d0447 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 18:44:46 +0200 Subject: [PATCH 025/362] Use urlparse_cached() for OriginWhenCrossOriginPolicy --- scrapy/spidermiddlewares/referer.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index bf2a3c037..60f21ae84 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -193,9 +193,9 @@ class OriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN def referrer(self, response, request): - stripped = self.strip_url_parsed(response, origin_only=True) - if stripped is not None: - return urlunparse(stripped) + origin = self.strip_url_parsed(response, origin_only=True) + if origin is not None: + return urlunparse(origin) class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -212,14 +212,13 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response, request): - target_url = request.url - referrer_source = response.url - source_origin = self.origin(referrer_source) - if source_origin == self.origin(target_url): - return self.strip_url(referrer_source, origin_only=False) + origin = self.origin_parsed(response) + if origin == self.origin_parsed(request): + stripped = self.strip_url_parsed(response) + if stripped is not None: + return urlunparse(stripped) else: - return source_origin - + return urlunparse(origin + ('', '', '')) class UnsafeUrlPolicy(ReferrerPolicy): From f6a800fde67447d94480d4271de62f5e9a60bfb0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 18:55:35 +0200 Subject: [PATCH 026/362] Remove all non-cached urlparsing references --- scrapy/spidermiddlewares/referer.py | 71 ++++++----------------------- 1 file changed, 14 insertions(+), 57 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 60f21ae84..46351576f 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,7 +2,7 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ -from six.moves.urllib.parse import urlparse, urlunparse, ParseResult +from six.moves.urllib.parse import ParseResult, urlunparse from scrapy.http import Request from scrapy.exceptions import NotConfigured @@ -28,7 +28,7 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() - def strip_url_parsed(self, req_or_resp, origin_only=False): + def strip_url(self, req_or_resp, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -68,53 +68,9 @@ class ReferrerPolicy(object): '' if origin_only else parsed.query, '') - def strip_url(self, url, origin_only=False): - """ - https://www.w3.org/TR/referrer-policy/#strip-url - - If url is null, return no referrer. - If url's scheme is a local scheme, then return no referrer. - Set url's username to the empty string. - Set url's password to null. - Set url's fragment to null. - If the origin-only flag is true, then: - Set url's path to null. - Set url's query to null. - Return url. - """ - if url is None or not url: - return None - parsed = urlparse(url, allow_fragments=True) - - if parsed.scheme in self.NOREFERRER_SCHEMES: - return None - - netloc = parsed.netloc - # strip username and password if present - if parsed.username or parsed.password: - netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') - - # strip standard protocol numbers - # Note: strictly speaking, standard port numbers should only be - # stripped when comparing origins - if parsed.port: - if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): - netloc = netloc.replace(':{p.port}'.format(p=parsed), '') - - return urlunparse(( - parsed.scheme, - netloc, - '/' if origin_only else parsed.path, - '' if origin_only else parsed.params, - '' if origin_only else parsed.query, - '')) - - def origin(self, url): - return self.strip_url(url, origin_only=True) - - def origin_parsed(self, req_or_resp): + def origin(self, req_or_resp): """Return (scheme, host, path) tuple for a request or response URL.""" - return tuple(self.strip_url_parsed(req_or_resp, origin_only=True)[:3]) + return tuple(self.strip_url(req_or_resp, origin_only=True)[:3]) class NoReferrerPolicy(ReferrerPolicy): @@ -157,7 +113,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): if urlparse_cached(response).scheme in ('https', 'ftps') and \ urlparse_cached(request).scheme in ('http',): return None - stripped = self.strip_url_parsed(response) + stripped = self.strip_url(response) if stripped is not None: return urlunparse(stripped) @@ -175,8 +131,8 @@ class SameOriginPolicy(ReferrerPolicy): name = POLICY_SAME_ORIGIN def referrer(self, response, request): - if self.origin_parsed(response) == self.origin_parsed(request): - stripped = self.strip_url_parsed(response) + if self.origin(response) == self.origin(request): + stripped = self.strip_url(response) if stripped is not None: return urlunparse(stripped) @@ -193,7 +149,7 @@ class OriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN def referrer(self, response, request): - origin = self.strip_url_parsed(response, origin_only=True) + origin = self.strip_url(response, origin_only=True) if origin is not None: return urlunparse(origin) @@ -212,9 +168,9 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN def referrer(self, response, request): - origin = self.origin_parsed(response) - if origin == self.origin_parsed(request): - stripped = self.strip_url_parsed(response) + origin = self.origin(response) + if origin == self.origin(request): + stripped = self.strip_url(response) if stripped is not None: return urlunparse(stripped) else: @@ -237,8 +193,9 @@ class UnsafeUrlPolicy(ReferrerPolicy): name = POLICY_UNSAFE_URL def referrer(self, response, request): - referrer_source = response.url - return self.strip_url(referrer_source) + stripped = self.strip_url(response) + if stripped is not None: + return urlunparse(stripped) class LegacyPolicy(ReferrerPolicy): From f6205778f31dae3ac0f60528a7aa4ef3ef6a181a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 5 Oct 2016 19:06:26 +0200 Subject: [PATCH 027/362] Refactor ReferrerPolicy methods --- scrapy/spidermiddlewares/referer.py | 30 ++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 46351576f..44a599433 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -28,6 +28,16 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() + def stripped_referrer(self, req_or_resp): + stripped = self.strip_url(req_or_resp) + if stripped is not None: + return urlunparse(stripped) + + def origin_referrer(self, req_or_resp): + stripped = self.strip_url(req_or_resp, origin_only=True) + if stripped is not None: + return urlunparse(stripped) + def strip_url(self, req_or_resp, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -113,9 +123,7 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): if urlparse_cached(response).scheme in ('https', 'ftps') and \ urlparse_cached(request).scheme in ('http',): return None - stripped = self.strip_url(response) - if stripped is not None: - return urlunparse(stripped) + return self.stripped_referrer(response) class SameOriginPolicy(ReferrerPolicy): @@ -132,9 +140,7 @@ class SameOriginPolicy(ReferrerPolicy): def referrer(self, response, request): if self.origin(response) == self.origin(request): - stripped = self.strip_url(response) - if stripped is not None: - return urlunparse(stripped) + return self.stripped_referrer(response) class OriginPolicy(ReferrerPolicy): @@ -149,9 +155,7 @@ class OriginPolicy(ReferrerPolicy): name = POLICY_ORIGIN def referrer(self, response, request): - origin = self.strip_url(response, origin_only=True) - if origin is not None: - return urlunparse(origin) + return self.origin_referrer(response) class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -170,9 +174,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): def referrer(self, response, request): origin = self.origin(response) if origin == self.origin(request): - stripped = self.strip_url(response) - if stripped is not None: - return urlunparse(stripped) + return self.stripped_referrer(response) else: return urlunparse(origin + ('', '', '')) @@ -193,9 +195,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): name = POLICY_UNSAFE_URL def referrer(self, response, request): - stripped = self.strip_url(response) - if stripped is not None: - return urlunparse(stripped) + return self.stripped_referrer(response) class LegacyPolicy(ReferrerPolicy): From 842ce131aa666dce0db61aeee701c420050a5a9c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 11 Oct 2016 18:27:31 +0200 Subject: [PATCH 028/362] Make default referrer policy customizable via settings --- scrapy/settings/default_settings.py | 1 + scrapy/spidermiddlewares/referer.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a5931a3d5..15a134dd3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -234,6 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True +REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 44a599433..88041d8f0 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -8,6 +8,7 @@ from scrapy.http import Request from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_native_str from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import load_object LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) @@ -221,16 +222,27 @@ _policy_classes = {p.name: p for p in ( class RefererMiddleware(object): - def __init__(self, policy_class=DefaultReferrerPolicy): - self.default_policy = policy_class + def __init__(self, settings={}): + policy = settings.get('REFERER_POLICY') + if policy is not None: + try: + self.default_policy = load_object(policy) + except ValueError: + try: + self.default_policy = _policy_classes[policy] + except: + raise NotConfigured("Unknown referrer policy name %r" % policy) + else: + self.default_policy = DefaultReferrerPolicy @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured - return cls() + return cls(crawler.settings) def policy(self, response, request): + # policy set in request's meta dict takes precedence over default policy policy_name = request.meta.get('referrer_policy') if policy_name is None: policy_name = to_native_str( From e72b6e33611f61dbb358f1ff97c5c195a7db696f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 11 Oct 2016 18:27:56 +0200 Subject: [PATCH 029/362] Add tests for referrer policy via settings and via Request meta --- tests/test_spidermiddleware_referer.py | 501 ++++++++++++++----------- 1 file changed, 292 insertions(+), 209 deletions(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 8458fe90b..b724d7999 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,233 +1,316 @@ from unittest import TestCase +from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request +from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.spidermiddlewares.referer import RefererMiddleware, \ POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ - POLICY_UNSAFE_URL + POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \ + DefaultReferrerPolicy, \ + NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ + OriginWhenCrossOriginPolicy, OriginPolicy, \ + SameOriginPolicy, UnsafeUrlPolicy class TestRefererMiddleware(TestCase): + req_meta = {} + resp_headers = {} + settings = {} + scenarii = [ + ('http://scrapytest.org', 'http://scrapytest.org/', b'http://scrapytest.org'), + ] + def setUp(self): self.spider = Spider('foo') - self.mw = RefererMiddleware() + settings = Settings(self.settings) + self.mw = RefererMiddleware(settings) - def test_process_spider_output(self): - res = Response('http://scrapytest.org') - reqs = [Request('http://scrapytest.org/')] + def get_request(self, target): + return Request(target, meta=self.req_meta) - out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out[0].headers.get('Referer'), - b'http://scrapytest.org') + def get_response(self, origin): + return Response(origin, headers=self.resp_headers) - def test_policy_default(self): - """ - Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + def test(self): - with some additional filtering of s3:// - """ - for origin, target, referrer in [ - ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), - ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), - ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), - ('https://example.com/', 'http://scrapy.org/', None), - - # no credentials leak - ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), - - # no referrer leak for local schemes - ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), - ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), - - # no referrer leak for s3 origins - ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), - ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), - ]: - response = Response(origin) - request = Request(target) + for origin, target, referrer in self.scenarii: + response = self.get_response(origin) + request = self.get_request(target) out = list(self.mw.process_spider_output(response, [request], self.spider)) self.assertEquals(out[0].headers.get('Referer'), referrer) - def test_policy_no_referrer(self): - for origin, target, referrer in [ - ('https://example.com/page.html', 'https://example.com/', None), - ('http://www.example.com/', 'https://scrapy.org/', None), - ('http://www.example.com/', 'http://scrapy.org/', None), - ('https://www.example.com/', 'http://scrapy.org/', None), - ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), +class MixinDefault(object): + """ + Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + + with some additional filtering of s3:// + """ + scenarii = [ + ('https://example.com/', 'https://scrapy.org/', b'https://example.com/'), + ('http://example.com/', 'http://scrapy.org/', b'http://example.com/'), + ('http://example.com/', 'https://scrapy.org/', b'http://example.com/'), + ('https://example.com/', 'http://scrapy.org/', None), + + # no credentials leak + ('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'), + + # no referrer leak for local schemes + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + + # no referrer leak for s3 origins + ('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None), + ('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None), + ] + + +class MixinNoReferrer(object): + scenarii = [ + ('https://example.com/page.html', 'https://example.com/', None), + ('http://www.example.com/', 'https://scrapy.org/', None), + ('http://www.example.com/', 'http://scrapy.org/', None), + ('https://www.example.com/', 'http://scrapy.org/', None), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', None), + ] + + +class MixinNoReferrerWhenDowngrade(object): + scenarii = [ + # TLS to TLS: send non-empty referrer + ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), + ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), + ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), + ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), + + # TLS to non-TLS: do not send referrer + ('https://example.com/page.html', 'http://not.example.com/', None), + ('https://example.com/page.html', 'http://scrapy.org/', None), + ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), + + # non-TLS to TLS or non-TLS: send referrer + ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), + ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), + ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), + ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), + ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), + ] + + +class MixinSameOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: do NOT send referrer + ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), + ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), + + # Different port: do NOT send referrer + ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), + ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), + ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), + + # Different protocols: do NOT send refferer + ('https://example.com/page.html', 'http://example.com/not-page.html', None), + ('https://example.com/page.html', 'http://not.example.com/', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), + ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), + ] + + +class MixinOrigin(object): + scenarii = [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), + ] + + +class MixinOriginWhenCrossOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # Different protocols: send origin as referrer + ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), + ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + # TLS to non-TLS downgrade: send origin + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), + ] + + +class MixinUnsafeUrl(object): + scenarii = [ + # TLS to TLS: send referrer + ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), + ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), + ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), + ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), + ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), + + # TLS to non-TLS: send referrer (yes, it's unsafe) + ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), + ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), + ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), + + # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) + ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), + ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), + ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), + ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), + ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), + ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), + + # test for user/password stripping + ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), + ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), + ] + + +class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware): + pass + + +# --- Tests using settings to set policy using class path +class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} + + +class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + + +class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + + +class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} + + +class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + + +class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + + +# --- Tests using Request meta dict to set policy +class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} + + +class TestRefererMiddlewareNoReferrer(MixinNoReferrer, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_NO_REFERRER} + + +class TestRefererMiddlewareNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE} + + +class TestRefererMiddlewareSameOrigin(MixinSameOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_SAME_ORIGIN} + + +class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_ORIGIN} + + +class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} + + +class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + + +class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + +class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + req_meta = {'referrer_policy': POLICY_NO_REFERRER} + + +class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + req_meta = {'referrer_policy': POLICY_UNSAFE_URL} + + +class TestRefererMiddlewareSettingsPolicyByName(TestCase): + + def test_valid_name(self): + for s, p in [ + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER}) + settings = Settings({'REFERER_POLICY': s}) + mw = RefererMiddleware(settings) + self.assertEquals(mw.default_policy, p) - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_no_referrer_when_downgrade(self): - - for origin, target, referrer in [ - # TLS to TLS: send non-empty referrer - ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), - ('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), - ('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'), - ('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'), - ('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'), - - # TLS to non-TLS: do not send referrer - ('https://example.com/page.html', 'http://not.example.com/', None), - ('https://example.com/page.html', 'http://scrapy.org/', None), - ('ftps://example.com/urls.zip', 'http://scrapy.org/', None), - - # non-TLS to TLS or non-TLS: send referrer - ('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), - ('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'), - ('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'), - ('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'), - ('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'), - ('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'), - ('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'), - - # test for user/password stripping - ('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_same_origin(self): - - for origin, target, referrer in [ - # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), - - # Different host: do NOT send referrer - ('https://example.com/page.html', 'https://not.example.com/otherpage.html', None), - ('http://example.com/page.html', 'http://not.example.com/otherpage.html', None), - ('http://example.com/page.html', 'http://www.example.com/otherpage.html', None), - - # Different port: do NOT send referrer - ('https://example.com:444/page.html', 'https://example.com/not-page.html', None), - ('http://example.com:81/page.html', 'http://example.com/not-page.html', None), - ('http://example.com/page.html', 'http://example.com:81/not-page.html', None), - - # Different protocols: do NOT send refferer - ('https://example.com/page.html', 'http://example.com/not-page.html', None), - ('https://example.com/page.html', 'http://not.example.com/', None), - ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), - ('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None), - ('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None), - - # test for user/password stripping - ('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_SAME_ORIGIN}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_origin(self): - - for origin, target, referrer in [ - # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), - ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), - ('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'), - ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), - - # test for user/password stripping - ('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_ORIGIN}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_origin_when_cross_origin(self): - - for origin, target, referrer in [ - # Same origin (protocol, host, port): send referrer - ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), - ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), - ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), - ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), - - # Different host: send origin as referrer - ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), - ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), - ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), - # exact match required - ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), - - # Different port: send origin as referrer - ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), - ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), - - # Different protocols: send origin as referrer - ('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'), - ('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'), - ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), - ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), - ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), - - # test for user/password stripping - ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), - # TLS to non-TLS downgrade: send origin - ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) - - def test_policy_unsafe_url(self): - - for origin, target, referrer in [ - # TLS to TLS: send referrer - ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), - ('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'), - ('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), - ('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'), - ('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'), - ('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'), - - # TLS to non-TLS: send referrer (yes, it's unsafe) - ('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'), - ('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'), - ('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'), - - # non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe) - ('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'), - ('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'), - ('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'), - ('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'), - ('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'), - ('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'), - ('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'), - ('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'), - - # test for user/password stripping - ('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'), - ('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'), - ]: - response = Response(origin) - request = Request(target, meta={'referrer_policy': POLICY_UNSAFE_URL}) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) + def test_invalid_name(self): + settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) + with self.assertRaises(NotConfigured): + mw = RefererMiddleware(settings) From 0344f57fefc0877bf9048a084002fc719335e31c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 11 Oct 2016 19:53:15 +0200 Subject: [PATCH 030/362] Support case-insensitive policy names in settings --- scrapy/spidermiddlewares/referer.py | 2 +- tests/test_spidermiddleware_referer.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 88041d8f0..deda7b284 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -229,7 +229,7 @@ class RefererMiddleware(object): self.default_policy = load_object(policy) except ValueError: try: - self.default_policy = _policy_classes[policy] + self.default_policy = _policy_classes[policy.lower()] except: raise NotConfigured("Unknown referrer policy name %r" % policy) else: diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index b724d7999..b1ab366a7 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -310,6 +310,20 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): mw = RefererMiddleware(settings) self.assertEquals(mw.default_policy, p) + def test_valid_name_casevariants(self): + for s, p in [ + (POLICY_SCRAPY_DEFAULT, DefaultReferrerPolicy), + (POLICY_NO_REFERRER, NoReferrerPolicy), + (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), + (POLICY_SAME_ORIGIN, SameOriginPolicy), + (POLICY_ORIGIN, OriginPolicy), + (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_UNSAFE_URL, UnsafeUrlPolicy), + ]: + settings = Settings({'REFERER_POLICY': s.upper()}) + mw = RefererMiddleware(settings) + self.assertEquals(mw.default_policy, p) + def test_invalid_name(self): settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) with self.assertRaises(NotConfigured): From ec8b4c1a9bfd004960d730b41078b9f0633627d8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 11 Oct 2016 20:00:34 +0200 Subject: [PATCH 031/362] Change __init__ default "settings" arg handling --- scrapy/spidermiddlewares/referer.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index deda7b284..2a3790bde 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -222,18 +222,18 @@ _policy_classes = {p.name: p for p in ( class RefererMiddleware(object): - def __init__(self, settings={}): - policy = settings.get('REFERER_POLICY') - if policy is not None: - try: - self.default_policy = load_object(policy) - except ValueError: + def __init__(self, settings=None): + self.default_policy = DefaultReferrerPolicy + if settings is not None: + policy = settings.get('REFERER_POLICY') + if policy is not None: try: - self.default_policy = _policy_classes[policy.lower()] - except: - raise NotConfigured("Unknown referrer policy name %r" % policy) - else: - self.default_policy = DefaultReferrerPolicy + self.default_policy = load_object(policy) + except ValueError: + try: + self.default_policy = _policy_classes[policy.lower()] + except: + raise NotConfigured("Unknown referrer policy name %r" % policy) @classmethod def from_crawler(cls, crawler): From e50e670eff2fe1b109f6b5dd026c4c706a93585c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 12 Oct 2016 16:16:53 +0200 Subject: [PATCH 032/362] Add test for custom referrer policy via settings --- scrapy/spidermiddlewares/referer.py | 4 ++++ tests/test_spidermiddleware_referer.py | 30 ++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 2a3790bde..0bba63cb7 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -227,9 +227,12 @@ class RefererMiddleware(object): if settings is not None: policy = settings.get('REFERER_POLICY') if policy is not None: + # expect a string for the path to the policy class try: self.default_policy = load_object(policy) except ValueError: + # otherwise try to interpret the string as standard + # https://www.w3.org/TR/referrer-policy/#referrer-policies try: self.default_policy = _policy_classes[policy.lower()] except: @@ -239,6 +242,7 @@ class RefererMiddleware(object): def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured + return cls(crawler.settings) def policy(self, response, request): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index b1ab366a7..cfc4b5296 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -11,7 +11,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \ DefaultReferrerPolicy, \ NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ OriginWhenCrossOriginPolicy, OriginPolicy, \ - SameOriginPolicy, UnsafeUrlPolicy + SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy class TestRefererMiddleware(TestCase): @@ -249,6 +249,33 @@ class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddlewa settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} +class CustomPythonOrgPolicy(ReferrerPolicy): + """ + A dummy policy that returns referrer as http(s)://python.org + depending on the scheme of the target URL. + """ + def referrer(self, response, request): + from scrapy.utils.httpobj import urlparse_cached + + scheme = urlparse_cached(request).scheme + if scheme == 'https': + return b'https://python.org/' + elif scheme == 'http': + return b'http://python.org/' + + +class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware): + settings = {'REFERER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} + scenarii = [ + ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), + ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), + ('http://example.com/', 'https://scrapy.org/', b'https://python.org/'), + ('https://example.com/', 'http://scrapy.org/', b'http://python.org/'), + ('file:///home/path/to/somefile.html', 'https://scrapy.org/', b'https://python.org/'), + ('file:///home/path/to/somefile.html', 'http://scrapy.org/', b'http://python.org/'), + + ] + # --- Tests using Request meta dict to set policy class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} @@ -278,7 +305,6 @@ class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_UNSAFE_URL} - class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} From d3d4d66ce8e5d01aa1fd6013a6d63337bb931460 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 12 Oct 2016 16:30:25 +0200 Subject: [PATCH 033/362] Add tests for referrer-policy set in response HTTP headers --- tests/test_spidermiddleware_referer.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index cfc4b5296..9555817d9 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -354,3 +354,16 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) with self.assertRaises(NotConfigured): mw = RefererMiddleware(settings) + + +class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} + +class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} + +class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} From 285d5bc03a7ebe8eaa558a5c24ff0693353c3e87 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 12 Oct 2016 17:34:12 +0200 Subject: [PATCH 034/362] Patch "Referer" header on HTTP redirects if necessary --- scrapy/spidermiddlewares/referer.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 0bba63cb7..01f1fdf85 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -4,8 +4,9 @@ originated it. """ from six.moves.urllib.parse import ParseResult, urlunparse -from scrapy.http import Request +from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured +from scrapy import signals from scrapy.utils.python import to_native_str from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object @@ -242,8 +243,9 @@ class RefererMiddleware(object): def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured - - return cls(crawler.settings) + mw = cls(crawler.settings) + crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled) + return mw def policy(self, response, request): # policy set in request's meta dict takes precedence over default policy @@ -264,3 +266,18 @@ class RefererMiddleware(object): return r return (_set_referer(r) for r in result or ()) + def request_scheduled(self, request, spider): + # check redirected request to patch "Referer" header if necessary + redirected_urls = request.meta.get('redirect_urls', []) + if redirected_urls: + request_referrer = request.headers.get('Referer') + # we don't patch the referrer value if there is none + if request_referrer is not None: + faked_response = Response(redirected_urls[0]) + policy_referrer = self.policy(faked_response, + request).referrer(faked_response, request) + if policy_referrer != request_referrer: + if policy_referrer is None: + request.headers.pop('Referer') + else: + request.headers['Referer'] = policy_referrer From c9c59db489575131d573e130015fd1fb6b133882 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 12 Oct 2016 18:29:47 +0200 Subject: [PATCH 035/362] Update documentation about REFERER_POLICY setting --- docs/topics/spider-middleware.rst | 38 ++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8360827e8..a9d3d4568 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -95,7 +95,7 @@ following methods: it has processed the response. :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` + :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` objects. :param response: the response which generated this output from the @@ -328,6 +328,42 @@ Default: ``True`` Whether to enable referer middleware. +.. setting:: REFERER_POLICY + +REFERER_POLICY +^^^^^^^^^^^^^^ + +.. versionadded:: 1.3 + +Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` + +`Referrer Policy`_ to apply when populating Request "Referer" header. + +This setting accepts: + +- a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass, + either a custom one or one of the built-in ones + (see ``scrapy.spidermiddlewares.referer``), +- or one of the standard W3C-defined string values, i.e. ``"no-referrer"``, + ``"no-referrer-when-downgrade"``, ``"same-origin"``, ``"origin"``, + ``"origin-when-cross-origin"`` or ``"unsafe-url"``. + (It can also be the non-standard value ``"scrapy-default"`` to use + Scrapy's default referrer policy.) + +Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, +with the addition that "Referrer" is not sent if the parent request was +using ``file://`` or ``s3://`` scheme. + +.. warning:: + By default, Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_, + will send a non-empty "Referer" header from any ``https://`` to any ``https://`` URL, + even if the domain is different. + ``same-origin`` may be a better choice if you want to remove referrer + information for cross-domain requests. + +.. _Referrer Policy: https://www.w3.org/TR/referrer-policy +.. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade + UrlLengthMiddleware ------------------- From 5dd7311cd48e676147138746229d7ab2b429b8a9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 19 Oct 2016 14:45:33 +0200 Subject: [PATCH 036/362] Move URL credentials stripping to a helper function --- scrapy/spidermiddlewares/referer.py | 43 +++----------- scrapy/utils/url.py | 34 ++++++++++- tests/test_utils_url.py | 92 ++++++++++++++++++++++++++++- 3 files changed, 133 insertions(+), 36 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 01f1fdf85..e40e798b8 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,14 +2,13 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ -from six.moves.urllib.parse import ParseResult, urlunparse - from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals from scrapy.utils.python import to_native_str from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object +from scrapy.utils.url import strip_url_credentials LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) @@ -31,14 +30,10 @@ class ReferrerPolicy(object): raise NotImplementedError() def stripped_referrer(self, req_or_resp): - stripped = self.strip_url(req_or_resp) - if stripped is not None: - return urlunparse(stripped) + return self.strip_url(req_or_resp) def origin_referrer(self, req_or_resp): - stripped = self.strip_url(req_or_resp, origin_only=True) - if stripped is not None: - return urlunparse(stripped) + return self.strip_url(req_or_resp, origin_only=True) def strip_url(self, req_or_resp, origin_only=False): """ @@ -56,33 +51,13 @@ class ReferrerPolicy(object): """ if req_or_resp.url is None or not req_or_resp.url: return None - parsed = urlparse_cached(req_or_resp) - - if parsed.scheme in self.NOREFERRER_SCHEMES: - return None - - netloc = parsed.netloc - # strip username and password if present - if parsed.username or parsed.password: - netloc = netloc.replace('{p.username}:{p.password}@'.format(p=parsed), '') - - # strip standard protocol numbers - # Note: strictly speaking, standard port numbers should only be - # stripped when comparing origins - if parsed.port: - if (parsed.scheme, parsed.port) in (('http', 80), ('https', 443)): - netloc = netloc.replace(':{p.port}'.format(p=parsed), '') - - return ParseResult(parsed.scheme, - netloc, - '/' if origin_only else parsed.path, - '' if origin_only else parsed.params, - '' if origin_only else parsed.query, - '') + parsed_url = urlparse_cached(req_or_resp) + if parsed_url.scheme not in self.NOREFERRER_SCHEMES: + return strip_url_credentials(parsed_url, origin_only=origin_only) def origin(self, req_or_resp): - """Return (scheme, host, path) tuple for a request or response URL.""" - return tuple(self.strip_url(req_or_resp, origin_only=True)[:3]) + """Return serialized origin (scheme, host, path) for a request or response URL.""" + return self.strip_url(req_or_resp, origin_only=True) class NoReferrerPolicy(ReferrerPolicy): @@ -178,7 +153,7 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): if origin == self.origin(request): return self.stripped_referrer(response) else: - return urlunparse(origin + ('', '', '')) + return origin class UnsafeUrlPolicy(ReferrerPolicy): diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index dc1cce4ac..f3ccfb0e8 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead. """ import posixpath import re -from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse) +from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse) # scrapy.utils.url was moved to w3lib.url and import * ensures this # move doesn't break old code @@ -103,3 +103,35 @@ def guess_scheme(url): return any_to_uri(url) else: return add_http_if_no_scheme(url) + + +def strip_url_credentials(url, origin_only=False, keep_fragments=False): + + if url is None: + return None + + if not isinstance(url, ParseResult): + parsed_url = urlparse(url) + else: + parsed_url = url + + netloc = parsed_url.netloc + # strip username and password if present + if parsed_url.username or parsed_url.password: + netloc = netloc.split('@')[-1] + + # strip standard protocol numbers + # Note: strictly speaking, standard port numbers should only be + # stripped when comparing origins + if parsed_url.port: + if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443)): + netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '') + + return urlunparse(( + parsed_url.scheme, + netloc, + '/' if origin_only else parsed_url.path, + '' if origin_only else parsed_url.params, + '' if origin_only else parsed_url.query, + '' if not keep_fragments else parsed_url.fragment + )) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index f46d1d927..f1a5c3196 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -6,7 +6,8 @@ from six.moves.urllib.parse import urlparse from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, - add_http_if_no_scheme, guess_scheme, parse_url) + add_http_if_no_scheme, guess_scheme, + parse_url, strip_url_credentials) __doctests__ = ['scrapy.utils.url'] @@ -241,5 +242,94 @@ for k, args in enumerate ([ setattr (GuessSchemeTest, t_method.__name__, t_method) +class StripUrlCredentials(unittest.TestCase): + + def test_noop(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com/index.html'), + 'http://www.example.com/index.html') + + def test_noop_query_string(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com/index.html?somekey=somevalue'), + 'http://www.example.com/index.html?somekey=somevalue') + + def test_fragments(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com/index.html?somekey=somevalue#section', keep_fragments=True), + 'http://www.example.com/index.html?somekey=somevalue#section') + + def test_noop_trailing_path(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com/'), + 'http://www.example.com/') + + def test_noop_trailing_path2(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com'), + 'http://www.example.com') + + def test_trailing_path_origin(self): + self.assertEqual(strip_url_credentials( + 'http://www.example.com', origin_only=True), + 'http://www.example.com/') + + def test_username(self): + # username is stripped (and fragment too) + self.assertEqual(strip_url_credentials( + 'http://username@www.example.com/index.html?somekey=somevalue#section'), + 'http://www.example.com/index.html?somekey=somevalue') + + def test_username_empty_pass(self): + # same as above + self.assertEqual(strip_url_credentials( + 'https://username:@www.example.com/index.html?somekey=somevalue#section'), + 'https://www.example.com/index.html?somekey=somevalue') + + def test_username_password(self): + self.assertEqual(strip_url_credentials( + 'ftp://username:password@www.example.com/index.html?somekey=somevalue#section'), + 'ftp://www.example.com/index.html?somekey=somevalue') + + def test_default_http_port(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com:80/index.html'), + 'http://www.example.com/index.html') + + def test_non_default_http_port(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com:8080/index.html'), + 'http://www.example.com:8080/index.html') + + def test_default_https_port(self): + self.assertEqual(strip_url_credentials( + 'https://username:password@www.example.com:443/index.html'), + 'https://www.example.com/index.html') + + def test_non_default_https_port(self): + self.assertEqual(strip_url_credentials( + 'https://username:password@www.example.com:442/index.html'), + 'https://www.example.com:442/index.html') + + def test_origin_only(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com/index.html', origin_only=True), + 'http://www.example.com/') + + def test_default_http_port_origin_only(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com:80/index.html', origin_only=True), + 'http://www.example.com/') + + def test_non_default_http_port_origin_only(self): + self.assertEqual(strip_url_credentials( + 'http://username:password@www.example.com:8008/index.html', origin_only=True), + 'http://www.example.com:8008/') + + def test_default_https_port_origin_only(self): + self.assertEqual(strip_url_credentials( + 'https://username:password@www.example.com:443/index.html', origin_only=True), + 'https://www.example.com/') + if __name__ == "__main__": unittest.main() From 8864d0e8c19d0ddb52175be1952a4013b13cd86f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 24 Oct 2016 18:21:49 +0200 Subject: [PATCH 037/362] Rename helper function to strip_url() + add more tests --- scrapy/spidermiddlewares/referer.py | 8 +- scrapy/utils/url.py | 30 +++-- tests/test_utils_url.py | 190 +++++++++++++++++++--------- 3 files changed, 150 insertions(+), 78 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index e40e798b8..24e5eac40 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -8,7 +8,7 @@ from scrapy import signals from scrapy.utils.python import to_native_str from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object -from scrapy.utils.url import strip_url_credentials +from scrapy.utils.url import strip_url LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',) @@ -53,7 +53,11 @@ class ReferrerPolicy(object): return None parsed_url = urlparse_cached(req_or_resp) if parsed_url.scheme not in self.NOREFERRER_SCHEMES: - return strip_url_credentials(parsed_url, origin_only=origin_only) + return strip_url(parsed_url, + strip_credentials=True, + strip_fragment=True, + strip_default_port=True, + origin_only=origin_only) def origin(self, req_or_resp): """Return serialized origin (scheme, host, path) for a request or response URL.""" diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index f3ccfb0e8..9864f353d 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -105,33 +105,37 @@ def guess_scheme(url): return add_http_if_no_scheme(url) -def strip_url_credentials(url, origin_only=False, keep_fragments=False): +def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True): + + """Strip URL string from some of its components: + + - `strip_credentials` removes "user:password@" + - `strip_default_port` removes ":80" (resp. ":443", ":21") + from http:// (resp. https://, ftp://) URLs + - `origin_only` replaces path component with "/", also dropping + query and fragment components ; it also strips credentials + - `strip_fragment` drops any #fragment component + """ if url is None: return None - if not isinstance(url, ParseResult): parsed_url = urlparse(url) else: parsed_url = url - netloc = parsed_url.netloc - # strip username and password if present - if parsed_url.username or parsed_url.password: + if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password): netloc = netloc.split('@')[-1] - - # strip standard protocol numbers - # Note: strictly speaking, standard port numbers should only be - # stripped when comparing origins - if parsed_url.port: - if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443)): + if strip_default_port and parsed_url.port: + if (parsed_url.scheme, parsed_url.port) in (('http', 80), + ('https', 443), + ('ftp', 21)): netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '') - return urlunparse(( parsed_url.scheme, netloc, '/' if origin_only else parsed_url.path, '' if origin_only else parsed_url.params, '' if origin_only else parsed_url.query, - '' if not keep_fragments else parsed_url.fragment + '' if strip_fragment else parsed_url.fragment )) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index f1a5c3196..1f9845d82 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -7,7 +7,7 @@ from six.moves.urllib.parse import urlparse from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, add_http_if_no_scheme, guess_scheme, - parse_url, strip_url_credentials) + parse_url, strip_url) __doctests__ = ['scrapy.utils.url'] @@ -242,94 +242,158 @@ for k, args in enumerate ([ setattr (GuessSchemeTest, t_method.__name__, t_method) -class StripUrlCredentials(unittest.TestCase): +class StripUrl(unittest.TestCase): def test_noop(self): - self.assertEqual(strip_url_credentials( + self.assertEqual(strip_url( 'http://www.example.com/index.html'), 'http://www.example.com/index.html') def test_noop_query_string(self): - self.assertEqual(strip_url_credentials( + self.assertEqual(strip_url( 'http://www.example.com/index.html?somekey=somevalue'), 'http://www.example.com/index.html?somekey=somevalue') def test_fragments(self): - self.assertEqual(strip_url_credentials( - 'http://www.example.com/index.html?somekey=somevalue#section', keep_fragments=True), + self.assertEqual(strip_url( + 'http://www.example.com/index.html?somekey=somevalue#section', strip_fragment=False), 'http://www.example.com/index.html?somekey=somevalue#section') - def test_noop_trailing_path(self): - self.assertEqual(strip_url_credentials( - 'http://www.example.com/'), - 'http://www.example.com/') + def test_path(self): + for input_url, origin, output_url in [ + ('http://www.example.com/', + False, + 'http://www.example.com/'), - def test_noop_trailing_path2(self): - self.assertEqual(strip_url_credentials( - 'http://www.example.com'), - 'http://www.example.com') + ('http://www.example.com', + False, + 'http://www.example.com'), - def test_trailing_path_origin(self): - self.assertEqual(strip_url_credentials( - 'http://www.example.com', origin_only=True), - 'http://www.example.com/') + ('http://www.example.com', + True, + 'http://www.example.com/'), + ]: + self.assertEqual(strip_url(input_url, origin_only=origin), output_url) + self.assertEqual(strip_url(urlparse(input_url), origin_only=origin), output_url) - def test_username(self): - # username is stripped (and fragment too) - self.assertEqual(strip_url_credentials( - 'http://username@www.example.com/index.html?somekey=somevalue#section'), - 'http://www.example.com/index.html?somekey=somevalue') + def test_credentials(self): + for i, o in [ + ('http://username@www.example.com/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), - def test_username_empty_pass(self): - # same as above - self.assertEqual(strip_url_credentials( - 'https://username:@www.example.com/index.html?somekey=somevalue#section'), - 'https://www.example.com/index.html?somekey=somevalue') + ('https://username:@www.example.com/index.html?somekey=somevalue#section', + 'https://www.example.com/index.html?somekey=somevalue'), - def test_username_password(self): - self.assertEqual(strip_url_credentials( - 'ftp://username:password@www.example.com/index.html?somekey=somevalue#section'), - 'ftp://www.example.com/index.html?somekey=somevalue') + ('ftp://username:password@www.example.com/index.html?somekey=somevalue#section', + 'ftp://www.example.com/index.html?somekey=somevalue'), + ]: + self.assertEqual(strip_url(i, strip_credentials=True), o) + self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) - def test_default_http_port(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com:80/index.html'), - 'http://www.example.com/index.html') + def test_default_ports_creds_off(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), - def test_non_default_http_port(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com:8080/index.html'), - 'http://www.example.com:8080/index.html') + ('http://username:password@www.example.com:8080/index.html#section', + 'http://www.example.com:8080/index.html'), - def test_default_https_port(self): - self.assertEqual(strip_url_credentials( - 'https://username:password@www.example.com:443/index.html'), - 'https://www.example.com/index.html') + ('http://username:password@www.example.com:443/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov'), - def test_non_default_https_port(self): - self.assertEqual(strip_url_credentials( - 'https://username:password@www.example.com:442/index.html'), - 'https://www.example.com:442/index.html') + ('https://username:password@www.example.com:443/index.html', + 'https://www.example.com/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://www.example.com/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i), o) + self.assertEqual(strip_url(urlparse(i)), o) + + def test_default_ports(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html', + 'http://username:password@www.example.com/index.html'), + + ('http://username:password@www.example.com:8080/index.html', + 'http://username:password@www.example.com:8080/index.html'), + + ('http://username:password@www.example.com:443/index.html', + 'http://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:443/index.html', + 'https://username:password@www.example.com/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://username:password@www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://username:password@www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://username:password@www.example.com/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://username:password@www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o) + self.assertEqual(strip_url(urlparse(i), strip_default_port=True, strip_credentials=False), o) + + def test_default_ports_keep(self): + for i, o in [ + ('http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov'), + + ('http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov#section', + 'http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov'), + + ('http://username:password@www.example.com:443/index.html', + 'http://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:443/index.html', + 'https://username:password@www.example.com:443/index.html'), + + ('https://username:password@www.example.com:442/index.html', + 'https://username:password@www.example.com:442/index.html'), + + ('https://username:password@www.example.com:80/index.html', + 'https://username:password@www.example.com:80/index.html'), + + ('ftp://username:password@www.example.com:21/file.txt', + 'ftp://username:password@www.example.com:21/file.txt'), + + ('ftp://username:password@www.example.com:221/file.txt', + 'ftp://username:password@www.example.com:221/file.txt'), + ]: + self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o) + self.assertEqual(strip_url(urlparse(i), strip_default_port=False, strip_credentials=False), o) def test_origin_only(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com/index.html', origin_only=True), - 'http://www.example.com/') + for i, o in [ + ('http://username:password@www.example.com/index.html', + 'http://www.example.com/'), - def test_default_http_port_origin_only(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com:80/index.html', origin_only=True), - 'http://www.example.com/') + ('http://username:password@www.example.com:80/foo/bar?query=value#somefrag', + 'http://www.example.com/'), - def test_non_default_http_port_origin_only(self): - self.assertEqual(strip_url_credentials( - 'http://username:password@www.example.com:8008/index.html', origin_only=True), - 'http://www.example.com:8008/') + ('http://username:password@www.example.com:8008/foo/bar?query=value#somefrag', + 'http://www.example.com:8008/'), + + ('https://username:password@www.example.com:443/index.html', + 'https://www.example.com/'), + ]: + self.assertEqual(strip_url(i, origin_only=True), o) + self.assertEqual(strip_url(urlparse(i), origin_only=True), o) - def test_default_https_port_origin_only(self): - self.assertEqual(strip_url_credentials( - 'https://username:password@www.example.com:443/index.html', origin_only=True), - 'https://www.example.com/') if __name__ == "__main__": unittest.main() From 0a0b60a59f75ef1b4976b811011bd50b78f9cec8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 26 Oct 2016 12:41:00 +0200 Subject: [PATCH 038/362] Add tests for stripping userinfo with percent-encoded delimiters --- tests/test_utils_url.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 1f9845d82..9182d0fda 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -290,6 +290,26 @@ class StripUrl(unittest.TestCase): self.assertEqual(strip_url(i, strip_credentials=True), o) self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) + def test_credentials_encoded_delims(self): + for i, o in [ + # user: "username@" + # password: none + ('http://username%40@www.example.com/index.html?somekey=somevalue#section', + 'http://www.example.com/index.html?somekey=somevalue'), + + # user: "username:pass" + # password: "" + ('https://username%3Apass:@www.example.com/index.html?somekey=somevalue#section', + 'https://www.example.com/index.html?somekey=somevalue'), + + # user: "me" + # password: "user@domain.com" + ('ftp://me:user%40domain.com@www.example.com/index.html?somekey=somevalue#section', + 'ftp://www.example.com/index.html?somekey=somevalue'), + ]: + self.assertEqual(strip_url(i, strip_credentials=True), o) + self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) + def test_default_ports_creds_off(self): for i, o in [ ('http://username:password@www.example.com:80/index.html?somekey=somevalue#section', From c808a97c74718f72f53a3c00eca77f60c26fb6ba Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 12 Jan 2017 18:22:18 +0100 Subject: [PATCH 039/362] Add new "strict-" policies --- scrapy/spidermiddlewares/referer.py | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 24e5eac40..5d2a0e7b1 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -17,7 +17,9 @@ POLICY_NO_REFERRER = "no-referrer" POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade" POLICY_SAME_ORIGIN = "same-origin" POLICY_ORIGIN = "origin" +POLICY_STRICT_ORIGIN = "strict-origin" POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin" +POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN = "strict-origin-when-cross-origin" POLICY_UNSAFE_URL = "unsafe-url" POLICY_SCRAPY_DEFAULT = "scrapy-default" @@ -139,6 +141,26 @@ class OriginPolicy(ReferrerPolicy): return self.origin_referrer(response) +class StrictOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin + + The "strict-origin" policy sends the ASCII serialization + of the origin of the request client when making requests: + - from a TLS-protected environment settings object to a potentially trustworthy URL, and + - from non-TLS-protected environment settings objects to any origin. + + Requests from TLS-protected request clients to non- potentially trustworthy URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = POLICY_STRICT_ORIGIN + + def referrer(self, response, request): + if urlparse_cached(response).scheme == urlparse_cached(request).scheme: + return self.origin_referrer(response) + + class OriginWhenCrossOriginPolicy(ReferrerPolicy): """ https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin @@ -160,6 +182,33 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): return origin +class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): + """ + https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin + + The "strict-origin-when-cross-origin" policy specifies that a full URL, + stripped for use as a referrer, is sent as referrer information + when making same-origin requests from a particular request client, + and only the ASCII serialization of the origin of the request client + when making cross-origin requests: + + - from a TLS-protected environment settings object to a potentially trustworthy URL, and + - from non-TLS-protected environment settings objects to any origin. + + Requests from TLS-protected clients to non- potentially trustworthy URLs, + on the other hand, will contain no referrer information. + A Referer HTTP header will not be sent. + """ + name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN + + def referrer(self, response, request): + origin = self.origin(response) + if origin == self.origin(request): + return self.stripped_referrer(response) + else: + return origin + + class UnsafeUrlPolicy(ReferrerPolicy): """ https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url From 5cef67ae75dca229bd1b1f3ac61f52309d1ca5a4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 14:18:33 +0100 Subject: [PATCH 040/362] Update Referrer tests for "strict-" policies --- tests/test_spidermiddleware_referer.py | 75 +++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 9555817d9..df20dfbb9 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -8,6 +8,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \ POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ POLICY_SCRAPY_DEFAULT, POLICY_UNSAFE_URL, \ + POLICY_STRICT_ORIGIN, POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, \ DefaultReferrerPolicy, \ NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ OriginWhenCrossOriginPolicy, OriginPolicy, \ @@ -39,7 +40,6 @@ class TestRefererMiddleware(TestCase): for origin, target, referrer in self.scenarii: response = self.get_response(origin) request = self.get_request(target) - out = list(self.mw.process_spider_output(response, [request], self.spider)) self.assertEquals(out[0].headers.get('Referer'), referrer) @@ -154,6 +154,25 @@ class MixinOrigin(object): ] +class MixinStrictOrigin(object): + scenarii = [ + # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), + ('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'), + + # downgrade: send nothing + ('https://example.com/page.html', 'http://scrapy.org', None), + + # upgrade: send origin + ('http://example.com/page.html', 'https://scrapy.org', b'http://example.com/'), + + # test for user/password stripping + ('https://user:password@example.com/page.html', 'https://scrapy.org', b'https://example.com/'), + ('https://user:password@example.com/page.html', 'http://scrapy.org', None), + ] + + class MixinOriginWhenCrossOrigin(object): scenarii = [ # Same origin (protocol, host, port): send referrer @@ -189,6 +208,44 @@ class MixinOriginWhenCrossOrigin(object): ] +class MixinStrictOriginWhenCrossOrigin(object): + scenarii = [ + # Same origin (protocol, host, port): send referrer + ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), + ('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'), + ('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'), + ('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'), + + # Different host: send origin as referrer + ('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'), + ('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'), + ('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'), + # exact match required + ('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'), + + # Different port: send origin as referrer + ('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'), + ('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'), + + # downgrade + ('https://example4.com/page.html', 'http://example4.com/not-page.html', None), + ('https://example4.com/page.html', 'http://not.example4.com/', None), + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', None), + + # Different protocols: send origin as referrer + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), + + # test for user/password stripping + ('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'), + + # TLS to non-TLS downgrade: send nothing + ('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None), + ] + + class MixinUnsafeUrl(object): scenarii = [ # TLS to TLS: send referrer @@ -241,10 +298,18 @@ class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} +class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} + + class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} +class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): + settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} + + class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} @@ -297,10 +362,18 @@ class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_ORIGIN} +class TestRefererMiddlewareSrictOrigin(MixinStrictOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN} + + class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} +class TestRefererMiddlewareStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): + req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} + + class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_UNSAFE_URL} From 77aec5a79681128c0608e65bc77e317252172d98 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 14:19:19 +0100 Subject: [PATCH 041/362] Fix implementation --- scrapy/spidermiddlewares/referer.py | 40 ++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 5d2a0e7b1..a6316cd0c 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -31,13 +31,13 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() - def stripped_referrer(self, req_or_resp): - return self.strip_url(req_or_resp) + def stripped_referrer(self, r): + return self.strip_url(r) - def origin_referrer(self, req_or_resp): - return self.strip_url(req_or_resp, origin_only=True) + def origin_referrer(self, r): + return self.strip_url(r, origin_only=True) - def strip_url(self, req_or_resp, origin_only=False): + def strip_url(self, r, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -51,9 +51,9 @@ class ReferrerPolicy(object): Set url's query to null. Return url. """ - if req_or_resp.url is None or not req_or_resp.url: + if r is None or not r.url: return None - parsed_url = urlparse_cached(req_or_resp) + parsed_url = urlparse_cached(r) if parsed_url.scheme not in self.NOREFERRER_SCHEMES: return strip_url(parsed_url, strip_credentials=True, @@ -61,9 +61,19 @@ class ReferrerPolicy(object): strip_default_port=True, origin_only=origin_only) - def origin(self, req_or_resp): + def origin(self, r): """Return serialized origin (scheme, host, path) for a request or response URL.""" - return self.strip_url(req_or_resp, origin_only=True) + return self.strip_url(r, origin_only=True) + + def potentially_trustworthy(self, r): + # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy + parsed_url = urlparse_cached(r) + if parsed_url.scheme in ('data',): + return False + return self.tls_protected(r) + + def tls_protected(self, r): + return urlparse_cached(r).scheme in ('https', 'ftps') class NoReferrerPolicy(ReferrerPolicy): @@ -157,7 +167,9 @@ class StrictOriginPolicy(ReferrerPolicy): name = POLICY_STRICT_ORIGIN def referrer(self, response, request): - if urlparse_cached(response).scheme == urlparse_cached(request).scheme: + if ((urlparse_cached(response).scheme == 'https' and + self.potentially_trustworthy(request)) + or urlparse_cached(response).scheme == 'http'): return self.origin_referrer(response) @@ -205,8 +217,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response) if origin == self.origin(request): return self.stripped_referrer(response) - else: - return origin + elif ((urlparse_cached(response).scheme in ('https', 'ftps') and + self.potentially_trustworthy(request)) + or urlparse_cached(response).scheme == 'http'): + return self.origin_referrer(response) class UnsafeUrlPolicy(ReferrerPolicy): @@ -244,7 +258,9 @@ _policy_classes = {p.name: p for p in ( NoReferrerWhenDowngradePolicy, SameOriginPolicy, OriginPolicy, + StrictOriginPolicy, OriginWhenCrossOriginPolicy, + StrictOriginWhenCrossOriginPolicy, UnsafeUrlPolicy, DefaultReferrerPolicy, )} From deb8567116db8488a8f5b890aad975df095ddcfe Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 16:22:49 +0100 Subject: [PATCH 042/362] Update NoReferrerWhenDowngradePolicy --- scrapy/spidermiddlewares/referer.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a6316cd0c..d64c79108 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -94,12 +94,11 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): """ https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade - The "no-referrer-when-downgrade" policy sends a full URL - along with requests from a TLS-protected environment settings object - to a a priori authenticated URL, - and requests from request clients which are not TLS-protected to any origin. + The "no-referrer-when-downgrade" policy sends a full URL along with requests + from a TLS-protected environment settings object to a potentially trustworthy URL, + and requests from clients which are not TLS-protected to any origin. - Requests from TLS-protected request clients to non-a priori authenticated URLs, + Requests from TLS-protected clients to non-potentially trustworthy URLs, on the other hand, will contain no referrer information. A Referer HTTP header will not be sent. @@ -108,15 +107,8 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): name = POLICY_NO_REFERRER_WHEN_DOWNGRADE def referrer(self, response, request): - # https://www.w3.org/TR/referrer-policy/#determine-requests-referrer: - # - # If environment is TLS-protected - # and the origin of request's current URL is not an a priori authenticated URL, - # then return no referrer. - if urlparse_cached(response).scheme in ('https', 'ftps') and \ - urlparse_cached(request).scheme in ('http',): - return None - return self.stripped_referrer(response) + if not self.tls_protected(response) or self.tls_protected(request): + return self.stripped_referrer(response) class SameOriginPolicy(ReferrerPolicy): From ebcacd3f549de034d05ed946c446e0f71b11bc6f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 17:18:00 +0100 Subject: [PATCH 043/362] Update StrictOriginPolicy --- scrapy/spidermiddlewares/referer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index d64c79108..8ee04120a 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -159,9 +159,8 @@ class StrictOriginPolicy(ReferrerPolicy): name = POLICY_STRICT_ORIGIN def referrer(self, response, request): - if ((urlparse_cached(response).scheme == 'https' and - self.potentially_trustworthy(request)) - or urlparse_cached(response).scheme == 'http'): + if ((self.tls_protected(response) and self.potentially_trustworthy(request)) + or not self.tls_protected(response)): return self.origin_referrer(response) From b6c761d2b4c9cafe94010075ebf39807921dec9c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 17:57:17 +0100 Subject: [PATCH 044/362] Fix tests --- scrapy/spidermiddlewares/referer.py | 7 ++++--- tests/test_spidermiddleware_referer.py | 8 +++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 8ee04120a..4f50db689 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -159,7 +159,8 @@ class StrictOriginPolicy(ReferrerPolicy): name = POLICY_STRICT_ORIGIN def referrer(self, response, request): - if ((self.tls_protected(response) and self.potentially_trustworthy(request)) + if ((self.tls_protected(response) and + self.potentially_trustworthy(request)) or not self.tls_protected(response)): return self.origin_referrer(response) @@ -208,9 +209,9 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): origin = self.origin(response) if origin == self.origin(request): return self.stripped_referrer(response) - elif ((urlparse_cached(response).scheme in ('https', 'ftps') and + elif ((self.tls_protected(response) and self.potentially_trustworthy(request)) - or urlparse_cached(response).scheme == 'http'): + or not self.tls_protected(response)): return self.origin_referrer(response) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index df20dfbb9..4779b0ed1 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -232,7 +232,13 @@ class MixinStrictOriginWhenCrossOrigin(object): # downgrade ('https://example4.com/page.html', 'http://example4.com/not-page.html', None), ('https://example4.com/page.html', 'http://not.example4.com/', None), - ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', None), + + # non-TLS to non-TLS + ('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'), + + # upgrade + ('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'), + ('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'), # Different protocols: send origin as referrer ('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'), From c86f568b9cd1fc69c67762ba39714e38c9ba70fb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 17 Jan 2017 22:31:29 +0100 Subject: [PATCH 045/362] Update docs with "strict-..." policies --- docs/topics/spider-middleware.rst | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a9d3d4568..e8325d7ef 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -344,9 +344,18 @@ This setting accepts: - a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass, either a custom one or one of the built-in ones (see ``scrapy.spidermiddlewares.referer``), -- or one of the standard W3C-defined string values, i.e. ``"no-referrer"``, - ``"no-referrer-when-downgrade"``, ``"same-origin"``, ``"origin"``, - ``"origin-when-cross-origin"`` or ``"unsafe-url"``. +- or one of the standard W3C-defined string values: + + - `"no-referrer" `_, + - `"no-referrer-when-downgrade" `_, + - `"same-origin" `_, + - `"origin" `_, + - `"strict-origin" `_, + - `"origin-when-cross-origin" `_, + - `"strict-origin-when-cross-origin" `_, + - or `"unsafe-url" `_ + (not recommended). + (It can also be the non-standard value ``"scrapy-default"`` to use Scrapy's default referrer policy.) From e249abc32bcb171e4676b8fd9f11f04201f80a75 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 18 Jan 2017 15:48:28 +0100 Subject: [PATCH 046/362] Update docs --- docs/topics/spider-middleware.rst | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index e8325d7ef..0ddf027ea 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -333,7 +333,7 @@ Whether to enable referer middleware. REFERER_POLICY ^^^^^^^^^^^^^^ -.. versionadded:: 1.3 +.. versionadded:: 1.4 Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` @@ -346,14 +346,14 @@ This setting accepts: (see ``scrapy.spidermiddlewares.referer``), - or one of the standard W3C-defined string values: - - `"no-referrer" `_, - - `"no-referrer-when-downgrade" `_, - - `"same-origin" `_, - - `"origin" `_, - - `"strict-origin" `_, - - `"origin-when-cross-origin" `_, - - `"strict-origin-when-cross-origin" `_, - - or `"unsafe-url" `_ + - `"no-referrer"`_, + - `"no-referrer-when-downgrade"`_, + - `"same-origin"`_, + - `"origin"`_, + - `"strict-origin"`_, + - `"origin-when-cross-origin"`_, + - `"strict-origin-when-cross-origin"`_, + - or `"unsafe-url"`_ (not recommended). (It can also be the non-standard value ``"scrapy-default"`` to use @@ -364,14 +364,22 @@ with the addition that "Referrer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. .. warning:: - By default, Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_, + Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_, will send a non-empty "Referer" header from any ``https://`` to any ``https://`` URL, even if the domain is different. ``same-origin`` may be a better choice if you want to remove referrer information for cross-domain requests. .. _Referrer Policy: https://www.w3.org/TR/referrer-policy +.. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer .. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade +.. _"same-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin +.. _"origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin +.. _"strict-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin +.. _"origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin +.. _"strict-origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin +.. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url + UrlLengthMiddleware ------------------- From 03ff19d1882edb709cf627795f3ce22934235254 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 18 Jan 2017 16:29:20 +0100 Subject: [PATCH 047/362] Update docs for new "referrer_policy" Request.meta key --- docs/topics/request-response.rst | 1 + docs/topics/spider-middleware.rst | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 3d110b02d..67f8ec285 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -307,6 +307,7 @@ Those are: * :reqmeta:`proxy` * ``ftp_user`` (See :setting:`FTP_USER` for more info) * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) +* :reqmeta:`referrer_policy` .. reqmeta:: bindaddress diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 0ddf027ea..a4ac45b41 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -337,6 +337,8 @@ REFERER_POLICY Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` +.. reqmeta:: referrer_policy + `Referrer Policy`_ to apply when populating Request "Referer" header. This setting accepts: @@ -370,6 +372,11 @@ using ``file://`` or ``s3://`` scheme. ``same-origin`` may be a better choice if you want to remove referrer information for cross-domain requests. +.. note:: + You can also override the Referrer Policy per request, + using the special ``"referrer_policy"`` :ref:`Request.meta ` key, + with the same acceptable values as for the ``REFERER_POLICY`` setting. + .. _Referrer Policy: https://www.w3.org/TR/referrer-policy .. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer .. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade From eb07285a63d6fdeefe113051943f4bf7e36f7b33 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 18 Jan 2017 17:20:35 +0100 Subject: [PATCH 048/362] Reword warning on no-referrer-when-downgrade policy --- docs/topics/spider-middleware.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a4ac45b41..ada4c46c0 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -349,7 +349,8 @@ This setting accepts: - or one of the standard W3C-defined string values: - `"no-referrer"`_, - - `"no-referrer-when-downgrade"`_, + - `"no-referrer-when-downgrade"`_ + (the W3C-recommended default, used by major web browsers), - `"same-origin"`_, - `"origin"`_, - `"strict-origin"`_, @@ -358,18 +359,19 @@ This setting accepts: - or `"unsafe-url"`_ (not recommended). - (It can also be the non-standard value ``"scrapy-default"`` to use - Scrapy's default referrer policy.) +It can also be the non-standard value ``"scrapy-default"`` to use +Scrapy's default referrer policy. Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, with the addition that "Referrer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. .. warning:: - Scrapy's default referrer policy, just like `"no-referrer-when-downgrade"`_, - will send a non-empty "Referer" header from any ``https://`` to any ``https://`` URL, + Scrapy's default referrer policy—just like `"no-referrer-when-downgrade"`_, + the W3C-recommended value for browsers—will send a non-empty + "Referer" header from any ``http(s)://`` to any ``https://`` URL, even if the domain is different. - ``same-origin`` may be a better choice if you want to remove referrer + `"same-origin"`_ may be a better choice if you want to remove referrer information for cross-domain requests. .. note:: From 605935f015b1862b83f051380cd08b2931cce784 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 26 Jan 2017 12:09:34 +0100 Subject: [PATCH 049/362] Edit text --- docs/topics/spider-middleware.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index ada4c46c0..a792364b3 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -363,12 +363,12 @@ It can also be the non-standard value ``"scrapy-default"`` to use Scrapy's default referrer policy. Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, -with the addition that "Referrer" is not sent if the parent request was +with the addition that "Referer" is not sent if the parent request was using ``file://`` or ``s3://`` scheme. .. warning:: - Scrapy's default referrer policy—just like `"no-referrer-when-downgrade"`_, - the W3C-recommended value for browsers—will send a non-empty + Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_, + the W3C-recommended value for browsers — will send a non-empty "Referer" header from any ``http(s)://`` to any ``https://`` URL, even if the domain is different. `"same-origin"`_ may be a better choice if you want to remove referrer From 3dc09eeceb16ca97a0c5851bfdc41f921772910b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 30 Jan 2017 15:25:49 +0100 Subject: [PATCH 050/362] Use table for referrer policy options --- docs/topics/spider-middleware.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a792364b3..12aa53012 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -346,21 +346,21 @@ This setting accepts: - a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass, either a custom one or one of the built-in ones (see ``scrapy.spidermiddlewares.referer``), -- or one of the standard W3C-defined string values: +- or one of the standard W3C-defined string values - - `"no-referrer"`_, - - `"no-referrer-when-downgrade"`_ - (the W3C-recommended default, used by major web browsers), - - `"same-origin"`_, - - `"origin"`_, - - `"strict-origin"`_, - - `"origin-when-cross-origin"`_, - - `"strict-origin-when-cross-origin"`_, - - or `"unsafe-url"`_ - (not recommended). - -It can also be the non-standard value ``"scrapy-default"`` to use -Scrapy's default referrer policy. +======================================= ======================================================================== ======================================================= +String value Class name +======================================= ======================================================================== ======================================================= +`"no-referrer"`_ ``'scrapy.spidermiddlewares.referer.NoReferrerPolicy'`` +`"no-referrer-when-downgrade"`_ ``'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'`` the W3C-recommended default, used by major web browsers +`"same-origin"`_ ``'scrapy.spidermiddlewares.referer.SameOriginPolicy'`` +`"origin"`_ ``'scrapy.spidermiddlewares.referer.OriginPolicy'`` +`"strict-origin"`_ ``'scrapy.spidermiddlewares.referer.StrictOriginPolicy'`` +`"origin-when-cross-origin"`_ ``'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'`` +`"strict-origin-when-cross-origin"`_ ``'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'`` +`"unsafe-url"`_ ``'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'`` NOT recommended +``"scrapy-default"`` ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` Scrapy's default policy (see below) +======================================= ======================================================================== ======================================================= Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, with the addition that "Referer" is not sent if the parent request was From 537683f945906ff9865cf4e2704a432612f10eb4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 21 Feb 2017 16:47:57 +0100 Subject: [PATCH 051/362] Add autoclass directives to document built-in policies --- docs/topics/spider-middleware.rst | 70 +++++++++++++++++++---------- scrapy/spidermiddlewares/referer.py | 6 ++- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 12aa53012..349dbb3a1 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -341,43 +341,65 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` `Referrer Policy`_ to apply when populating Request "Referer" header. -This setting accepts: +.. note:: + You can also set the Referrer Policy per request, + using the special ``"referrer_policy"`` :ref:`Request.meta ` key, + with the same acceptable values as for the ``REFERER_POLICY`` setting. -- a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass, - either a custom one or one of the built-in ones - (see ``scrapy.spidermiddlewares.referer``), -- or one of the standard W3C-defined string values +Acceptable values for REFERER_POLICY +************************************ -======================================= ======================================================================== ======================================================= -String value Class name -======================================= ======================================================================== ======================================================= -`"no-referrer"`_ ``'scrapy.spidermiddlewares.referer.NoReferrerPolicy'`` -`"no-referrer-when-downgrade"`_ ``'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'`` the W3C-recommended default, used by major web browsers -`"same-origin"`_ ``'scrapy.spidermiddlewares.referer.SameOriginPolicy'`` -`"origin"`_ ``'scrapy.spidermiddlewares.referer.OriginPolicy'`` -`"strict-origin"`_ ``'scrapy.spidermiddlewares.referer.StrictOriginPolicy'`` -`"origin-when-cross-origin"`_ ``'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'`` -`"strict-origin-when-cross-origin"`_ ``'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'`` -`"unsafe-url"`_ ``'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'`` NOT recommended -``"scrapy-default"`` ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` Scrapy's default policy (see below) -======================================= ======================================================================== ======================================================= +- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` + subclass — a custom policy or one of the built-in ones (see classes below), +- or one of the standard W3C-defined string values, +- or the special ``"scrapy-default"``. -Scrapy's default referrer policy is a variant of `"no-referrer-when-downgrade"`_, -with the addition that "Referer" is not sent if the parent request was -using ``file://`` or ``s3://`` scheme. +======================================= ======================================================================== +String value Class name (as a string) +======================================= ======================================================================== +``"scrapy-default"`` (default) :class:`scrapy.spidermiddlewares.referer.DefaultReferrerPolicy` +`"no-referrer"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerPolicy` +`"no-referrer-when-downgrade"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy` +`"same-origin"`_ :class:`scrapy.spidermiddlewares.referer.SameOriginPolicy` +`"origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginPolicy` +`"strict-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginPolicy` +`"origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy` +`"strict-origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy` +`"unsafe-url"`_ :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy` +======================================= ======================================================================== +.. autoclass:: DefaultReferrerPolicy .. warning:: Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_, the W3C-recommended value for browsers — will send a non-empty "Referer" header from any ``http(s)://`` to any ``https://`` URL, even if the domain is different. + `"same-origin"`_ may be a better choice if you want to remove referrer information for cross-domain requests. +.. autoclass:: NoReferrerPolicy + +.. autoclass:: NoReferrerWhenDowngradePolicy .. note:: - You can also override the Referrer Policy per request, - using the special ``"referrer_policy"`` :ref:`Request.meta ` key, - with the same acceptable values as for the ``REFERER_POLICY`` setting. + "no-referrer-when-downgrade" policy is the W3C-recommended default, + and is used by major web browsers. + + However, it is NOT Scrapy's default referrer policy (see :class:`DefaultReferrerPolicy`). + +.. autoclass:: SameOriginPolicy + +.. autoclass:: OriginPolicy + +.. autoclass:: StrictOriginPolicy + +.. autoclass:: OriginWhenCrossOriginPolicy + +.. autoclass:: StrictOriginWhenCrossOriginPolicy + +.. autoclass:: UnsafeUrlPolicy +.. warning:: + "unsafe-url" policy is NOT recommended. .. _Referrer Policy: https://www.w3.org/TR/referrer-policy .. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 4f50db689..c015e13c8 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -240,7 +240,11 @@ class LegacyPolicy(ReferrerPolicy): class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): - + """ + A variant of "no-referrer-when-downgrade", + with the addition that "Referer" is not sent if the parent request was + using ``file://`` or ``s3://`` scheme. + """ NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3') name = POLICY_SCRAPY_DEFAULT From bc200d1155e0f93f37897ac206aa32945daf89bd Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 21 Feb 2017 17:08:39 +0100 Subject: [PATCH 052/362] Rename setting to REFERRER_POLICY (with 2 Rs) --- docs/topics/spider-middleware.rst | 12 ++++----- scrapy/settings/default_settings.py | 2 +- scrapy/spidermiddlewares/referer.py | 2 +- tests/test_spidermiddleware_referer.py | 36 +++++++++++++------------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 349dbb3a1..9a0ccd0c1 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -328,10 +328,10 @@ Default: ``True`` Whether to enable referer middleware. -.. setting:: REFERER_POLICY +.. setting:: REFERRER_POLICY -REFERER_POLICY -^^^^^^^^^^^^^^ +REFERRER_POLICY +^^^^^^^^^^^^^^^ .. versionadded:: 1.4 @@ -344,10 +344,10 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` .. note:: You can also set the Referrer Policy per request, using the special ``"referrer_policy"`` :ref:`Request.meta ` key, - with the same acceptable values as for the ``REFERER_POLICY`` setting. + with the same acceptable values as for the ``REFERRER_POLICY`` setting. -Acceptable values for REFERER_POLICY -************************************ +Acceptable values for REFERRER_POLICY +************************************* - either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass — a custom policy or one of the built-in ones (see classes below), diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 15a134dd3..35d9844a7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -234,7 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True -REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' +REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index c015e13c8..24c163089 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -266,7 +266,7 @@ class RefererMiddleware(object): def __init__(self, settings=None): self.default_policy = DefaultReferrerPolicy if settings is not None: - policy = settings.get('REFERER_POLICY') + policy = settings.get('REFERRER_POLICY') if policy is not None: # expect a string for the path to the policy class try: diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 4779b0ed1..81868efab 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -289,35 +289,35 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware): # --- Tests using settings to set policy using class path class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} class CustomPythonOrgPolicy(ReferrerPolicy): @@ -336,7 +336,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware): - settings = {'REFERER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} + settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} scenarii = [ ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), @@ -385,17 +385,17 @@ class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} req_meta = {'referrer_policy': POLICY_NO_REFERRER} class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} @@ -411,7 +411,7 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: - settings = Settings({'REFERER_POLICY': s}) + settings = Settings({'REFERRER_POLICY': s}) mw = RefererMiddleware(settings) self.assertEquals(mw.default_policy, p) @@ -425,24 +425,24 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: - settings = Settings({'REFERER_POLICY': s.upper()}) + settings = Settings({'REFERRER_POLICY': s.upper()}) mw = RefererMiddleware(settings) self.assertEquals(mw.default_policy, p) def test_invalid_name(self): - settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) + settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'}) with self.assertRaises(NotConfigured): mw = RefererMiddleware(settings) class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): - settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} From 04e4d08612364480c4d72d2b55e1cac161397136 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 27 Feb 2017 16:04:37 +0100 Subject: [PATCH 053/362] Pass URLs around instead of Request/Responses --- scrapy/spidermiddlewares/referer.py | 53 ++++++++++++++++------------- scrapy/utils/url.py | 7 +--- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 24c163089..f2910a0c8 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,6 +2,8 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ +from six.moves.urllib.parse import urlparse + from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals @@ -32,12 +34,14 @@ class ReferrerPolicy(object): raise NotImplementedError() def stripped_referrer(self, r): - return self.strip_url(r) + if urlparse(r).scheme not in self.NOREFERRER_SCHEMES: + return self.strip_url(r) def origin_referrer(self, r): - return self.strip_url(r, origin_only=True) + if urlparse(r).scheme not in self.NOREFERRER_SCHEMES: + return self.origin(r) - def strip_url(self, r, origin_only=False): + def strip_url(self, url, origin_only=False): """ https://www.w3.org/TR/referrer-policy/#strip-url @@ -51,29 +55,27 @@ class ReferrerPolicy(object): Set url's query to null. Return url. """ - if r is None or not r.url: + if not url: return None - parsed_url = urlparse_cached(r) - if parsed_url.scheme not in self.NOREFERRER_SCHEMES: - return strip_url(parsed_url, - strip_credentials=True, - strip_fragment=True, - strip_default_port=True, - origin_only=origin_only) + return strip_url(url, + strip_credentials=True, + strip_fragment=True, + strip_default_port=True, + origin_only=origin_only) def origin(self, r): """Return serialized origin (scheme, host, path) for a request or response URL.""" return self.strip_url(r, origin_only=True) - def potentially_trustworthy(self, r): + def potentially_trustworthy(self, url): # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy - parsed_url = urlparse_cached(r) + parsed_url = urlparse(url) if parsed_url.scheme in ('data',): return False - return self.tls_protected(r) + return self.tls_protected(url) - def tls_protected(self, r): - return urlparse_cached(r).scheme in ('https', 'ftps') + def tls_protected(self, url): + return urlparse(url).scheme in ('https', 'ftps') class NoReferrerPolicy(ReferrerPolicy): @@ -287,12 +289,17 @@ class RefererMiddleware(object): crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled) return mw - def policy(self, response, request): + def policy(self, resp_or_url, request): + """ + Determine Referrer-Policy to use from a parent Response (or URL), + and a Request to be sent. + """ # policy set in request's meta dict takes precedence over default policy policy_name = request.meta.get('referrer_policy') if policy_name is None: - policy_name = to_native_str( - response.headers.get('Referrer-Policy', '').decode('latin1')) + if isinstance(resp_or_url, Response): + policy_name = to_native_str( + resp_or_url.headers.get('Referrer-Policy', '').decode('latin1')) cls = _policy_classes.get(policy_name.lower(), self.default_policy) return cls() @@ -300,7 +307,7 @@ class RefererMiddleware(object): def process_spider_output(self, response, result, spider): def _set_referer(r): if isinstance(r, Request): - referrer = self.policy(response, r).referrer(response, r) + referrer = self.policy(response, r).referrer(response.url, r.url) if referrer is not None: r.headers.setdefault('Referer', referrer) return r @@ -313,9 +320,9 @@ class RefererMiddleware(object): request_referrer = request.headers.get('Referer') # we don't patch the referrer value if there is none if request_referrer is not None: - faked_response = Response(redirected_urls[0]) - policy_referrer = self.policy(faked_response, - request).referrer(faked_response, request) + initial_url = redirected_urls[0] + policy_referrer = self.policy(initial_url, + request).referrer(orig_url, request.url) if policy_referrer != request_referrer: if policy_referrer is None: request.headers.pop('Referer') diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 9864f353d..8eed31060 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -117,12 +117,7 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only= - `strip_fragment` drops any #fragment component """ - if url is None: - return None - if not isinstance(url, ParseResult): - parsed_url = urlparse(url) - else: - parsed_url = url + parsed_url = urlparse(url) netloc = parsed_url.netloc if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password): netloc = netloc.split('@')[-1] From d2aa51c0fb81b4e264fc351b9184d9ce06855021 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 27 Feb 2017 16:05:22 +0100 Subject: [PATCH 054/362] Update tests --- tests/test_spidermiddleware_referer.py | 43 ++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 81868efab..39bbaab5d 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,3 +1,4 @@ +from six.moves.urllib.parse import urlparse from unittest import TestCase from scrapy.exceptions import NotConfigured @@ -326,9 +327,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): depending on the scheme of the target URL. """ def referrer(self, response, request): - from scrapy.utils.httpobj import urlparse_cached - - scheme = urlparse_cached(request).scheme + scheme = urlparse(request).scheme if scheme == 'https': return b'https://python.org/' elif scheme == 'http': @@ -446,3 +445,41 @@ class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefere class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} + + +class TestReferrerPolicyOnRedirect(TestRefererMiddleware): + + req_meta = {} + resp_headers = {} + #settings = {} + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} + scenarii = [ + ( # origin + 'http://scrapytest.org/1', + + # target + redirection + ['http://scrapytest.org/2', + 'http://scrapytest.org/3',], + + b'http://scrapytest.org/1', # expected initial referer + b'http://scrapytest.org/1', # expected referer for the redirection request + + ), + ] + + def setUp(self): + self.spider = Spider('foo') + settings = Settings(self.settings) + self.mw = RefererMiddleware(settings) + + def test_(self): + + for origin, target_chain, init_referrer, final_referrer in self.scenarii: + response = self.get_response(origin) + request = self.get_request(target_chain.pop()) + + + out = list(self.mw.process_spider_output(response, [request], self.spider)) + self.assertEquals(out[0].headers.get('Referer'), init_referrer) + + request.meta['redirected_urls'] = target_chain From 8226e77010d79f64c7ec5ebdb9e7cee7803600ed Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 1 Mar 2017 12:41:33 +0100 Subject: [PATCH 055/362] Add test for Referer header on HTTP redirections --- scrapy/spidermiddlewares/referer.py | 16 +- tests/test_spidermiddleware_referer.py | 398 ++++++++++++++++++++++--- 2 files changed, 364 insertions(+), 50 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index f2910a0c8..bb184cc12 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -4,6 +4,8 @@ originated it. """ from six.moves.urllib.parse import urlparse +from w3lib.url import safe_url_string + from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals @@ -300,8 +302,7 @@ class RefererMiddleware(object): if isinstance(resp_or_url, Response): policy_name = to_native_str( resp_or_url.headers.get('Referrer-Policy', '').decode('latin1')) - - cls = _policy_classes.get(policy_name.lower(), self.default_policy) + cls = _policy_classes.get(policy_name.lower()) if policy_name else self.default_policy return cls() def process_spider_output(self, response, result, spider): @@ -320,9 +321,14 @@ class RefererMiddleware(object): request_referrer = request.headers.get('Referer') # we don't patch the referrer value if there is none if request_referrer is not None: - initial_url = redirected_urls[0] - policy_referrer = self.policy(initial_url, - request).referrer(orig_url, request.url) + # the request's referrer header value acts as a surrogate + # for the parent response URL + # + # Note: if the 3xx response contained a Referrer-Policy header, + # the information is not available using this hook + parent_url = safe_url_string(request_referrer) + policy_referrer = self.policy(parent_url, request).referrer( + parent_url, request.url) if policy_referrer != request_referrer: if policy_referrer is None: request.headers.pop('Referer') diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 39bbaab5d..28c694169 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -5,6 +5,7 @@ from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request from scrapy.settings import Settings from scrapy.spiders import Spider +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.spidermiddlewares.referer import RefererMiddleware, \ POLICY_NO_REFERRER, POLICY_NO_REFERRER_WHEN_DOWNGRADE, \ POLICY_SAME_ORIGIN, POLICY_ORIGIN, POLICY_ORIGIN_WHEN_CROSS_ORIGIN, \ @@ -13,6 +14,7 @@ from scrapy.spidermiddlewares.referer import RefererMiddleware, \ DefaultReferrerPolicy, \ NoReferrerPolicy, NoReferrerWhenDowngradePolicy, \ OriginWhenCrossOriginPolicy, OriginPolicy, \ + StrictOriginWhenCrossOriginPolicy, StrictOriginPolicy, \ SameOriginPolicy, UnsafeUrlPolicy, ReferrerPolicy @@ -289,35 +291,35 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware): # --- Tests using settings to set policy using class path -class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): +class TestSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} -class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): +class TestSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} -class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): +class TestSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} -class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): +class TestSettingsOrigin(MixinOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} -class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): +class TestSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} -class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): +class TestSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} -class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): +class TestSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} -class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): +class TestSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} @@ -334,7 +336,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy): return b'http://python.org/' -class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware): +class TestSettingsCustomPolicy(TestRefererMiddleware): settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} scenarii = [ ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), @@ -347,58 +349,58 @@ class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware): ] # --- Tests using Request meta dict to set policy -class TestRefererMiddlewareDefaultMeta(MixinDefault, TestRefererMiddleware): +class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} -class TestRefererMiddlewareNoReferrer(MixinNoReferrer, TestRefererMiddleware): +class TestRequestMetaNoReferrer(MixinNoReferrer, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_NO_REFERRER} -class TestRefererMiddlewareNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): +class TestRequestMetaNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE} -class TestRefererMiddlewareSameOrigin(MixinSameOrigin, TestRefererMiddleware): +class TestRequestMetaSameOrigin(MixinSameOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_SAME_ORIGIN} -class TestRefererMiddlewareOrigin(MixinOrigin, TestRefererMiddleware): +class TestRequestMetaOrigin(MixinOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_ORIGIN} -class TestRefererMiddlewareSrictOrigin(MixinStrictOrigin, TestRefererMiddleware): +class TestRequestMetaSrictOrigin(MixinStrictOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN} -class TestRefererMiddlewareOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): +class TestRequestMetaOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} -class TestRefererMiddlewareStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): +class TestRequestMetaStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} -class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): +class TestRequestMetaUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_UNSAFE_URL} -class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): +class TestRequestMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} -class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): +class TestRequestMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} req_meta = {'referrer_policy': POLICY_NO_REFERRER} -class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): +class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} req_meta = {'referrer_policy': POLICY_UNSAFE_URL} -class TestRefererMiddlewareSettingsPolicyByName(TestCase): +class TestSettingsPolicyByName(TestCase): def test_valid_name(self): for s, p in [ @@ -407,7 +409,9 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), (POLICY_SAME_ORIGIN, SameOriginPolicy), (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: settings = Settings({'REFERRER_POLICY': s}) @@ -421,7 +425,9 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): (POLICY_NO_REFERRER_WHEN_DOWNGRADE, NoReferrerWhenDowngradePolicy), (POLICY_SAME_ORIGIN, SameOriginPolicy), (POLICY_ORIGIN, OriginPolicy), + (POLICY_STRICT_ORIGIN, StrictOriginPolicy), (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), + (POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN, StrictOriginWhenCrossOriginPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy), ]: settings = Settings({'REFERRER_POLICY': s.upper()}) @@ -434,52 +440,354 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase): mw = RefererMiddleware(settings) -class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): +class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} -class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): +class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} -class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): +class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} -class TestReferrerPolicyOnRedirect(TestRefererMiddleware): +class TestReferrerOnRedirect(TestRefererMiddleware): - req_meta = {} - resp_headers = {} - #settings = {} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} scenarii = [ - ( # origin - 'http://scrapytest.org/1', - - # target + redirection - ['http://scrapytest.org/2', - 'http://scrapytest.org/3',], - + ( 'http://scrapytest.org/1', # parent + 'http://scrapytest.org/2', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/3'), + (301, 'http://scrapytest.org/4'), + ), b'http://scrapytest.org/1', # expected initial referer b'http://scrapytest.org/1', # expected referer for the redirection request - + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.org/2', + ( + # redirecting to non-secure URL + (301, 'http://scrapytest.org/3'), + ), + b'https://scrapytest.org/1', + b'https://scrapytest.org/1', + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.com/2', + ( + # redirecting to non-secure URL: different origin + (301, 'http://scrapytest.com/3'), + ), + b'https://scrapytest.org/1', + b'https://scrapytest.org/1', ), ] def setUp(self): self.spider = Spider('foo') settings = Settings(self.settings) - self.mw = RefererMiddleware(settings) + self.referrermw = RefererMiddleware(settings) + self.redirectmw = RedirectMiddleware(settings) - def test_(self): + def test(self): - for origin, target_chain, init_referrer, final_referrer in self.scenarii: - response = self.get_response(origin) - request = self.get_request(target_chain.pop()) + for parent, target, redirections, init_referrer, final_referrer in self.scenarii: + response = self.get_response(parent) + request = self.get_request(target) - - out = list(self.mw.process_spider_output(response, [request], self.spider)) + out = list(self.referrermw.process_spider_output(response, [request], self.spider)) self.assertEquals(out[0].headers.get('Referer'), init_referrer) - request.meta['redirected_urls'] = target_chain + for status, url in redirections: + response = Response(request.url, headers={'Location': url}, status=status) + request = self.redirectmw.process_response(request, response, self.spider) + self.referrermw.request_scheduled(request, self.spider) + + assert isinstance(request, Request) + self.assertEquals(request.headers.get('Referer'), final_referrer) + + +class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): + """ + No Referrer policy never sets the "Referer" header. + HTTP redirections should not change that. + """ + settings = {'REFERRER_POLICY': 'no-referrer'} + scenarii = [ + ( 'http://scrapytest.org/1', # parent + 'http://scrapytest.org/2', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/3'), + (301, 'http://scrapytest.org/4'), + ), + None, # expected initial "Referer" + None, # expected "Referer" for the redirection request + ), + ( 'https://scrapytest.org/1', + 'https://scrapytest.org/2', + ( + (301, 'http://scrapytest.org/3'), + ), + None, + None, + ), + ( 'https://scrapytest.org/1', + 'https://example.com/2', # different origin + ( + (301, 'http://scrapytest.com/3'), + ), + None, + None, + ), + ] + + +class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect): + """ + Same Origin policy sends the full URL as "Referer" if the target origin + is the same as the parent response (same protocol, same domain, same port). + + HTTP redirections to a different domain or a lower secure level + should have the "Referer" removed. + """ + settings = {'REFERRER_POLICY': 'same-origin'} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial "Referer" + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting from secure to non-secure URL == different origin + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # different domain == different origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + None, + ), + ] + + +class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect): + """ + Strict Origin policy will always send the "origin" as referrer + (think of it as the parent URL without the path part), + unless the security level is lower and no "Referer" is sent. + + Redirections from secure to non-secure URLs should have the + "Referrer" header removed if necessary. + """ + settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', + 'http://scrapytest.org/102', + ( + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/', # send origin + b'http://scrapytest.org/', # redirects to same origin: send origin + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: no referrer + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): no referrer + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/', + None, + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # HTTPS all along, so origin referrer is kept as-is + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/601', + 'http://scrapytest.org/602', # TLS to non-TLS: no referrer + ( + (301, 'https://scrapytest.org/603'), # TLS URL again: (still) no referrer + ), + None, + None, + ), + ] + + +class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect): + """ + Origin When Cross-Origin policy sends the full URL as "Referer", + unless the target's origin is different (different domain, different protocol) + in which case only the origin is sent. + + Redirections to a different origin should strip the "Referer" + to the parent origin. + """ + settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + redirection + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: send origin + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + b'https://scrapytest.org/', + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): send origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + b'https://scrapytest.org/', + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # all different domains: send origin + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/301', + 'http://scrapytest.org/302', # TLS to non-TLS: send origin + ( + (301, 'https://scrapytest.org/303'), # TLS URL again: send origin (also) + ), + b'https://scrapytest.org/', + b'https://scrapytest.org/', + ), + ] + + +class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect): + """ + Strict Origin When Cross-Origin policy sends the full URL as "Referer", + unless the target's origin is different (different domain, different protocol) + in which case only the origin is sent... + Unless there's also a downgrade in security and then the "Referer" header + is not sent. + + Redirections to a different origin should strip the "Referer" to the parent origin, + and from https:// to http:// will remove the "Referer" header. + """ + settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN} + scenarii = [ + ( 'http://scrapytest.org/101', # origin + 'http://scrapytest.org/102', # target + redirection + ( + # redirections: code, URL + (301, 'http://scrapytest.org/103'), + (301, 'http://scrapytest.org/104'), + ), + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request + ), + ( 'https://scrapytest.org/201', + 'https://scrapytest.org/202', + ( + # redirecting to non-secure URL: do not send the "Referer" header + (301, 'http://scrapytest.org/203'), + ), + b'https://scrapytest.org/201', + None, + ), + ( 'https://scrapytest.org/301', + 'https://scrapytest.org/302', + ( + # redirecting to non-secure URL (different domain): send origin + (301, 'http://example.com/303'), + ), + b'https://scrapytest.org/301', + None, + ), + ( 'http://scrapy.org/401', + 'http://example.com/402', + ( + (301, 'http://scrapytest.org/403'), + ), + b'http://scrapy.org/', + b'http://scrapy.org/', + ), + ( 'https://scrapy.org/501', + 'https://example.com/502', + ( + # all different domains: send origin + (301, 'https://google.com/503'), + (301, 'https://facebook.com/504'), + ), + b'https://scrapy.org/', + b'https://scrapy.org/', + ), + ( 'https://scrapytest.org/601', + 'http://scrapytest.org/602', # TLS to non-TLS: do not send "Referer" + ( + (301, 'https://scrapytest.org/603'), # TLS URL again: (still) send nothing + ), + None, + None, + ), + ] From 6916dd6240940cab06e2d02b3fceeb32c70691de Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 1 Mar 2017 17:42:46 +0100 Subject: [PATCH 056/362] Warn or fail with exception on unknown policies --- scrapy/spidermiddlewares/referer.py | 55 +++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index bb184cc12..5ed40791b 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -3,6 +3,7 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ from six.moves.urllib.parse import urlparse +import warnings from w3lib.url import safe_url_string @@ -265,45 +266,69 @@ _policy_classes = {p.name: p for p in ( DefaultReferrerPolicy, )} + +def _load_policy_class(policy, warning_only=False): + """ + Expect a string for the path to the policy class, + otherwise try to interpret the string as a standard value + from https://www.w3.org/TR/referrer-policy/#referrer-policies + """ + try: + return load_object(policy) + except ValueError: + try: + return _policy_classes[policy.lower()] + except KeyError: + msg = "Could not load referrer policy %r" % policy + if not warning_only: + raise RuntimeError(msg) + else: + warnings.warn(msg, RuntimeWarning) + return None + + class RefererMiddleware(object): def __init__(self, settings=None): self.default_policy = DefaultReferrerPolicy if settings is not None: - policy = settings.get('REFERRER_POLICY') - if policy is not None: - # expect a string for the path to the policy class - try: - self.default_policy = load_object(policy) - except ValueError: - # otherwise try to interpret the string as standard - # https://www.w3.org/TR/referrer-policy/#referrer-policies - try: - self.default_policy = _policy_classes[policy.lower()] - except: - raise NotConfigured("Unknown referrer policy name %r" % policy) + self.default_policy = _load_policy_class( + settings.get('REFERRER_POLICY')) @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('REFERER_ENABLED'): raise NotConfigured mw = cls(crawler.settings) + + # Note: this hook is a bit of a hack to intercept redirections crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled) + return mw def policy(self, resp_or_url, request): """ Determine Referrer-Policy to use from a parent Response (or URL), and a Request to be sent. + + - if a valid policy is set in Request meta, it is used. + - if the policy is set in meta but is wrong (e.g. a typo error), + the policy from settings is used + - if the policy is not set in Request meta, + but there is a Referrer-policy header in the parent response, + it is used if valid + - otherwise, the policy from settings is used. """ - # policy set in request's meta dict takes precedence over default policy policy_name = request.meta.get('referrer_policy') if policy_name is None: if isinstance(resp_or_url, Response): policy_name = to_native_str( resp_or_url.headers.get('Referrer-Policy', '').decode('latin1')) - cls = _policy_classes.get(policy_name.lower()) if policy_name else self.default_policy - return cls() + if policy_name is None: + return self.default_policy() + + cls = _load_policy_class(policy_name, warning_only=True) + return cls() if cls else self.default_policy() def process_spider_output(self, response, result, spider): def _set_referer(r): From efa50039ec4dc8d5cd5103e64c47ad8c86b9ed4e Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 1 Mar 2017 17:43:47 +0100 Subject: [PATCH 057/362] Add tests for policy fallback on unknown policies from meta and headers --- tests/test_spidermiddleware_referer.py | 76 +++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 28c694169..b1c815876 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,5 +1,6 @@ from six.moves.urllib.parse import urlparse from unittest import TestCase +import warnings from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request @@ -400,6 +401,79 @@ class TestRequestMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_UNSAFE_URL} +class TestRequestMetaSettingFallback(TestCase): + + params = [ + ( + # When an unknown policy is referenced in Request.meta + # (here, a typo error), + # the policy defined in settings takes precedence + {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}, + {}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # same as above but with string value for settings policy + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # request meta references a wrong policy but it is set, + # so the Referrer-Policy header in response is not used, + # and the settings' policy is applied + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unsafe-url'}, + {'referrer_policy': 'ssscrapy-default'}, + OriginWhenCrossOriginPolicy, + True + ), + ( + # here, request meta does not set the policy + # so response headers take precedence + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unsafe-url'}, + {}, + UnsafeUrlPolicy, + False + ), + ( + # here, request meta does not set the policy, + # but response headers also use an unknown policy, + # so the settings' policy is used + {'REFERRER_POLICY': 'origin-when-cross-origin'}, + {'Referrer-Policy': 'unknown'}, + {}, + OriginWhenCrossOriginPolicy, + True + ) + ] + + def test(self): + + origin = 'http://www.scrapy.org' + target = 'http://www.example.com' + + for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]: + spider = Spider('foo') + mw = RefererMiddleware(Settings(settings)) + + response = Response(origin, headers=response_headers) + request = Request(target, meta=request_meta) + + with warnings.catch_warnings(record=True) as w: + policy = mw.policy(response, request) + self.assertIsInstance(policy, policy_class) + + if check_warning: + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, RuntimeWarning, w[0].message) + + class TestSettingsPolicyByName(TestCase): def test_valid_name(self): @@ -436,7 +510,7 @@ class TestSettingsPolicyByName(TestCase): def test_invalid_name(self): settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'}) - with self.assertRaises(NotConfigured): + with self.assertRaises(RuntimeError): mw = RefererMiddleware(settings) From 2d55d838ca1e2936e5729e67a6ab34a4bb58ba3b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 1 Mar 2017 20:59:52 +0100 Subject: [PATCH 058/362] Fix strip_url() tests --- tests/test_utils_url.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 9182d0fda..c2b9fc176 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -274,7 +274,6 @@ class StripUrl(unittest.TestCase): 'http://www.example.com/'), ]: self.assertEqual(strip_url(input_url, origin_only=origin), output_url) - self.assertEqual(strip_url(urlparse(input_url), origin_only=origin), output_url) def test_credentials(self): for i, o in [ @@ -288,7 +287,6 @@ class StripUrl(unittest.TestCase): 'ftp://www.example.com/index.html?somekey=somevalue'), ]: self.assertEqual(strip_url(i, strip_credentials=True), o) - self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) def test_credentials_encoded_delims(self): for i, o in [ @@ -308,7 +306,6 @@ class StripUrl(unittest.TestCase): 'ftp://www.example.com/index.html?somekey=somevalue'), ]: self.assertEqual(strip_url(i, strip_credentials=True), o) - self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o) def test_default_ports_creds_off(self): for i, o in [ @@ -337,7 +334,6 @@ class StripUrl(unittest.TestCase): 'ftp://www.example.com:221/file.txt'), ]: self.assertEqual(strip_url(i), o) - self.assertEqual(strip_url(urlparse(i)), o) def test_default_ports(self): for i, o in [ @@ -366,7 +362,6 @@ class StripUrl(unittest.TestCase): 'ftp://username:password@www.example.com:221/file.txt'), ]: self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o) - self.assertEqual(strip_url(urlparse(i), strip_default_port=True, strip_credentials=False), o) def test_default_ports_keep(self): for i, o in [ @@ -395,7 +390,6 @@ class StripUrl(unittest.TestCase): 'ftp://username:password@www.example.com:221/file.txt'), ]: self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o) - self.assertEqual(strip_url(urlparse(i), strip_default_port=False, strip_credentials=False), o) def test_origin_only(self): for i, o in [ @@ -412,7 +406,6 @@ class StripUrl(unittest.TestCase): 'https://www.example.com/'), ]: self.assertEqual(strip_url(i, origin_only=True), o) - self.assertEqual(strip_url(urlparse(i), origin_only=True), o) if __name__ == "__main__": From a70ec30e19eff13d3bf18e5c392612db018d1b4f Mon Sep 17 00:00:00 2001 From: Oto Brglez Date: Wed, 1 Mar 2017 23:02:42 +0100 Subject: [PATCH 059/362] Adding new options. --- scrapy/pipelines/files.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 843b4d3ec..7041a0a78 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -40,7 +40,6 @@ class FileException(Exception): class FSFilesStore(object): - def __init__(self, basedir): if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -79,9 +78,12 @@ class FSFilesStore(object): class S3FilesStore(object): - AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None + AWS_ENDPOINT_URL = None + AWS_REGION_NAME = None + AWS_USE_SSL = None + AWS_VERIFY = None POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in # FilesPipeline.from_settings. @@ -95,8 +97,14 @@ class S3FilesStore(object): import botocore.session session = botocore.session.get_session() self.s3_client = session.create_client( - 's3', aws_access_key_id=self.AWS_ACCESS_KEY_ID, - aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY) + 's3', + aws_access_key_id=self.AWS_ACCESS_KEY_ID, + aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, + endpoint_url=self.AWS_ENDPOINT_URL, + region_name=self.AWS_REGION_NAME, + use_ssl=self.AWS_USE_SSL, + verify=self.AWS_VERIFY + ) else: from boto.s3.connection import S3Connection self.S3Connection = S3Connection @@ -181,7 +189,7 @@ class S3FilesStore(object): 'X-Amz-Grant-Read': 'GrantRead', 'X-Amz-Grant-Read-ACP': 'GrantReadACP', 'X-Amz-Grant-Write-ACP': 'GrantWriteACP', - }) + }) extra = {} for key, value in six.iteritems(headers): try: @@ -226,7 +234,7 @@ class FilesPipeline(MediaPipeline): def __init__(self, store_uri, download_func=None, settings=None): if not store_uri: raise NotConfigured - + if isinstance(settings, dict) or settings is None: settings = Settings(settings) @@ -256,6 +264,10 @@ class FilesPipeline(MediaPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] + s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] + s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] + s3store.AWS_VERIFY = settings['AWS_VERIFY'] s3store.POLICY = settings['FILES_STORE_S3_ACL'] store_uri = settings['FILES_STORE'] @@ -423,4 +435,5 @@ class FilesPipeline(MediaPipeline): # deprecated def file_key(self, url): return self.file_path(url) + file_key._base = True From 97d84d920bf97f3ec1becd291596b3f0ee966328 Mon Sep 17 00:00:00 2001 From: jorenham Date: Thu, 2 Mar 2017 11:04:16 +0100 Subject: [PATCH 060/362] Logging the cache directory at HttpCacheMiddleware instantiation #2604 --- scrapy/downloadermiddlewares/httpcache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 30e49b886..6f1ccce68 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,3 +1,5 @@ +import logging + from email.utils import formatdate from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ @@ -9,6 +11,9 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.utils.misc import load_object +logger = logging.getLogger(__name__) + + class HttpCacheMiddleware(object): DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, @@ -24,6 +29,8 @@ class HttpCacheMiddleware(object): self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') self.stats = stats + logger.debug("Using cache directory %(cachedir)s" % {'cachedir': self.storage.cachedir}) + @classmethod def from_crawler(cls, crawler): o = cls(crawler.settings, crawler.stats) From f3b75c940d2fefc3ff5bb4435e507c5959ff903c Mon Sep 17 00:00:00 2001 From: MrMenezes Date: Fri, 21 Oct 2016 18:02:16 -0300 Subject: [PATCH 061/362] Fix warning to duplicated spider. Issue 2181 --- scrapy/spiderloader.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index d4f0f663f..e6c3e64a5 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -23,7 +23,12 @@ class SpiderLoader(object): def _load_spiders(self, module): for spcls in iter_spider_classes(module): - self._spiders[spcls.name] = spcls + if spcls.name in self._spiders.keys(): + import warnings + warnings.warn("There are several spiders with the same name (" + spcls.name + + "), this can cause unexpected behavior", UserWarning) + self._spiders[spcls.name] = spcls + def _load_all_spiders(self): for name in self.spider_modules: From 6abd9ba843e54ecb869af9571192c3f33375a2b6 Mon Sep 17 00:00:00 2001 From: Erick Date: Fri, 21 Oct 2016 18:24:59 -0300 Subject: [PATCH 062/362] Fix warning to duplicated spider. Issue 2181 --- scrapy/spiderloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index e6c3e64a5..6093c07c6 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -27,7 +27,7 @@ class SpiderLoader(object): import warnings warnings.warn("There are several spiders with the same name (" + spcls.name + "), this can cause unexpected behavior", UserWarning) - self._spiders[spcls.name] = spcls + self._spiders[spcls.name] = spcls def _load_all_spiders(self): From 5be5ef57f38b5d6f8ed01ccf9c9cac1e1e02fe03 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 23 Nov 2016 11:01:31 +0100 Subject: [PATCH 063/362] Remove extra blank line --- scrapy/spiderloader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 6093c07c6..1322c01d1 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -28,7 +28,6 @@ class SpiderLoader(object): warnings.warn("There are several spiders with the same name (" + spcls.name + "), this can cause unexpected behavior", UserWarning) self._spiders[spcls.name] = spcls - def _load_all_spiders(self): for name in self.spider_modules: From e71803c833edd67400848a3b0d4dbb01e0ec80f7 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 23 Nov 2016 11:52:02 +0100 Subject: [PATCH 064/362] Add tests for duplicate spider name warnings --- tests/test_spiderloader/__init__.py | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index b2ad93b3f..ac5f0ddab 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -101,3 +101,52 @@ class SpiderLoaderTest(unittest.TestCase): spiders = spider_loader.list() self.assertEqual(spiders, []) + + +class DuplicateSpiderNameLoaderTest(unittest.TestCase): + + def setUp(self): + orig_spiders_dir = os.path.join(module_dir, 'test_spiders') + self.tmpdir = self.mktemp() + os.mkdir(self.tmpdir) + self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') + shutil.copytree(orig_spiders_dir, self.spiders_dir) + sys.path.append(self.tmpdir) + self.settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) + + def tearDown(self): + del sys.modules['test_spiders_xxx'] + sys.path.remove(self.tmpdir) + + def test_dupename_warning(self): + # copy 1 spider module so as to have duplicate spider name + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider3.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx/spider3dupe.py')) + + with warnings.catch_warnings(record=True) as w: + spider_loader = SpiderLoader.from_settings(self.settings) + + self.assertEqual(len(w), 1) + self.assertIn("several spiders with the same name (spider3)", str(w[0].message)) + + spiders = set(spider_loader.list()) + self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3'])) + + def test_multiple_dupename_warning(self): + # copy 2 spider modules so as to have duplicate spider name + # This should issue 2 warning, 1 for each duplicate spider name + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider1.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx/spider1dupe.py')) + shutil.copyfile(os.path.join(self.tmpdir, 'test_spiders_xxx/spider2.py'), + os.path.join(self.tmpdir, 'test_spiders_xxx/spider2dupe.py')) + + with warnings.catch_warnings(record=True) as w: + spider_loader = SpiderLoader.from_settings(self.settings) + + self.assertEqual(len(w), 2) + msgs = sorted(str(wrn.message) for wrn in w) + self.assertIn("several spiders with the same name (spider1)", msgs[0]) + self.assertIn("several spiders with the same name (spider2)", msgs[1]) + + spiders = set(spider_loader.list()) + self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3'])) From 12a8ddecab8c64923d1789571955699270255211 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Mar 2017 13:03:18 +0100 Subject: [PATCH 065/362] Fix tests --- tests/test_spiderloader/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index ac5f0ddab..302bf9a10 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -130,7 +130,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): self.assertIn("several spiders with the same name (spider3)", str(w[0].message)) spiders = set(spider_loader.list()) - self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3'])) + self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) def test_multiple_dupename_warning(self): # copy 2 spider modules so as to have duplicate spider name @@ -149,4 +149,4 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): self.assertIn("several spiders with the same name (spider2)", msgs[1]) spiders = set(spider_loader.list()) - self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3'])) + self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) From f96490df2ce532e3b8757b2464f5f3cc54181ce5 Mon Sep 17 00:00:00 2001 From: jorenham Date: Thu, 2 Mar 2017 16:17:51 +0100 Subject: [PATCH 066/362] Move cache storage logging to the individual storage classes --- scrapy/downloadermiddlewares/httpcache.py | 7 ------- scrapy/extensions/httpcache.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 6f1ccce68..30e49b886 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,5 +1,3 @@ -import logging - from email.utils import formatdate from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ @@ -11,9 +9,6 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.utils.misc import load_object -logger = logging.getLogger(__name__) - - class HttpCacheMiddleware(object): DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, @@ -29,8 +24,6 @@ class HttpCacheMiddleware(object): self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') self.stats = stats - logger.debug("Using cache directory %(cachedir)s" % {'cachedir': self.storage.cachedir}) - @classmethod def from_crawler(cls, crawler): o = cls(crawler.settings, crawler.stats) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 247cac64e..8025efe77 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,6 +1,7 @@ from __future__ import print_function import os import gzip +import logging from six.moves import cPickle as pickle from importlib import import_module from time import time @@ -15,6 +16,9 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode +logger = logging.getLogger(__name__) + + class DummyPolicy(object): def __init__(self, settings): @@ -216,6 +220,8 @@ class DbmCacheStorage(object): self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE']) self.db = None + logger.debug("Using DBM cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) + def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) self.db = self.dbmodule.open(dbpath, 'c') @@ -271,6 +277,8 @@ class FilesystemCacheStorage(object): self.use_gzip = settings.getbool('HTTPCACHE_GZIP') self._open = gzip.open if self.use_gzip else open + logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) + def open_spider(self, spider): pass @@ -344,6 +352,8 @@ class LeveldbCacheStorage(object): self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') self.db = None + logger.debug("Using LevelDB cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) + def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name) self.db = self._leveldb.LevelDB(dbpath) From b50d0370f4ebdb485f4625ef6d71398ac8538c79 Mon Sep 17 00:00:00 2001 From: Artur Gaspar Date: Thu, 2 Mar 2017 14:46:33 -0300 Subject: [PATCH 067/362] Test response attributes in data URI download handler. --- tests/test_downloader_handlers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index b27245a36..74203dbfe 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -838,6 +838,16 @@ class DataURITestCase(unittest.TestCase): self.download_request = self.download_handler.download_request self.spider = Spider('foo') + def test_response_attrs(self): + uri = "data:,A%20brief%20note" + + def _test(response): + self.assertEquals(response.url, uri) + self.assertFalse(response.headers) + + request = Request(uri) + return self.download_request(request, self.spider).addCallback(_test) + def test_default_mediatype_encoding(self): def _test(response): self.assertEquals(response.text, 'A brief note') From c2c503192f24a198b4515ab006e3174e28062731 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Mar 2017 22:53:27 +0100 Subject: [PATCH 068/362] Rename arguments --- scrapy/spidermiddlewares/referer.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 5ed40791b..2bc8b1782 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -36,13 +36,13 @@ class ReferrerPolicy(object): def referrer(self, response, request): raise NotImplementedError() - def stripped_referrer(self, r): - if urlparse(r).scheme not in self.NOREFERRER_SCHEMES: - return self.strip_url(r) + def stripped_referrer(self, url): + if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: + return self.strip_url(url) - def origin_referrer(self, r): - if urlparse(r).scheme not in self.NOREFERRER_SCHEMES: - return self.origin(r) + def origin_referrer(self, url): + if urlparse(url).scheme not in self.NOREFERRER_SCHEMES: + return self.origin(url) def strip_url(self, url, origin_only=False): """ @@ -66,9 +66,9 @@ class ReferrerPolicy(object): strip_default_port=True, origin_only=origin_only) - def origin(self, r): + def origin(self, url): """Return serialized origin (scheme, host, path) for a request or response URL.""" - return self.strip_url(r, origin_only=True) + return self.strip_url(url, origin_only=True) def potentially_trustworthy(self, url): # Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy @@ -241,7 +241,7 @@ class UnsafeUrlPolicy(ReferrerPolicy): class LegacyPolicy(ReferrerPolicy): def referrer(self, response, request): - return response.url + return response class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): From db176f872b91c4f51f10ea1e732875dc40a9e7bb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Mar 2017 22:56:23 +0100 Subject: [PATCH 069/362] Remove Legacy Policy which is equivalent to UnsafeUrl Policy --- scrapy/spidermiddlewares/referer.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 2bc8b1782..7b41bfaf7 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -239,11 +239,6 @@ class UnsafeUrlPolicy(ReferrerPolicy): return self.stripped_referrer(response) -class LegacyPolicy(ReferrerPolicy): - def referrer(self, response, request): - return response - - class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): """ A variant of "no-referrer-when-downgrade", From fad499ab602e7f12bf56a514f5aae1a2f80670f7 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Mar 2017 23:06:04 +0100 Subject: [PATCH 070/362] Rename arguments (bis) --- scrapy/spidermiddlewares/referer.py | 58 ++++++++++++++--------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 7b41bfaf7..b444e34bb 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -33,7 +33,7 @@ class ReferrerPolicy(object): NOREFERRER_SCHEMES = LOCAL_SCHEMES - def referrer(self, response, request): + def referrer(self, response_url, request_url): raise NotImplementedError() def stripped_referrer(self, url): @@ -91,7 +91,7 @@ class NoReferrerPolicy(ReferrerPolicy): """ name = POLICY_NO_REFERRER - def referrer(self, response, request): + def referrer(self, response_url, request_url): return None @@ -111,9 +111,9 @@ class NoReferrerWhenDowngradePolicy(ReferrerPolicy): """ name = POLICY_NO_REFERRER_WHEN_DOWNGRADE - def referrer(self, response, request): - if not self.tls_protected(response) or self.tls_protected(request): - return self.stripped_referrer(response) + def referrer(self, response_url, request_url): + if not self.tls_protected(response_url) or self.tls_protected(request_url): + return self.stripped_referrer(response_url) class SameOriginPolicy(ReferrerPolicy): @@ -128,9 +128,9 @@ class SameOriginPolicy(ReferrerPolicy): """ name = POLICY_SAME_ORIGIN - def referrer(self, response, request): - if self.origin(response) == self.origin(request): - return self.stripped_referrer(response) + def referrer(self, response_url, request_url): + if self.origin(response_url) == self.origin(request_url): + return self.stripped_referrer(response_url) class OriginPolicy(ReferrerPolicy): @@ -144,8 +144,8 @@ class OriginPolicy(ReferrerPolicy): """ name = POLICY_ORIGIN - def referrer(self, response, request): - return self.origin_referrer(response) + def referrer(self, response_url, request_url): + return self.origin_referrer(response_url) class StrictOriginPolicy(ReferrerPolicy): @@ -163,11 +163,11 @@ class StrictOriginPolicy(ReferrerPolicy): """ name = POLICY_STRICT_ORIGIN - def referrer(self, response, request): - if ((self.tls_protected(response) and - self.potentially_trustworthy(request)) - or not self.tls_protected(response)): - return self.origin_referrer(response) + def referrer(self, response_url, request_url): + if ((self.tls_protected(response_url) and + self.potentially_trustworthy(request_url)) + or not self.tls_protected(response_url)): + return self.origin_referrer(response_url) class OriginWhenCrossOriginPolicy(ReferrerPolicy): @@ -183,10 +183,10 @@ class OriginWhenCrossOriginPolicy(ReferrerPolicy): """ name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response, request): - origin = self.origin(response) - if origin == self.origin(request): - return self.stripped_referrer(response) + def referrer(self, response_url, request_url): + origin = self.origin(response_url) + if origin == self.origin(request_url): + return self.stripped_referrer(response_url) else: return origin @@ -210,14 +210,14 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy): """ name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN - def referrer(self, response, request): - origin = self.origin(response) - if origin == self.origin(request): - return self.stripped_referrer(response) - elif ((self.tls_protected(response) and - self.potentially_trustworthy(request)) - or not self.tls_protected(response)): - return self.origin_referrer(response) + def referrer(self, response_url, request_url): + origin = self.origin(response_url) + if origin == self.origin(request_url): + return self.stripped_referrer(response_url) + elif ((self.tls_protected(response_url) and + self.potentially_trustworthy(request_url)) + or not self.tls_protected(response_url)): + return self.origin_referrer(response_url) class UnsafeUrlPolicy(ReferrerPolicy): @@ -235,8 +235,8 @@ class UnsafeUrlPolicy(ReferrerPolicy): """ name = POLICY_UNSAFE_URL - def referrer(self, response, request): - return self.stripped_referrer(response) + def referrer(self, response_url, request_url): + return self.stripped_referrer(response_url) class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy): From 42b429dc37e17a3a24d9f5f6d9365b6d78479630 Mon Sep 17 00:00:00 2001 From: jorenham Date: Fri, 3 Mar 2017 15:15:59 +0100 Subject: [PATCH 071/362] Log full cache file path instead of cache directory for the storages that cache to single files. --- scrapy/extensions/httpcache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 8025efe77..fe8c55c6b 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -220,12 +220,12 @@ class DbmCacheStorage(object): self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE']) self.db = None - logger.debug("Using DBM cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) - def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) self.db = self.dbmodule.open(dbpath, 'c') + logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) + def close_spider(self, spider): self.db.close() @@ -352,12 +352,12 @@ class LeveldbCacheStorage(object): self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') self.db = None - logger.debug("Using LevelDB cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) - def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name) self.db = self._leveldb.LevelDB(dbpath) + logger.debug("Using LevelDB cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) + def close_spider(self, spider): # Do compactation each time to save space and also recreate files to # avoid them being removed in storages with timestamp-based autoremoval. From 5e89db548419c4114ff71bd0504155c9a83868fa Mon Sep 17 00:00:00 2001 From: jorenham Date: Fri, 3 Mar 2017 15:32:20 +0100 Subject: [PATCH 072/362] Moved cache dir logging to `open_spider` in FilesystemCacheStorage for consistency --- scrapy/extensions/httpcache.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index fe8c55c6b..2fb4b6a15 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -277,10 +277,9 @@ class FilesystemCacheStorage(object): self.use_gzip = settings.getbool('HTTPCACHE_GZIP') self._open = gzip.open if self.use_gzip else open - logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) - def open_spider(self, spider): - pass + logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}, + extra={'spider': spider}) def close_spider(self, spider): pass From 5d0058492c07d3f89f46ccfd2b8e33849f8bd30e Mon Sep 17 00:00:00 2001 From: Bernardas Date: Tue, 23 Aug 2016 16:54:10 +0000 Subject: [PATCH 073/362] add media pipeline settings to enable redirection and handling of certain http statuses --- scrapy/pipelines/files.py | 2 +- scrapy/pipelines/media.py | 34 +++++++++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 843b4d3ec..4ae7e1d89 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -249,7 +249,7 @@ class FilesPipeline(MediaPipeline): resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD ) - super(FilesPipeline, self).__init__(download_func=download_func) + super(FilesPipeline, self).__init__(download_func=download_func, settings=settings) @classmethod def from_settings(cls, settings): diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 57f70499e..4177d294f 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,5 +1,6 @@ from __future__ import print_function +import functools import logging from collections import defaultdict from twisted.internet.defer import Deferred, DeferredList @@ -16,6 +17,7 @@ logger = logging.getLogger(__name__) class MediaPipeline(object): LOG_FAILED_RESULTS = True + ALLOW_REDIRECTS = False class SpiderInfo(object): def __init__(self, spider): @@ -24,9 +26,16 @@ class MediaPipeline(object): self.downloaded = {} self.waiting = defaultdict(list) - def __init__(self, download_func=None): + def __init__(self, download_func=None, settings=None): self.download_func = download_func - + resolve = functools.partial(self._key_for_pipe, + base_class_name="MediaPipeline") + self.allow_redirects = settings.getbool( + resolve('MEDIA_ALLOW_REDIRECTS'), self.ALLOW_REDIRECTS + ) + self.allow_httpstatus_list = settings.getlist( + resolve('MEDIA_HTTPSTATUS_LIST'), [] + ) def _key_for_pipe(self, key, base_class_name=None, settings=None): @@ -93,6 +102,25 @@ class MediaPipeline(object): ) return dfd.addBoth(lambda _: wad) # it must return wad at last + def _modify_media_request(self, request): + httpstatus_list = [] + if self.allow_httpstatus_list: + httpstatus_list = self.allow_httpstatus_list + elif self.allow_redirects: + if not httpstatus_list: + httpstatus_list = list(range(0, 300)) + list(range(400, 1000)) + else: + for i in range(300, 400): + try: + httpstatus_list.remove(i) + except ValueError: + pass + if httpstatus_list: + request.meta['handle_httpstatus_list'] = httpstatus_list + else: + request.meta['handle_httpstatus_all'] = True + return request + def _check_media_to_download(self, result, request, info): if result is not None: return result @@ -103,7 +131,7 @@ class MediaPipeline(object): callback=self.media_downloaded, callbackArgs=(request, info), errback=self.media_failed, errbackArgs=(request, info)) else: - request.meta['handle_httpstatus_all'] = True + request = self._modify_media_request(request) dfd = self.crawler.engine.download(request, info.spider) dfd.addCallbacks( callback=self.media_downloaded, callbackArgs=(request, info), From 25ed491219924b5fc98a67f64d355249a1b68f50 Mon Sep 17 00:00:00 2001 From: Bernardas Date: Tue, 23 Aug 2016 16:55:34 +0000 Subject: [PATCH 074/362] add description for media pipeline MEDIA_ALLOW_REDIRECTS and MEDIA_HTTPSTATUS_LIST settings --- docs/topics/media-pipeline.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 82c0aaa88..a86bab4bf 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -322,6 +322,22 @@ By default, there are no size constraints, so all images are processed. .. _topics-media-pipeline-override: +Allowing redirection and handling various http statuses +------------------------------------------------------- + +.. setting:: MEDIA_ALLOW_REDIRECTS +.. setting:: MEDIA_HTTPSTATUS_LIST + +By default media pipelines ignore redirects. To allow redirecting(all 300 codes) set: + + MEDIA_ALLOW_REDIRECTS = True + +To only allow specific codes through set: + + MEDIA_HTTpSTATUS_LIST = + # example: + MEDIA_HTTPSTATUS_LIST = [303, 404] + Extending the Media Pipelines ============================= From 6a4221471610b7f3a5d44d93306294c21d550406 Mon Sep 17 00:00:00 2001 From: Bernardas Date: Tue, 23 Aug 2016 16:56:31 +0000 Subject: [PATCH 075/362] add tests for media pipeline MEDIA_ALLOW_REDIRECTS and MEDIA_HTTPSTATUS_LIST settings --- tests/test_pipeline_media.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index f30b4fea3..41ee9962f 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -6,6 +6,7 @@ from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks from scrapy.http import Request, Response +from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.request import request_fingerprint from scrapy.pipelines.media import MediaPipeline @@ -25,7 +26,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase): def setUp(self): self.spider = Spider('media.com') - self.pipe = self.pipeline_class(download_func=_mocked_download_func) + self.pipe = self.pipeline_class(download_func=_mocked_download_func, + settings=Settings()) self.pipe.open_spider(self.spider) self.info = self.pipe.spiderinfo @@ -82,6 +84,21 @@ class BaseMediaPipelineTestCase(unittest.TestCase): new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item + def test_modify_media_request(self): + request = Request('http://url') + assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True} + + request = Request('http://url') + self.pipe.allow_httpstatus_list = list(range(100)) + assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_list': list(range(100))} + self.pipe.allow_httpstatus_list = None + + request = Request('http://url') + self.pipe.allow_redirects = True + correct = {'handle_httpstatus_list': list(range(300)) + list(range(400,1000))} + assert self.pipe._modify_media_request(request).meta == correct + self.pipe.allow_redirects = False + class MockedMediaPipeline(MediaPipeline): From 854278085494053361a8cf070237683557b642ae Mon Sep 17 00:00:00 2001 From: Bernardas Date: Tue, 23 Aug 2016 17:09:43 +0000 Subject: [PATCH 076/362] typo and clarify handling --- docs/topics/media-pipeline.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index a86bab4bf..bfc405d9e 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -332,11 +332,11 @@ By default media pipelines ignore redirects. To allow redirecting(all 300 codes) MEDIA_ALLOW_REDIRECTS = True -To only allow specific codes through set: +To only allow handling only specific codes set (default: any code): - MEDIA_HTTpSTATUS_LIST = + MEDIA_HTTPSTATUS_LIST = # example: - MEDIA_HTTPSTATUS_LIST = [303, 404] + MEDIA_HTTPSTATUS_LIST = [303, 404] # will not go through pipelines Extending the Media Pipelines ============================= From 2e052c86150502c7441c8f92cd1488de6232ec7f Mon Sep 17 00:00:00 2001 From: Bernardas Date: Tue, 23 Aug 2016 17:13:09 +0000 Subject: [PATCH 077/362] fix error when settings are not provided --- scrapy/pipelines/media.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 4177d294f..976b5032b 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -6,6 +6,7 @@ from collections import defaultdict from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure +from scrapy.settings import Settings from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter @@ -28,6 +29,8 @@ class MediaPipeline(object): def __init__(self, download_func=None, settings=None): self.download_func = download_func + if isinstance(settings, dict) or settings is None: + settings = Settings(settings) resolve = functools.partial(self._key_for_pipe, base_class_name="MediaPipeline") self.allow_redirects = settings.getbool( From f0b4077f812619640df078a2485f8a460fafa58a Mon Sep 17 00:00:00 2001 From: Bernardas Date: Wed, 24 Aug 2016 08:39:30 +0000 Subject: [PATCH 078/362] expose allowed_status tuple for media pipeline --- scrapy/downloadermiddlewares/redirect.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 26677e527..ae4ad8891 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -57,6 +57,8 @@ class RedirectMiddleware(BaseRedirectMiddleware): Handle redirection of requests based on response status and meta-refresh html tag. """ + allowed_status = (301, 302, 303, 307) + def process_response(self, request, response, spider): if (request.meta.get('dont_redirect', False) or response.status in getattr(spider, 'handle_httpstatus_list', []) or @@ -64,8 +66,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): request.meta.get('handle_httpstatus_all', False)): return response - allowed_status = (301, 302, 303, 307) - if 'Location' not in response.headers or response.status not in allowed_status: + if 'Location' not in response.headers or response.status not in self.allowed_status: return response location = safe_url_string(response.headers['location']) From 3cef1cd451c8f28df8075aa4e3b65cedf5ecedfa Mon Sep 17 00:00:00 2001 From: Bernardas Date: Wed, 24 Aug 2016 08:40:46 +0000 Subject: [PATCH 079/362] adjust variable wording and redirect logic --- scrapy/pipelines/media.py | 19 +++++++++---------- tests/test_pipeline_media.py | 4 ++-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 976b5032b..b11e7095b 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -6,6 +6,7 @@ from collections import defaultdict from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.settings import Settings from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy.utils.request import request_fingerprint @@ -36,7 +37,7 @@ class MediaPipeline(object): self.allow_redirects = settings.getbool( resolve('MEDIA_ALLOW_REDIRECTS'), self.ALLOW_REDIRECTS ) - self.allow_httpstatus_list = settings.getlist( + self.handle_httpstatus_list = settings.getlist( resolve('MEDIA_HTTPSTATUS_LIST'), [] ) @@ -107,17 +108,15 @@ class MediaPipeline(object): def _modify_media_request(self, request): httpstatus_list = [] - if self.allow_httpstatus_list: - httpstatus_list = self.allow_httpstatus_list - elif self.allow_redirects: + if self.handle_httpstatus_list: + httpstatus_list = self.handle_httpstatus_list + if self.allow_redirects: if not httpstatus_list: - httpstatus_list = list(range(0, 300)) + list(range(400, 1000)) + httpstatus_list = [i for i in range(1000) + if i not in RedirectMiddleware.allowed_status] else: - for i in range(300, 400): - try: - httpstatus_list.remove(i) - except ValueError: - pass + httpstatus_list = [i for i in httpstatus_list + if i not in RedirectMiddleware.allowed_status] if httpstatus_list: request.meta['handle_httpstatus_list'] = httpstatus_list else: diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 41ee9962f..66e98db29 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -89,9 +89,9 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True} request = Request('http://url') - self.pipe.allow_httpstatus_list = list(range(100)) + self.pipe.handle_httpstatus_list = list(range(100)) assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_list': list(range(100))} - self.pipe.allow_httpstatus_list = None + self.pipe.handle_httpstatus_list = None request = Request('http://url') self.pipe.allow_redirects = True From 11b31c9fbddd75fe7fec60152a1bec57acccc039 Mon Sep 17 00:00:00 2001 From: Bernardas Date: Wed, 24 Aug 2016 08:44:56 +0000 Subject: [PATCH 080/362] fix redirect change --- tests/test_pipeline_media.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 66e98db29..f1b8076fd 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -5,6 +5,7 @@ from twisted.python.failure import Failure from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks +from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider @@ -95,7 +96,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase): request = Request('http://url') self.pipe.allow_redirects = True - correct = {'handle_httpstatus_list': list(range(300)) + list(range(400,1000))} + correct = {'handle_httpstatus_list': [i for i in range(1000) + if i not in RedirectMiddleware.allowed_status]} assert self.pipe._modify_media_request(request).meta == correct self.pipe.allow_redirects = False From c64ebee06253ef385564c83806bd2422bff6f6d6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Mar 2017 22:40:10 +0100 Subject: [PATCH 081/362] Refactor (WIP) --- scrapy/pipelines/media.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index b11e7095b..0712101b0 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -19,7 +19,6 @@ logger = logging.getLogger(__name__) class MediaPipeline(object): LOG_FAILED_RESULTS = True - ALLOW_REDIRECTS = False class SpiderInfo(object): def __init__(self, spider): @@ -35,12 +34,23 @@ class MediaPipeline(object): resolve = functools.partial(self._key_for_pipe, base_class_name="MediaPipeline") self.allow_redirects = settings.getbool( - resolve('MEDIA_ALLOW_REDIRECTS'), self.ALLOW_REDIRECTS + resolve('MEDIA_ALLOW_REDIRECTS'), False ) self.handle_httpstatus_list = settings.getlist( resolve('MEDIA_HTTPSTATUS_LIST'), [] ) + self.httpstatus_list = [] + if self.handle_httpstatus_list: + self.httpstatus_list = self.handle_httpstatus_list + if self.allow_redirects: + if not self.httpstatus_list: + self.httpstatus_list = [i for i in range(1000) + if i not in RedirectMiddleware.allowed_status] + else: + self.httpstatus_list = [i for i in self.httpstatus_list + if i not in RedirectMiddleware.allowed_status] + def _key_for_pipe(self, key, base_class_name=None, settings=None): """ @@ -107,18 +117,8 @@ class MediaPipeline(object): return dfd.addBoth(lambda _: wad) # it must return wad at last def _modify_media_request(self, request): - httpstatus_list = [] - if self.handle_httpstatus_list: - httpstatus_list = self.handle_httpstatus_list - if self.allow_redirects: - if not httpstatus_list: - httpstatus_list = [i for i in range(1000) - if i not in RedirectMiddleware.allowed_status] - else: - httpstatus_list = [i for i in httpstatus_list - if i not in RedirectMiddleware.allowed_status] - if httpstatus_list: - request.meta['handle_httpstatus_list'] = httpstatus_list + if self.httpstatus_list: + request.meta['handle_httpstatus_list'] = self.httpstatus_list else: request.meta['handle_httpstatus_all'] = True return request From 72fbb687d7a35ddf52f82b165df0a842d0b653e0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 3 Mar 2017 12:31:05 +0100 Subject: [PATCH 082/362] Revert "expose allowed_status tuple for media pipeline" This reverts commit 052809c73ed20b9a728a8fd7df3de5f45f2dad8d. --- scrapy/downloadermiddlewares/redirect.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index ae4ad8891..26677e527 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -57,8 +57,6 @@ class RedirectMiddleware(BaseRedirectMiddleware): Handle redirection of requests based on response status and meta-refresh html tag. """ - allowed_status = (301, 302, 303, 307) - def process_response(self, request, response, spider): if (request.meta.get('dont_redirect', False) or response.status in getattr(spider, 'handle_httpstatus_list', []) or @@ -66,7 +64,8 @@ class RedirectMiddleware(BaseRedirectMiddleware): request.meta.get('handle_httpstatus_all', False)): return response - if 'Location' not in response.headers or response.status not in self.allowed_status: + allowed_status = (301, 302, 303, 307) + if 'Location' not in response.headers or response.status not in allowed_status: return response location = safe_url_string(response.headers['location']) From ecde166ee18935456bcf4c6fce89ab909bbffbaf Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 3 Mar 2017 15:50:34 +0100 Subject: [PATCH 083/362] Refactor without MEDIA_HTTPSTATUS_LIST setting --- docs/topics/media-pipeline.rst | 15 +++++-------- scrapy/pipelines/media.py | 25 ++++++++------------- tests/test_pipeline_media.py | 41 +++++++++++++++++++++++++--------- 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index bfc405d9e..f258ff748 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -322,21 +322,18 @@ By default, there are no size constraints, so all images are processed. .. _topics-media-pipeline-override: -Allowing redirection and handling various http statuses -------------------------------------------------------- +Allowing redirections +--------------------- .. setting:: MEDIA_ALLOW_REDIRECTS -.. setting:: MEDIA_HTTPSTATUS_LIST -By default media pipelines ignore redirects. To allow redirecting(all 300 codes) set: +By default media pipelines ignore redirects, i.e. an HTTP redirection +to a media file URL request will mean the media download is considered failed. + +To handle media redirections, set this settings to ``True``: MEDIA_ALLOW_REDIRECTS = True -To only allow handling only specific codes set (default: any code): - - MEDIA_HTTPSTATUS_LIST = - # example: - MEDIA_HTTPSTATUS_LIST = [303, 404] # will not go through pipelines Extending the Media Pipelines ============================= diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 0712101b0..02daf8d2c 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -6,8 +6,8 @@ from collections import defaultdict from twisted.internet.defer import Deferred, DeferredList from twisted.python.failure import Failure -from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.settings import Settings +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter @@ -29,6 +29,7 @@ class MediaPipeline(object): def __init__(self, download_func=None, settings=None): self.download_func = download_func + if isinstance(settings, dict) or settings is None: settings = Settings(settings) resolve = functools.partial(self._key_for_pipe, @@ -36,20 +37,12 @@ class MediaPipeline(object): self.allow_redirects = settings.getbool( resolve('MEDIA_ALLOW_REDIRECTS'), False ) - self.handle_httpstatus_list = settings.getlist( - resolve('MEDIA_HTTPSTATUS_LIST'), [] - ) + self._handle_statuses(self.allow_redirects) - self.httpstatus_list = [] - if self.handle_httpstatus_list: - self.httpstatus_list = self.handle_httpstatus_list - if self.allow_redirects: - if not self.httpstatus_list: - self.httpstatus_list = [i for i in range(1000) - if i not in RedirectMiddleware.allowed_status] - else: - self.httpstatus_list = [i for i in self.httpstatus_list - if i not in RedirectMiddleware.allowed_status] + def _handle_statuses(self, allow_redirects): + self.handle_httpstatus_list = None + if allow_redirects: + self.handle_httpstatus_list = SequenceExclude(range(300, 400)) def _key_for_pipe(self, key, base_class_name=None, settings=None): @@ -117,8 +110,8 @@ class MediaPipeline(object): return dfd.addBoth(lambda _: wad) # it must return wad at last def _modify_media_request(self, request): - if self.httpstatus_list: - request.meta['handle_httpstatus_list'] = self.httpstatus_list + if self.handle_httpstatus_list: + request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list else: request.meta['handle_httpstatus_all'] = True return request diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index f1b8076fd..4797956a0 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -24,11 +24,12 @@ def _mocked_download_func(request, info): class BaseMediaPipelineTestCase(unittest.TestCase): pipeline_class = MediaPipeline + settings = None def setUp(self): self.spider = Spider('media.com') self.pipe = self.pipeline_class(download_func=_mocked_download_func, - settings=Settings()) + settings=Settings(self.settings)) self.pipe.open_spider(self.spider) self.info = self.pipe.spiderinfo @@ -89,17 +90,37 @@ class BaseMediaPipelineTestCase(unittest.TestCase): request = Request('http://url') assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True} - request = Request('http://url') - self.pipe.handle_httpstatus_list = list(range(100)) - assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_list': list(range(100))} - self.pipe.handle_httpstatus_list = None +class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase): + + pipeline_class = MediaPipeline + settings = { + 'MEDIA_ALLOW_REDIRECTS': True + } + + def test_modify_media_request(self): request = Request('http://url') - self.pipe.allow_redirects = True - correct = {'handle_httpstatus_list': [i for i in range(1000) - if i not in RedirectMiddleware.allowed_status]} - assert self.pipe._modify_media_request(request).meta == correct - self.pipe.allow_redirects = False + meta = self.pipe._modify_media_request(request).meta + self.assertIn('handle_httpstatus_list', meta) + for status, check in [ + (200, True), + + # These are the status codes we want + # the downloader to handle itself + (301, False), + (302, False), + (302, False), + (307, False), + (308, False), + + # we still want to get 4xx and 5xx + (400, True), + (404, True), + (500, True)]: + if check: + self.assertIn(status, meta['handle_httpstatus_list']) + else: + self.assertNotIn(status, meta['handle_httpstatus_list']) class MockedMediaPipeline(MediaPipeline): From f7e11b198efd0213bb51205b6829123476ccf2ba Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 3 Mar 2017 16:00:59 +0100 Subject: [PATCH 084/362] Cleanup --- scrapy/pipelines/media.py | 3 +-- tests/test_pipeline_media.py | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 02daf8d2c..921e9e1c9 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -114,7 +114,6 @@ class MediaPipeline(object): request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list else: request.meta['handle_httpstatus_all'] = True - return request def _check_media_to_download(self, result, request, info): if result is not None: @@ -126,7 +125,7 @@ class MediaPipeline(object): callback=self.media_downloaded, callbackArgs=(request, info), errback=self.media_failed, errbackArgs=(request, info)) else: - request = self._modify_media_request(request) + self._modify_media_request(request) dfd = self.crawler.engine.download(request, info.spider) dfd.addCallbacks( callback=self.media_downloaded, callbackArgs=(request, info), diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 4797956a0..cfa2fc42b 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -5,7 +5,6 @@ from twisted.python.failure import Failure from twisted.internet import reactor from twisted.internet.defer import Deferred, inlineCallbacks -from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider @@ -88,7 +87,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase): def test_modify_media_request(self): request = Request('http://url') - assert self.pipe._modify_media_request(request).meta == {'handle_httpstatus_all': True} + self.pipe._modify_media_request(request) + assert request.meta == {'handle_httpstatus_all': True} class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase): @@ -100,8 +100,8 @@ class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase): def test_modify_media_request(self): request = Request('http://url') - meta = self.pipe._modify_media_request(request).meta - self.assertIn('handle_httpstatus_list', meta) + self.pipe._modify_media_request(request) + self.assertIn('handle_httpstatus_list', request.meta) for status, check in [ (200, True), @@ -118,9 +118,9 @@ class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase): (404, True), (500, True)]: if check: - self.assertIn(status, meta['handle_httpstatus_list']) + self.assertIn(status, request.meta['handle_httpstatus_list']) else: - self.assertNotIn(status, meta['handle_httpstatus_list']) + self.assertNotIn(status, request.meta['handle_httpstatus_list']) class MockedMediaPipeline(MediaPipeline): From ef04cfd237ec3d072a487f92e217bad68195f2d8 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 21 Feb 2017 19:55:52 +0300 Subject: [PATCH 085/362] Respect log settings in custom_settings: fixes GH-1612 A new root logger is installed when a crawler is created if one was already installed before. This allows to respect custom settings related to logging, such as LOG_LEVEL, LOG_FILE, etc. --- scrapy/crawler.py | 9 +++++++-- scrapy/utils/log.py | 22 ++++++++++++++++++--- tests/test_crawl.py | 4 ++-- tests/test_crawler.py | 46 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 7 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 443a9aa2f..7b8518832 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -16,7 +16,9 @@ from scrapy.signalmanager import SignalManager from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object -from scrapy.utils.log import LogCounterHandler, configure_logging, log_scrapy_info +from scrapy.utils.log import ( + LogCounterHandler, configure_logging, log_scrapy_info, + get_scrapy_root_handler, install_scrapy_root_handler) from scrapy import signals logger = logging.getLogger(__name__) @@ -35,8 +37,11 @@ class Crawler(object): self.signals = SignalManager(self) self.stats = load_object(self.settings['STATS_CLASS'])(self) - handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL')) + handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL')) logging.root.addHandler(handler) + if get_scrapy_root_handler() is not None: + # scrapy root handler alread installed: update it with new settings + install_scrapy_root_handler(self.settings) # lambda is assigned to Crawler attribute because this way it is not # garbage collected after leaving __init__ scope self.__remove_handler = lambda: logging.root.removeHandler(handler) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index f33ce7017..6ceb61a82 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -96,9 +96,25 @@ def configure_logging(settings=None, install_root_handler=True): sys.stdout = StreamLogger(logging.getLogger('stdout')) if install_root_handler: - logging.root.setLevel(logging.NOTSET) - handler = _get_handler(settings) - logging.root.addHandler(handler) + install_scrapy_root_handler(settings) + + +def install_scrapy_root_handler(settings): + global _scrapy_root_handler + + if (_scrapy_root_handler is not None + and _scrapy_root_handler in logging.root.handlers): + logging.root.removeHandler(_scrapy_root_handler) + logging.root.setLevel(logging.NOTSET) + _scrapy_root_handler = _get_handler(settings) + logging.root.addHandler(_scrapy_root_handler) + + +def get_scrapy_root_handler(): + return _scrapy_root_handler + + +_scrapy_root_handler = None def _get_handler(settings): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d5babdded..3c5d9b958 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -97,8 +97,8 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): + crawler = self.runner.create_crawler(BrokenStartRequestsSpider) with LogCapture('scrapy', level=logging.ERROR) as l: - crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_before_yield=1) self.assertEqual(len(l.records), 1) @@ -108,8 +108,8 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_yielding(self): + crawler = self.runner.create_crawler(BrokenStartRequestsSpider) with LogCapture('scrapy', level=logging.ERROR) as l: - crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_yielding=1) self.assertEqual(len(l.records), 1) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 53a1202e3..ba0d709ff 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,3 +1,6 @@ +import logging +import os +import tempfile import warnings import unittest @@ -5,6 +8,7 @@ import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader +from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object from scrapy.extensions.throttle import AutoThrottle @@ -74,6 +78,48 @@ class SpiderSettingsTestCase(unittest.TestCase): self.assertIn(AutoThrottle, enabled_exts) +class CrawlerLoggingTestCase(unittest.TestCase): + def test_no_root_handler_installed(self): + handler = get_scrapy_root_handler() + if handler is not None: + logging.root.removeHandler(handler) + + class MySpider(scrapy.Spider): + name = 'spider' + + crawler = Crawler(MySpider, {}) + assert get_scrapy_root_handler() is None + + def test_spider_custom_settings_log_level(self): + with tempfile.NamedTemporaryFile() as log_file: + class MySpider(scrapy.Spider): + name = 'spider' + custom_settings = { + 'LOG_LEVEL': 'INFO', + 'LOG_FILE': log_file.name, + } + + configure_logging() + self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) + crawler = Crawler(MySpider, {}) + self.assertEqual(get_scrapy_root_handler().level, logging.INFO) + info_count = crawler.stats.get_value('log_count/INFO') + logging.debug('debug message') + logging.info('info message') + logging.warning('warning message') + logging.error('error message') + logged = log_file.read().decode('utf8') + self.assertNotIn('debug message', logged) + self.assertIn('info message', logged) + self.assertIn('warning message', logged) + self.assertIn('error message', logged) + self.assertEqual(crawler.stats.get_value('log_count/ERROR'), 1) + self.assertEqual(crawler.stats.get_value('log_count/WARNING'), 1) + self.assertEqual( + crawler.stats.get_value('log_count/INFO') - info_count, 1) + self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0) + + class SpiderLoaderWithWrongInterface(object): def unneeded_method(self): From c3b6feca0e15309386c080d71b0b03a07d4c8635 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 3 Mar 2017 16:29:07 +0100 Subject: [PATCH 086/362] Fix setting lookup for MEDIA_ALLOWED_REDIRECTS --- scrapy/pipelines/media.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 921e9e1c9..404bbf5bf 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -33,7 +33,8 @@ class MediaPipeline(object): if isinstance(settings, dict) or settings is None: settings = Settings(settings) resolve = functools.partial(self._key_for_pipe, - base_class_name="MediaPipeline") + base_class_name="MediaPipeline", + settings=settings) self.allow_redirects = settings.getbool( resolve('MEDIA_ALLOW_REDIRECTS'), False ) From c68f99eed843bd35224d8bf0e22b0c66bdc2122b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 3 Mar 2017 17:03:25 +0100 Subject: [PATCH 087/362] Refactor settings tests --- tests/test_pipeline_media.py | 90 +++++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index cfa2fc42b..5f6a6d9e6 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -91,38 +91,6 @@ class BaseMediaPipelineTestCase(unittest.TestCase): assert request.meta == {'handle_httpstatus_all': True} -class MediaPipelineAllowRedirectsTestCase(BaseMediaPipelineTestCase): - - pipeline_class = MediaPipeline - settings = { - 'MEDIA_ALLOW_REDIRECTS': True - } - - def test_modify_media_request(self): - request = Request('http://url') - self.pipe._modify_media_request(request) - self.assertIn('handle_httpstatus_list', request.meta) - for status, check in [ - (200, True), - - # These are the status codes we want - # the downloader to handle itself - (301, False), - (302, False), - (302, False), - (307, False), - (308, False), - - # we still want to get 4xx and 5xx - (400, True), - (404, True), - (500, True)]: - if check: - self.assertIn(status, request.meta['handle_httpstatus_list']) - else: - self.assertNotIn(status, request.meta['handle_httpstatus_list']) - - class MockedMediaPipeline(MediaPipeline): def __init__(self, *args, **kwargs): @@ -289,3 +257,61 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): self.assertEqual(new_item['results'], [(True, 'ITSME')]) self.assertEqual(self.pipe._mockcalled, \ ['get_media_requests', 'media_to_download', 'item_completed']) + + +class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase): + + def _assert_request_no3xx(self, pipeline_class, settings): + pipe = pipeline_class(settings=Settings(settings)) + request = Request('http://url') + pipe._modify_media_request(request) + + self.assertIn('handle_httpstatus_list', request.meta) + for status, check in [ + (200, True), + + # These are the status codes we want + # the downloader to handle itself + (301, False), + (302, False), + (302, False), + (307, False), + (308, False), + + # we still want to get 4xx and 5xx + (400, True), + (404, True), + (500, True)]: + if check: + self.assertIn(status, request.meta['handle_httpstatus_list']) + else: + self.assertNotIn(status, request.meta['handle_httpstatus_list']) + + def test_standard_setting(self): + self._assert_request_no3xx( + MediaPipeline, + { + 'MEDIA_ALLOW_REDIRECTS': True + }) + + def test_subclass_standard_setting(self): + + class UserDefinedPipeline(MediaPipeline): + pass + + self._assert_request_no3xx( + UserDefinedPipeline, + { + 'MEDIA_ALLOW_REDIRECTS': True + }) + + def test_subclass_specific_setting(self): + + class UserDefinedPipeline(MediaPipeline): + pass + + self._assert_request_no3xx( + UserDefinedPipeline, + { + 'USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS': True + }) From 30d812eea233914b3aead55b8053a9c804e594d3 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 3 Mar 2017 17:15:37 +0100 Subject: [PATCH 088/362] Remove redundant slot.add_request() call in ExecutionEngine --- scrapy/core/engine.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 2b5770138..37fe0a873 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -218,10 +218,8 @@ class ExecutionEngine(object): request=request, spider=spider) def download(self, request, spider): - slot = self.slot - slot.add_request(request) d = self._download(request, spider) - d.addBoth(self._downloaded, slot, request, spider) + d.addBoth(self._downloaded, self.slot, request, spider) return d def _downloaded(self, response, slot, request, spider): From a8b47d6c689cd8f221acb4caf3974ff0057b6a2d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 6 Mar 2017 14:25:52 +0100 Subject: [PATCH 089/362] Update release notes for 1.0.7, 1.1.4 and 1.2.3 --- docs/news.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index ff1e4ce03..31f4d3026 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -101,6 +101,12 @@ Dependencies & Cleanups downloader middlewares. +Scrapy 1.2.3 (2017-03-03) +------------------------- + +- Packaging fix: disallow unsupported Twisted versions in setup.py + + Scrapy 1.2.2 (2016-12-06) ------------------------- @@ -229,6 +235,12 @@ Documentation - Add StackOverflow as a support channel (:issue:`2257`). +Scrapy 1.1.4 (2017-03-03) +------------------------- + +- Packaging fix: disallow unsupported Twisted versions in setup.py + + Scrapy 1.1.3 (2016-09-22) ------------------------- @@ -501,6 +513,12 @@ Bugfixes to same remote host (:issue:`1912`). +Scrapy 1.0.7 (2017-03-03) +------------------------- + +- Packaging fix: disallow unsupported Twisted versions in setup.py + + Scrapy 1.0.6 (2016-05-04) ------------------------- From 768f3155e57ed54419f0c137fc066a95197d1b46 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 6 Mar 2017 16:20:37 +0100 Subject: [PATCH 090/362] Fix referrer policy from response headers and support explicit empty string --- scrapy/spidermiddlewares/referer.py | 8 ++++++-- tests/test_spidermiddleware_referer.py | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index b444e34bb..1ddfb37f4 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -261,6 +261,9 @@ _policy_classes = {p.name: p for p in ( DefaultReferrerPolicy, )} +# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string +_policy_classes[''] = NoReferrerWhenDowngradePolicy + def _load_policy_class(policy, warning_only=False): """ @@ -317,8 +320,9 @@ class RefererMiddleware(object): policy_name = request.meta.get('referrer_policy') if policy_name is None: if isinstance(resp_or_url, Response): - policy_name = to_native_str( - resp_or_url.headers.get('Referrer-Policy', '').decode('latin1')) + policy_header = resp_or_url.headers.get('Referrer-Policy') + if policy_header is not None: + policy_name = to_native_str(policy_header.decode('latin1')) if policy_name is None: return self.default_policy() diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index b1c815876..f27f31b74 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -526,6 +526,13 @@ class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMid settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} +class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): + """ + The empty string means "no-referrer-when-downgrade" + """ + settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} + resp_headers = {'Referrer-Policy': ''} + class TestReferrerOnRedirect(TestRefererMiddleware): From 2a7d391e0b379143810fdcaed241664e159e0d1d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 6 Mar 2017 17:30:32 +0100 Subject: [PATCH 091/362] DOC Mention brotli support in HttpCompressionMiddleware section --- docs/topics/downloader-middleware.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0ef3fb071..c3a454279 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -645,6 +645,12 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. + This middleware also supports decoding `brotli-compressed`_ responses, + provided `brotlipy`_ is installed. + +.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt +.. _brotlipy: https://pypi.python.org/pypi/brotlipy + HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From e42b846a9fc68a624e94052aac2ba44224b05cad Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 10 Nov 2016 16:26:55 +0100 Subject: [PATCH 092/362] Use body to chose response type after decompression content --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- .../test_downloadermiddleware_httpcompression.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 19d6345e4..eb00d8923 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -38,7 +38,7 @@ class HttpCompressionMiddleware(object): encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) respcls = responsetypes.from_args(headers=response.headers, \ - url=response.url) + url=response.url, body=decoded_body) kwargs = dict(cls=respcls, body=decoded_body) if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 7924fb3b5..5403e8f52 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -7,6 +7,7 @@ from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \ ACCEPTED_ENCODINGS +from scrapy.responsetypes import responsetypes from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -152,6 +153,20 @@ class HttpCompressionTest(TestCase): self.assertEqual(newresponse.body, plainbody) self.assertEqual(newresponse.encoding, resolve_encoding('gb2312')) + def test_process_response_no_content_type_header(self): + headers = { + 'Content-Encoding': 'identity', + } + plainbody = b"""Some page""" + respcls = responsetypes.from_args(url="http://www.example.com/index", headers=headers, body=plainbody) + response = respcls("http://www.example.com/index", headers=headers, body=plainbody) + request = Request("http://www.example.com/index") + + newresponse = self.mw.process_response(request, response, self.spider) + assert isinstance(newresponse, respcls) + self.assertEqual(newresponse.body, plainbody) + self.assertEqual(newresponse.encoding, resolve_encoding('gb2312')) + def test_process_response_gzipped_contenttype(self): response = self._getresponse('gzip') response.headers['Content-Type'] = 'application/gzip' From 11cdf58abe26a9f65d8c1aefa717b6735e5734c9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 9 Nov 2016 23:08:23 +0100 Subject: [PATCH 093/362] Always decompress Content-Encoding: gzip at HttpCompression stage Let SitemapSpider handle decoding of .xml.gz files if necessary --- .../downloadermiddlewares/httpcompression.py | 2 +- scrapy/spiders/sitemap.py | 23 +++++-- scrapy/utils/gz.py | 4 ++ ...st_downloadermiddleware_httpcompression.py | 61 ++++++++++++++++--- tests/test_spider.py | 4 ++ 5 files changed, 78 insertions(+), 16 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index eb00d8923..dd32c62de 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -34,7 +34,7 @@ class HttpCompressionMiddleware(object): return response if isinstance(response, Response): content_encoding = response.headers.getlist('Content-Encoding') - if content_encoding and not is_gzipped(response): + if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) respcls = responsetypes.from_args(headers=response.headers, \ diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 9e45637c3..10af90259 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -5,7 +5,8 @@ import six from scrapy.spiders import Spider from scrapy.http import Request, XmlResponse from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots -from scrapy.utils.gz import gunzip, is_gzipped +from scrapy.utils.gz import gunzip, gzip_magic_number + logger = logging.getLogger(__name__) @@ -59,12 +60,22 @@ class SitemapSpider(Spider): """ if isinstance(response, XmlResponse): return response.body - elif is_gzipped(response): - return gunzip(response.body) - elif response.url.endswith('.xml'): + elif gzip_magic_number(response): + try: + return gunzip(response.body) + except: + pass + # actual gzipped sitemap files are decompressed above ; + # if we are here (response body is not gzipped) + # and have a response for .xml.gz, + # it usually means that it was already gunzipped + # by HttpCompression middleware, + # the HTTP response being sent with "Content-Encoding: gzip" + # without actually being a .xml.gz file in the first place, + # merely XML gzip-compressed on the fly, + # in other word, here, we have plain XML + elif response.url.endswith('.xml') or response.url.endswith('.xml.gz'): return response.body - elif response.url.endswith('.xml.gz'): - return gunzip(response.body) def regex(x): diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 73c2eb73b..22cf58986 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -59,3 +59,7 @@ def is_gzipped(response): cenc = response.headers.get('Content-Encoding', b'').lower() return (_is_gzipped(ctype) or (_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip'))) + + +def gzip_magic_number(response): + return response.body[:2] == b'\x1f\x8b' diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 5403e8f52..5f56c99ec 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -8,6 +8,7 @@ from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \ ACCEPTED_ENCODINGS from scrapy.responsetypes import responsetypes +from scrapy.utils.gz import gunzip from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -173,9 +174,9 @@ class HttpCompressionTest(TestCase): request = response.request newresponse = self.mw.process_response(request, response, self.spider) - self.assertIs(newresponse, response) - self.assertEqual(response.headers['Content-Encoding'], b'gzip') - self.assertEqual(response.headers['Content-Type'], b'application/gzip') + assert newresponse is not response + assert newresponse.body.startswith(b' + + + http://www.example.com/ + 2009-08-16 + daily + 1 + + + http://www.example.com/Special-Offers.html + 2009-08-16 + weekly + 0.8 + +""" + gz_file = GzipFile(fileobj=f, mode='wb') + gz_file.write(plainbody) + gz_file.close() + + # build a gzipped response body containing this gzipped file + r = BytesIO() + gz_resp = GzipFile(fileobj=r, mode='wb') + gz_resp.write(f.getvalue()) + gz_resp.close() + + response = Response("http;//www.example.com/", headers=headers, body=r.getvalue()) + request = Request("http://www.example.com/") + + newresponse = self.mw.process_response(request, response, self.spider) + self.assertEqual(gunzip(newresponse.body), plainbody) def test_process_response_head_request_no_decode_required(self): response = self._getresponse('gzip') diff --git a/tests/test_spider.py b/tests/test_spider.py index 371b8c1ac..e55f0fa6d 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -328,6 +328,10 @@ class SitemapSpiderTest(SpiderTest): r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY) self.assertSitemapBody(r, self.BODY) + # .xml.gz but body decoded by HttpCompression middleware already + r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.BODY) + self.assertSitemapBody(r, self.BODY) + def test_get_sitemap_urls_from_robotstxt(self): robots = b"""# Sitemap files Sitemap: http://example.com/sitemap.xml From b174744b80fd12efc6e4008ea5eb38256f21038a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 10 Nov 2016 10:50:34 +0100 Subject: [PATCH 094/362] Do not silently fail on gzip unzipping --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- scrapy/spiders/sitemap.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index dd32c62de..203dee42d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,6 @@ import zlib -from scrapy.utils.gz import gunzip, is_gzipped +from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes from scrapy.exceptions import NotConfigured diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 10af90259..e54001d88 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -61,10 +61,7 @@ class SitemapSpider(Spider): if isinstance(response, XmlResponse): return response.body elif gzip_magic_number(response): - try: - return gunzip(response.body) - except: - pass + return gunzip(response.body) # actual gzipped sitemap files are decompressed above ; # if we are here (response body is not gzipped) # and have a response for .xml.gz, From 4caceccd594f2563a3d32ea708579ce8e70132f3 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 7 Mar 2017 10:51:34 +0100 Subject: [PATCH 095/362] Use 3-bytes for gzip archive type sniffing --- scrapy/utils/gz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 22cf58986..16c9ce539 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -62,4 +62,4 @@ def is_gzipped(response): def gzip_magic_number(response): - return response.body[:2] == b'\x1f\x8b' + return response.body[:3] == b'\x1f\x8b\x08' From 6f55ca4643dfff7a163714d696eef66d06cb81a8 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Tue, 7 Mar 2017 14:20:52 +0300 Subject: [PATCH 096/362] Revert unneeded test_crawl changes --- tests/test_crawl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3c5d9b958..d5babdded 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -97,8 +97,8 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): - crawler = self.runner.create_crawler(BrokenStartRequestsSpider) with LogCapture('scrapy', level=logging.ERROR) as l: + crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_before_yield=1) self.assertEqual(len(l.records), 1) @@ -108,8 +108,8 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_start_requests_bug_yielding(self): - crawler = self.runner.create_crawler(BrokenStartRequestsSpider) with LogCapture('scrapy', level=logging.ERROR) as l: + crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_yielding=1) self.assertEqual(len(l.records), 1) From b6378c7ef6393412165239fd6bf489d45a1c5196 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 7 Mar 2017 12:28:24 +0100 Subject: [PATCH 097/362] Revert to using self.assert methods --- ...est_downloadermiddleware_httpcompression.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 5f56c99ec..0678fcb14 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -174,9 +174,9 @@ class HttpCompressionTest(TestCase): request = response.request newresponse = self.mw.process_response(request, response, self.spider) - assert newresponse is not response - assert newresponse.body.startswith(b' Date: Tue, 7 Mar 2017 14:44:27 +0100 Subject: [PATCH 098/362] Warn about modules where duplicate spider names were found --- scrapy/spiderloader.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 1322c01d1..27a909c9f 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +from collections import defaultdict import traceback import warnings @@ -19,14 +20,21 @@ class SpiderLoader(object): def __init__(self, settings): self.spider_modules = settings.getlist('SPIDER_MODULES') self._spiders = {} + self._found = defaultdict(list) self._load_all_spiders() def _load_spiders(self, module): for spcls in iter_spider_classes(module): + self._found[spcls.name].append((module.__name__, spcls.__name__)) if spcls.name in self._spiders.keys(): import warnings - warnings.warn("There are several spiders with the same name (" + spcls.name + - "), this can cause unexpected behavior", UserWarning) + msg = ("There are several spiders with the same name {!r}:\n" + "{}\n This can cause unexpected behavior.".format( + spcls.name, + "\n".join( + " {1} (in {0})".format(mod, cls) + for (mod, cls) in self._found[spcls.name]))) + warnings.warn(msg, UserWarning) self._spiders[spcls.name] = spcls def _load_all_spiders(self): From 978306a2236403f6275bad11e85d1e42ca37d6f1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 7 Mar 2017 14:48:16 +0100 Subject: [PATCH 099/362] Fix dupe spider name warning string tests --- tests/test_spiderloader/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 302bf9a10..4600e53dc 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -127,7 +127,7 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): spider_loader = SpiderLoader.from_settings(self.settings) self.assertEqual(len(w), 1) - self.assertIn("several spiders with the same name (spider3)", str(w[0].message)) + self.assertIn("several spiders with the same name 'spider3'", str(w[0].message)) spiders = set(spider_loader.list()) self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) @@ -145,8 +145,8 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): self.assertEqual(len(w), 2) msgs = sorted(str(wrn.message) for wrn in w) - self.assertIn("several spiders with the same name (spider1)", msgs[0]) - self.assertIn("several spiders with the same name (spider2)", msgs[1]) + self.assertIn("several spiders with the same name 'spider1'", msgs[0]) + self.assertIn("several spiders with the same name 'spider2'", msgs[1]) spiders = set(spider_loader.list()) self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) From 0b9a18e1a120751b8a2b2f1cc04c62b97967d612 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 7 Mar 2017 15:41:17 +0100 Subject: [PATCH 100/362] Warn only once for all spiders --- scrapy/spiderloader.py | 22 +++++++++++++--------- tests/test_spiderloader/__init__.py | 13 ++++++++----- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 27a909c9f..486a4637e 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -23,18 +23,21 @@ class SpiderLoader(object): self._found = defaultdict(list) self._load_all_spiders() + def _check_name_duplicates(self): + dupes = ["\n".join(" {cls} named {name!r} (in {module})".format( + module=mod, cls=cls, name=name) + for (mod, cls) in locations) + for name, locations in self._found.items() + if len(locations)>1] + if dupes: + msg = ("There are several spiders with the same name:\n\n" + "{}\n\n This can cause unexpected behavior.".format( + "\n\n".join(dupes))) + warnings.warn(msg, UserWarning) + def _load_spiders(self, module): for spcls in iter_spider_classes(module): self._found[spcls.name].append((module.__name__, spcls.__name__)) - if spcls.name in self._spiders.keys(): - import warnings - msg = ("There are several spiders with the same name {!r}:\n" - "{}\n This can cause unexpected behavior.".format( - spcls.name, - "\n".join( - " {1} (in {0})".format(mod, cls) - for (mod, cls) in self._found[spcls.name]))) - warnings.warn(msg, UserWarning) self._spiders[spcls.name] = spcls def _load_all_spiders(self): @@ -47,6 +50,7 @@ class SpiderLoader(object): "Check SPIDER_MODULES setting".format( modname=name, tb=traceback.format_exc())) warnings.warn(msg, RuntimeWarning) + self._check_name_duplicates() @classmethod def from_settings(cls, settings): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 4600e53dc..673a2d302 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -127,7 +127,9 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): spider_loader = SpiderLoader.from_settings(self.settings) self.assertEqual(len(w), 1) - self.assertIn("several spiders with the same name 'spider3'", str(w[0].message)) + msg = str(w[0].message) + self.assertIn("several spiders with the same name", msg) + self.assertIn("'spider3'", msg) spiders = set(spider_loader.list()) self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) @@ -143,10 +145,11 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): with warnings.catch_warnings(record=True) as w: spider_loader = SpiderLoader.from_settings(self.settings) - self.assertEqual(len(w), 2) - msgs = sorted(str(wrn.message) for wrn in w) - self.assertIn("several spiders with the same name 'spider1'", msgs[0]) - self.assertIn("several spiders with the same name 'spider2'", msgs[1]) + self.assertEqual(len(w), 1) + msg = str(w[0].message) + self.assertIn("several spiders with the same name", msg) + self.assertIn("'spider1'", msg) + self.assertIn("'spider2'", msg) spiders = set(spider_loader.list()) self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) From 7e9153b38d7ef70f1f19a82506b669433d134b01 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 19 Dec 2016 10:43:04 -0300 Subject: [PATCH 101/362] Feed exports: beautify JSON and XML --- docs/topics/exporters.rst | 9 ++- docs/topics/feed-exports.rst | 14 ++++ scrapy/exporters.py | 30 ++++++-- scrapy/extensions/feedexport.py | 3 +- scrapy/settings/default_settings.py | 1 + tests/test_feedexport.py | 113 ++++++++++++++++++++++++++-- 6 files changed, 156 insertions(+), 14 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 85c73222d..4114eda58 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -140,7 +140,7 @@ output examples, which assume you're exporting these two items:: BaseItemExporter ---------------- -.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8') +.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent_width=None) This is the (abstract) base class for all Item Exporters. It provides support for common features used by all (concrete) Item Exporters, such as @@ -149,7 +149,7 @@ BaseItemExporter These features can be configured through the constructor arguments which populate their respective instance attributes: :attr:`fields_to_export`, - :attr:`export_empty_fields`, :attr:`encoding`. + :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent_width`. .. method:: export_item(item) @@ -216,6 +216,11 @@ BaseItemExporter encoding). Other value types are passed unchanged to the specific serialization library. + .. attribute:: indent_width + + Amount of spaces used to indent the output on each level. + Defaults to ``None``, which disables indentation. + .. highlight:: none XmlItemExporter diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index efdd8c46b..ce3b5fd75 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -209,6 +209,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORE_EMPTY` * :setting:`FEED_EXPORT_ENCODING` * :setting:`FEED_EXPORT_FIELDS` + * :setting:`FEED_EXPORT_INDENT_WIDTH` .. currentmodule:: scrapy.extensions.feedexport @@ -266,6 +267,19 @@ If an exporter requires a fixed set of fields (this is the case for is empty or None, then Scrapy tries to infer field names from the exported data - currently it uses field names from the first item. +.. setting:: FEED_EXPORT_INDENT_WIDTH + +FEED_EXPORT_INDENT_WIDTH +------------------------ + +Default: ``None`` + +Amount of spaces to indent on each level. +Set to `None` to disable indentation. + +Currently used by :class:`~scrapy.exporters.JsonItemExporter` +and :class:`~scrapy.exporters.XmlItemExporter` + .. setting:: FEED_STORE_EMPTY FEED_STORE_EMPTY diff --git a/scrapy/exporters.py b/scrapy/exporters.py index c4b1b3476..69e6c15e0 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -36,6 +36,7 @@ class BaseItemExporter(object): self.encoding = options.pop('encoding', None) self.fields_to_export = options.pop('fields_to_export', None) self.export_empty_fields = options.pop('export_empty_fields', False) + self.indent_width = options.pop('indent_width', None) if not dont_fail and options: raise TypeError("Unexpected options: %s" % ', '.join(options.keys())) @@ -99,7 +100,7 @@ class JsonItemExporter(BaseItemExporter): self._configure(kwargs, dont_fail=True) self.file = file kwargs.setdefault('ensure_ascii', not self.encoding) - self.encoder = ScrapyJSONEncoder(**kwargs) + self.encoder = ScrapyJSONEncoder(indent=self.indent_width, **kwargs) self.first_item = True def start_exporting(self): @@ -128,33 +129,52 @@ class XmlItemExporter(BaseItemExporter): self.encoding = 'utf-8' self.xg = XMLGenerator(file, encoding=self.encoding) + def _beautify_newline(self): + if self.indent_width: + self._xg_characters('\n') + + def _beautify_indent(self, depth=1): + if self.indent_width: + self._xg_characters(' ' * self.indent_width * depth) + def start_exporting(self): self.xg.startDocument() self.xg.startElement(self.root_element, {}) + self._beautify_newline() def export_item(self, item): + self._beautify_indent(depth=1) self.xg.startElement(self.item_element, {}) + self._beautify_newline() for name, value in self._get_serialized_fields(item, default_value=''): - self._export_xml_field(name, value) + self._export_xml_field(name, value, depth=2) + self._beautify_indent(depth=1) self.xg.endElement(self.item_element) + self._beautify_newline() def finish_exporting(self): self.xg.endElement(self.root_element) self.xg.endDocument() - def _export_xml_field(self, name, serialized_value): + def _export_xml_field(self, name, serialized_value, depth): + self._beautify_indent(depth=depth) self.xg.startElement(name, {}) if hasattr(serialized_value, 'items'): + self._beautify_newline() for subname, value in serialized_value.items(): - self._export_xml_field(subname, value) + self._export_xml_field(subname, value, depth=depth+1) + self._beautify_indent(depth=depth) elif is_listlike(serialized_value): + self._beautify_newline() for value in serialized_value: - self._export_xml_field('value', value) + self._export_xml_field('value', value, depth=depth+1) + self._beautify_indent(depth=depth) elif isinstance(serialized_value, six.text_type): self._xg_characters(serialized_value) else: self._xg_characters(str(serialized_value)) self.xg.endElement(name) + self._beautify_newline() # Workaround for http://bugs.python.org/issue17606 # Before Python 2.7.4 xml.sax.saxutils required bytes; diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 85d328528..26024e5e9 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -172,6 +172,7 @@ class FeedExporter(object): self.store_empty = settings.getbool('FEED_STORE_EMPTY') self._exporting = False self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None + self.indent_width = settings.getint('FEED_EXPORT_INDENT_WIDTH') or None uripar = settings['FEED_URI_PARAMS'] self._uripar = load_object(uripar) if uripar else lambda x, y: None @@ -188,7 +189,7 @@ class FeedExporter(object): storage = self._get_storage(uri) file = storage.open(spider) exporter = self._get_exporter(file, fields_to_export=self.export_fields, - encoding=self.export_encoding) + encoding=self.export_encoding, indent_width=self.indent_width) if self.store_empty: exporter.start_exporting() self._exporting = True diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d73c595d2..cca0d3889 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -161,6 +161,7 @@ FEED_EXPORTERS_BASE = { 'marshal': 'scrapy.exporters.MarshalItemExporter', 'pickle': 'scrapy.exporters.PickleItemExporter', } +FEED_EXPORT_INDENT_WIDTH = None FILES_STORE_S3_ACL = 'private' diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2d137edf4..bf002bec7 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -431,10 +431,10 @@ class FeedExportTest(unittest.TestCase): 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } - for format in formats: - settings = {'FEED_FORMAT': format} + for format, expected in formats.items(): + settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT_WIDTH': None} data = yield self.exported_data(items, settings) - self.assertEqual(formats[format], data) + self.assertEqual(expected, data) formats = { 'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'), @@ -443,7 +443,108 @@ class FeedExportTest(unittest.TestCase): 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } - for format in formats: - settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'} + settings = {'FEED_EXPORT_INDENT_WIDTH': None, 'FEED_EXPORT_ENCODING': 'latin-1'} + for format, expected in formats.items(): + settings['FEED_FORMAT'] = format data = yield self.exported_data(items, settings) - self.assertEqual(formats[format], data) + self.assertEqual(expected, data) + + @defer.inlineCallbacks + def test_export_indentation(self): + items = [dict({'foo': ['bar']})] + + output = [ + # JSON + { + 'format': 'json', + 'indent_width': None, + 'expected': b'[\n{"foo": ["bar"]}\n]', + }, + { + 'format': 'json', + 'indent_width': 2, + 'expected': b""" +[ +{ + "foo": [ + "bar" + ] +} +]""", + }, + { + 'format': 'json', + 'indent_width': 4, + 'expected': b""" +[ +{ + "foo": [ + "bar" + ] +} +]""", + }, + { + 'format': 'json', + 'indent_width': 5, + 'expected': b""" +[ +{ + "foo": [ + "bar" + ] +} +]""", + }, + + # XML + { + 'format': 'xml', + 'indent_width': None, + 'expected': b'\nbar', + }, + { + 'format': 'xml', + 'indent_width': 2, + 'expected': b""" + + + + + bar + + +""", + }, + { + 'format': 'xml', + 'indent_width': 4, + 'expected': b""" + + + + + bar + + +""", + }, + { + 'format': 'xml', + 'indent_width': 5, + 'expected': b""" + + + + + bar + + +""", + }, + ] + + for row in output: + settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT_WIDTH': row['indent_width']} + data = yield self.exported_data(items, settings) + self.assertEqual(row['expected'].strip(), data) From 766b2c84539d58ee871e1f301df1ad0ae0d44079 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 23 Feb 2017 10:21:33 -0300 Subject: [PATCH 102/362] Feed exports: enforce difference between None and 0 on indent Also rename params and settings from "indent_width" to just "indent" --- docs/topics/exporters.rst | 14 ++-- docs/topics/feed-exports.rst | 14 ++-- scrapy/exporters.py | 24 ++++-- scrapy/extensions/feedexport.py | 6 +- scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 116 ++++++++++++++++++++++++---- 6 files changed, 137 insertions(+), 39 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 4114eda58..ad559fb35 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -140,7 +140,7 @@ output examples, which assume you're exporting these two items:: BaseItemExporter ---------------- -.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent_width=None) +.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=None) This is the (abstract) base class for all Item Exporters. It provides support for common features used by all (concrete) Item Exporters, such as @@ -149,7 +149,7 @@ BaseItemExporter These features can be configured through the constructor arguments which populate their respective instance attributes: :attr:`fields_to_export`, - :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent_width`. + :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`. .. method:: export_item(item) @@ -216,10 +216,14 @@ BaseItemExporter encoding). Other value types are passed unchanged to the specific serialization library. - .. attribute:: indent_width + .. attribute:: indent - Amount of spaces used to indent the output on each level. - Defaults to ``None``, which disables indentation. + Amount of spaces used to indent the output on each level. Defaults to ``None``, + which disables indentation. This argument behaves like ``indent`` in python's + JSON module (both for JSON and XML exporters): "If ``indent`` is a non-negative + integer, then array elements and object members will be pretty-printed with that + indent level. An indent level of 0, or negative, will only insert newlines. + ``None`` (the default) selects the most compact representation" .. highlight:: none diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index ce3b5fd75..afaa972e5 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -209,7 +209,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_STORE_EMPTY` * :setting:`FEED_EXPORT_ENCODING` * :setting:`FEED_EXPORT_FIELDS` - * :setting:`FEED_EXPORT_INDENT_WIDTH` + * :setting:`FEED_EXPORT_INDENT` .. currentmodule:: scrapy.extensions.feedexport @@ -267,15 +267,17 @@ If an exporter requires a fixed set of fields (this is the case for is empty or None, then Scrapy tries to infer field names from the exported data - currently it uses field names from the first item. -.. setting:: FEED_EXPORT_INDENT_WIDTH +.. setting:: FEED_EXPORT_INDENT -FEED_EXPORT_INDENT_WIDTH ------------------------- +FEED_EXPORT_INDENT +------------------ Default: ``None`` -Amount of spaces to indent on each level. -Set to `None` to disable indentation. +Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT`` +is a non-negative integer, then array elements and object members will be pretty-printed +with that indent level. An indent level of 0, or negative, will only insert newlines. +``None`` (the default) selects the most compact representation Currently used by :class:`~scrapy.exporters.JsonItemExporter` and :class:`~scrapy.exporters.XmlItemExporter` diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 69e6c15e0..1dfa2af85 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -36,7 +36,7 @@ class BaseItemExporter(object): self.encoding = options.pop('encoding', None) self.fields_to_export = options.pop('fields_to_export', None) self.export_empty_fields = options.pop('export_empty_fields', False) - self.indent_width = options.pop('indent_width', None) + self.indent = options.pop('indent', None) if not dont_fail and options: raise TypeError("Unexpected options: %s" % ', '.join(options.keys())) @@ -100,20 +100,28 @@ class JsonItemExporter(BaseItemExporter): self._configure(kwargs, dont_fail=True) self.file = file kwargs.setdefault('ensure_ascii', not self.encoding) - self.encoder = ScrapyJSONEncoder(indent=self.indent_width, **kwargs) + kwargs.setdefault('indent', self.indent) + self.encoder = ScrapyJSONEncoder(**kwargs) self.first_item = True + def _beautify_newline(self): + if self.indent is not None: + self.file.write(b'\n') + def start_exporting(self): - self.file.write(b"[\n") + self.file.write(b"[") + self._beautify_newline() def finish_exporting(self): - self.file.write(b"\n]") + self._beautify_newline() + self.file.write(b"]") def export_item(self, item): if self.first_item: self.first_item = False else: - self.file.write(b',\n') + self.file.write(b',') + self._beautify_newline() itemdict = dict(self._get_serialized_fields(item)) data = self.encoder.encode(itemdict) self.file.write(to_bytes(data, self.encoding)) @@ -130,12 +138,12 @@ class XmlItemExporter(BaseItemExporter): self.xg = XMLGenerator(file, encoding=self.encoding) def _beautify_newline(self): - if self.indent_width: + if self.indent is not None: self._xg_characters('\n') def _beautify_indent(self, depth=1): - if self.indent_width: - self._xg_characters(' ' * self.indent_width * depth) + if self.indent: + self._xg_characters(' ' * self.indent * depth) def start_exporting(self): self.xg.startDocument() diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 26024e5e9..5f133fbde 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -172,7 +172,9 @@ class FeedExporter(object): self.store_empty = settings.getbool('FEED_STORE_EMPTY') self._exporting = False self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None - self.indent_width = settings.getint('FEED_EXPORT_INDENT_WIDTH') or None + self.indent = None + if settings.get('FEED_EXPORT_INDENT') is not None: + self.indent = settings.getint('FEED_EXPORT_INDENT') uripar = settings['FEED_URI_PARAMS'] self._uripar = load_object(uripar) if uripar else lambda x, y: None @@ -189,7 +191,7 @@ class FeedExporter(object): storage = self._get_storage(uri) file = storage.open(spider) exporter = self._get_exporter(file, fields_to_export=self.export_fields, - encoding=self.export_encoding, indent_width=self.indent_width) + encoding=self.export_encoding, indent=self.indent) if self.store_empty: exporter.start_exporting() self._exporting = True diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index cca0d3889..fc265e2ba 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -161,7 +161,7 @@ FEED_EXPORTERS_BASE = { 'marshal': 'scrapy.exporters.MarshalItemExporter', 'pickle': 'scrapy.exporters.PickleItemExporter', } -FEED_EXPORT_INDENT_WIDTH = None +FEED_EXPORT_INDENT = None FILES_STORE_S3_ACL = 'private' diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index bf002bec7..2b82bba0c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -319,7 +319,7 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def test_export_no_items_store_empty(self): formats = ( - ('json', b'[\n\n]'), + ('json', b'[]'), ('jsonlines', b''), ('xml', b'\n'), ('csv', b''), @@ -425,25 +425,25 @@ class FeedExportTest(unittest.TestCase): header = ['foo'] formats = { - 'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'), + 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'), 'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'), 'xml': u'\nTest\xd6'.encode('utf-8'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } for format, expected in formats.items(): - settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT_WIDTH': None} + settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT': None} data = yield self.exported_data(items, settings) self.assertEqual(expected, data) formats = { - 'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'), + 'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'), 'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'), 'xml': u'\nTest\xd6'.encode('latin-1'), 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } - settings = {'FEED_EXPORT_INDENT_WIDTH': None, 'FEED_EXPORT_ENCODING': 'latin-1'} + settings = {'FEED_EXPORT_INDENT': None, 'FEED_EXPORT_ENCODING': 'latin-1'} for format, expected in formats.items(): settings['FEED_FORMAT'] = format data = yield self.exported_data(items, settings) @@ -451,48 +451,89 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def test_export_indentation(self): - items = [dict({'foo': ['bar']})] + items = [dict({'foo': ['bar']}), dict({'key': 'value'})] output = [ # JSON { 'format': 'json', - 'indent_width': None, - 'expected': b'[\n{"foo": ["bar"]}\n]', + 'indent': None, + 'expected': b'[{"foo": ["bar"]},{"key": "value"}]', }, { 'format': 'json', - 'indent_width': 2, + 'indent': -1, + 'expected': b""" +[ +{ +"foo": [ +"bar" +] +}, +{ +"key": "value" +} +] +""", + }, + { + 'format': 'json', + 'indent': 0, + 'expected': b""" +[ +{ +"foo": [ +"bar" +] +}, +{ +"key": "value" +} +] +""", + }, + { + 'format': 'json', + 'indent': 2, 'expected': b""" [ { "foo": [ "bar" ] +}, +{ + "key": "value" } ]""", }, { 'format': 'json', - 'indent_width': 4, + 'indent': 4, 'expected': b""" [ { "foo": [ "bar" ] +}, +{ + "key": "value" } ]""", }, { 'format': 'json', - 'indent_width': 5, + 'indent': 5, 'expected': b""" [ { "foo": [ "bar" ] +}, +{ + "key": "value" } ]""", }, @@ -500,12 +541,44 @@ class FeedExportTest(unittest.TestCase): # XML { 'format': 'xml', - 'indent_width': None, - 'expected': b'\nbar', + 'indent': None, + 'expected': b'\nbarvalue', }, { 'format': 'xml', - 'indent_width': 2, + 'indent': -1, + 'expected': b""" + + + + +bar + + + +value + +""", + }, + { + 'format': 'xml', + 'indent': 0, + 'expected': b""" + + + + +bar + + + +value + +""", + }, + { + 'format': 'xml', + 'indent': 2, 'expected': b""" @@ -514,11 +587,14 @@ class FeedExportTest(unittest.TestCase): bar + + value + """, }, { 'format': 'xml', - 'indent_width': 4, + 'indent': 4, 'expected': b""" @@ -527,11 +603,14 @@ class FeedExportTest(unittest.TestCase): bar + + value + """, }, { 'format': 'xml', - 'indent_width': 5, + 'indent': 5, 'expected': b""" @@ -540,11 +619,14 @@ class FeedExportTest(unittest.TestCase): bar + + value + """, }, ] for row in output: - settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT_WIDTH': row['indent_width']} + settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']} data = yield self.exported_data(items, settings) self.assertEqual(row['expected'].strip(), data) From c7bb2fa8ce2633d92a7ec2840f84b174a5494428 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 7 Mar 2017 11:55:26 -0300 Subject: [PATCH 103/362] Feed exports: consistent and backwards compatible behaviour on indent --- docs/topics/exporters.rst | 14 ++++---- docs/topics/feed-exports.rst | 6 ++-- scrapy/exporters.py | 14 +++++--- scrapy/settings/default_settings.py | 2 +- tests/test_feedexport.py | 56 ++++++++++------------------- 5 files changed, 39 insertions(+), 53 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index ad559fb35..2ad77c905 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -140,7 +140,7 @@ output examples, which assume you're exporting these two items:: BaseItemExporter ---------------- -.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=None) +.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0) This is the (abstract) base class for all Item Exporters. It provides support for common features used by all (concrete) Item Exporters, such as @@ -218,12 +218,12 @@ BaseItemExporter .. attribute:: indent - Amount of spaces used to indent the output on each level. Defaults to ``None``, - which disables indentation. This argument behaves like ``indent`` in python's - JSON module (both for JSON and XML exporters): "If ``indent`` is a non-negative - integer, then array elements and object members will be pretty-printed with that - indent level. An indent level of 0, or negative, will only insert newlines. - ``None`` (the default) selects the most compact representation" + Amount of spaces used to indent the output on each level. Defaults to ``0``. + + * ``indent=None`` selects the most compact representation, + all items in the same line with no indentation + * ``indent<=0`` each item on it's own line, no indentation + * ``indent>0`` each item on it's own line, indentated with the provided numeric value .. highlight:: none diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index afaa972e5..e57a4e776 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -272,12 +272,12 @@ exported data - currently it uses field names from the first item. FEED_EXPORT_INDENT ------------------ -Default: ``None`` +Default: ``0`` Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT`` is a non-negative integer, then array elements and object members will be pretty-printed -with that indent level. An indent level of 0, or negative, will only insert newlines. -``None`` (the default) selects the most compact representation +with that indent level. An indent level of ``0``, or negative, will put each item on a new line. +``None`` selects the most compact representation Currently used by :class:`~scrapy.exporters.JsonItemExporter` and :class:`~scrapy.exporters.XmlItemExporter` diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 1dfa2af85..e2d42b6ab 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -99,8 +99,12 @@ class JsonItemExporter(BaseItemExporter): def __init__(self, file, **kwargs): self._configure(kwargs, dont_fail=True) self.file = file + # there is a small difference between the behaviour or JsonItemExporter.indent + # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent + # the addition of newlines everywhere + json_indent = self.indent if self.indent is not None and self.indent > 0 else None + kwargs.setdefault('indent', json_indent) kwargs.setdefault('ensure_ascii', not self.encoding) - kwargs.setdefault('indent', self.indent) self.encoder = ScrapyJSONEncoder(**kwargs) self.first_item = True @@ -137,8 +141,8 @@ class XmlItemExporter(BaseItemExporter): self.encoding = 'utf-8' self.xg = XMLGenerator(file, encoding=self.encoding) - def _beautify_newline(self): - if self.indent is not None: + def _beautify_newline(self, new_item=False): + if self.indent is not None and (self.indent > 0 or new_item): self._xg_characters('\n') def _beautify_indent(self, depth=1): @@ -148,7 +152,7 @@ class XmlItemExporter(BaseItemExporter): def start_exporting(self): self.xg.startDocument() self.xg.startElement(self.root_element, {}) - self._beautify_newline() + self._beautify_newline(new_item=True) def export_item(self, item): self._beautify_indent(depth=1) @@ -158,7 +162,7 @@ class XmlItemExporter(BaseItemExporter): self._export_xml_field(name, value, depth=2) self._beautify_indent(depth=1) self.xg.endElement(self.item_element) - self._beautify_newline() + self._beautify_newline(new_item=True) def finish_exporting(self): self.xg.endElement(self.root_element) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index fc265e2ba..bbc02cfdb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -161,7 +161,7 @@ FEED_EXPORTERS_BASE = { 'marshal': 'scrapy.exporters.MarshalItemExporter', 'pickle': 'scrapy.exporters.PickleItemExporter', } -FEED_EXPORT_INDENT = None +FEED_EXPORT_INDENT = 0 FILES_STORE_S3_ACL = 'private' diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2b82bba0c..c66c470a8 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -326,7 +326,7 @@ class FeedExportTest(unittest.TestCase): ) for fmt, expctd in formats: - settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True} + settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None} data = yield self.exported_no_data(settings) self.assertEqual(data, expctd) @@ -451,9 +451,12 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def test_export_indentation(self): - items = [dict({'foo': ['bar']}), dict({'key': 'value'})] + items = [ + {'foo': ['bar']}, + {'key': 'value'}, + ] - output = [ + test_cases = [ # JSON { 'format': 'json', @@ -465,14 +468,8 @@ class FeedExportTest(unittest.TestCase): 'indent': -1, 'expected': b""" [ -{ -"foo": [ -"bar" -] -}, -{ -"key": "value" -} +{"foo": ["bar"]}, +{"key": "value"} ] """, }, @@ -481,14 +478,8 @@ class FeedExportTest(unittest.TestCase): 'indent': 0, 'expected': b""" [ -{ -"foo": [ -"bar" -] -}, -{ -"key": "value" -} +{"foo": ["bar"]}, +{"key": "value"} ] """, }, @@ -542,7 +533,9 @@ class FeedExportTest(unittest.TestCase): { 'format': 'xml', 'indent': None, - 'expected': b'\nbarvalue', + 'expected': b""" + +barvalue""", }, { 'format': 'xml', @@ -550,14 +543,8 @@ class FeedExportTest(unittest.TestCase): 'expected': b""" - - -bar - - - -value - +bar +value """, }, { @@ -566,14 +553,8 @@ class FeedExportTest(unittest.TestCase): 'expected': b""" - - -bar - - - -value - +bar +value """, }, { @@ -626,7 +607,8 @@ class FeedExportTest(unittest.TestCase): }, ] - for row in output: + for row in test_cases: settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']} data = yield self.exported_data(items, settings) + print(row['format'], row['indent']) self.assertEqual(row['expected'].strip(), data) From 7be773e14ae27501914c3f3922f5504a7ac72f48 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 7 Mar 2017 17:40:40 +0100 Subject: [PATCH 104/362] Add SPIDER_LOADER_WARN_ONLY to toggle between spiderloader failure and warning --- scrapy/commands/list.py | 3 ++- scrapy/commands/runspider.py | 1 + scrapy/commands/settings.py | 3 ++- scrapy/commands/startproject.py | 5 +++-- scrapy/commands/version.py | 3 ++- scrapy/settings/default_settings.py | 1 + scrapy/spiderloader.py | 12 ++++++++---- tests/test_spiderloader/__init__.py | 11 +++++++++-- 8 files changed, 28 insertions(+), 11 deletions(-) diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index a255b3b94..185a77a40 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -4,7 +4,8 @@ from scrapy.commands import ScrapyCommand class Command(ScrapyCommand): requires_project = True - default_settings = {'LOG_ENABLED': False} + default_settings = {'LOG_ENABLED': False, + 'SPIDER_LOADER_WARN_ONLY': True} def short_desc(self): return "List available spiders" diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 1da09e4da..a98033dd1 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -28,6 +28,7 @@ def _import_file(filepath): class Command(ScrapyCommand): requires_project = False + default_settings = {'SPIDER_LOADER_WARN_ONLY': True} def syntax(self): return "[options] " diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index bce4e6086..bee52f06a 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -7,7 +7,8 @@ from scrapy.settings import BaseSettings class Command(ScrapyCommand): requires_project = False - default_settings = {'LOG_ENABLED': False} + default_settings = {'LOG_ENABLED': False, + 'SPIDER_LOADER_WARN_ONLY': True} def syntax(self): return "[options]" diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 594106632..c17aaf442 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -26,7 +26,8 @@ IGNORE = ignore_patterns('*.pyc', '.svn') class Command(ScrapyCommand): requires_project = False - default_settings = {'LOG_ENABLED': False} + default_settings = {'LOG_ENABLED': False, + 'SPIDER_LOADER_WARN_ONLY': True} def syntax(self): return " [project_dir]" @@ -118,4 +119,4 @@ class Command(ScrapyCommand): _templates_base_dir = self.settings['TEMPLATES_DIR'] or \ join(scrapy.__path__[0], 'templates') return join(_templates_base_dir, 'project') - + diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index a9954edb0..e22f98f5a 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -11,7 +11,8 @@ from scrapy.commands import ScrapyCommand class Command(ScrapyCommand): - default_settings = {'LOG_ENABLED': False} + default_settings = {'LOG_ENABLED': False, + 'SPIDER_LOADER_WARN_ONLY': True} def syntax(self): return "[-v]" diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d73c595d2..854cefc9c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -250,6 +250,7 @@ SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue' SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue' SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' +SPIDER_LOADER_WARN_ONLY = False SPIDER_MIDDLEWARES = {} diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 486a4637e..7478faa78 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -19,6 +19,7 @@ class SpiderLoader(object): """ def __init__(self, settings): self.spider_modules = settings.getlist('SPIDER_MODULES') + self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY') self._spiders = {} self._found = defaultdict(list) self._load_all_spiders() @@ -46,10 +47,13 @@ class SpiderLoader(object): for module in walk_modules(name): self._load_spiders(module) except ImportError as e: - msg = ("\n{tb}Could not load spiders from module '{modname}'. " - "Check SPIDER_MODULES setting".format( - modname=name, tb=traceback.format_exc())) - warnings.warn(msg, RuntimeWarning) + if self.warn_only: + msg = ("\n{tb}Could not load spiders from module '{modname}'. " + "See above traceback for details.".format( + modname=name, tb=traceback.format_exc())) + warnings.warn(msg, RuntimeWarning) + else: + raise self._check_name_duplicates() @classmethod diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 673a2d302..99a61daea 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -91,18 +91,25 @@ class SpiderLoaderTest(unittest.TestCase): self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) self.assertEqual(crawler.spidercls.name, 'spider1') + def test_bad_spider_modules_exception(self): + + module = 'tests.test_spiderloader.test_spiders.doesnotexist' + settings = Settings({'SPIDER_MODULES': [module]}) + with self.assertRaises(ImportError): + SpiderLoader.from_settings(settings) + def test_bad_spider_modules_warning(self): with warnings.catch_warnings(record=True) as w: module = 'tests.test_spiderloader.test_spiders.doesnotexist' - settings = Settings({'SPIDER_MODULES': [module]}) + settings = Settings({'SPIDER_MODULES': [module], + 'SPIDER_LOADER_WARN_ONLY': True}) spider_loader = SpiderLoader.from_settings(settings) self.assertIn("Could not load spiders from module", str(w[0].message)) spiders = spider_loader.list() self.assertEqual(spiders, []) - class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self): From ac63d3a3cf0a45b9ae66c9847d07d287154078e6 Mon Sep 17 00:00:00 2001 From: jorenham Date: Thu, 9 Mar 2017 10:56:23 +0100 Subject: [PATCH 105/362] Removed contrib section; contrib is deprecated --- docs/contributing.rst | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index b0a435ad2..ab3779395 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -124,15 +124,6 @@ Scrapy: * Don't put your name in the code you contribute. Our policy is to keep the contributor's name in the `AUTHORS`_ file distributed with Scrapy. -Scrapy Contrib -============== - -Scrapy contrib shares a similar rationale as Django contrib, which is explained -in `this post `_. If you -are working on a new functionality, please follow that rationale to decide -whether it should be a Scrapy contrib. If unsure, you can ask in -`scrapy-users`_. - Documentation policies ====================== From 9cfe9ae0989069b1a7a2ae0af916ffccb5a8bcee Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 9 Mar 2017 12:21:03 +0100 Subject: [PATCH 106/362] Do not use self.assertRaises() as context manager --- tests/test_spiderloader/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 99a61daea..1cd59b99a 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -95,8 +95,7 @@ class SpiderLoaderTest(unittest.TestCase): module = 'tests.test_spiderloader.test_spiders.doesnotexist' settings = Settings({'SPIDER_MODULES': [module]}) - with self.assertRaises(ImportError): - SpiderLoader.from_settings(settings) + self.assertRaises(ImportError, SpiderLoader.from_settings, settings) def test_bad_spider_modules_warning(self): From f2ac24eb7b353d1140729ba7d8c81ad9635d5899 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 9 Mar 2017 17:38:15 +0100 Subject: [PATCH 107/362] Do not only warn on wrong spider modules for "scrapy list" --- scrapy/commands/list.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 185a77a40..a255b3b94 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -4,8 +4,7 @@ from scrapy.commands import ScrapyCommand class Command(ScrapyCommand): requires_project = True - default_settings = {'LOG_ENABLED': False, - 'SPIDER_LOADER_WARN_ONLY': True} + default_settings = {'LOG_ENABLED': False} def short_desc(self): return "List available spiders" From 9628a739723983f2d3c4079d228dcaa79f558e73 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 9 Mar 2017 17:40:34 +0100 Subject: [PATCH 108/362] Update settings docs for new SPIDER_LOADER_WARN_ONLY --- docs/topics/settings.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ccdd02c4e..569b71518 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1180,6 +1180,29 @@ Default: ``'scrapy.spiderloader.SpiderLoader'`` The class that will be used for loading spiders, which must implement the :ref:`topics-api-spiderloader`. +.. setting:: SPIDER_LOADER_WARN_ONLY + +SPIDER_LOADER_WARN_ONLY +----------------------- + +.. versionadded:: 1.4 + +Default: ``False`` + +By default, when scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, +it will fail loudly if there is any ``ImportError`` exception. +But you can choose to silence this exception and turn it into a simple +warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. + +.. note:: + Some :ref:`scrapy commands ` run with this setting to ``True`` + already (i.e. they will only issue a warning and will not fail) + since they do not actually need to load spider classes to work: + :command:`scrapy runspider `, + :command:`scrapy settings `, + :command:`scrapy startproject `, + :command:`scrapy version `. + .. setting:: SPIDER_MIDDLEWARES SPIDER_MIDDLEWARES From d8865b33045c239e18704d7ab5012e334b128a32 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 10 Mar 2017 14:02:00 +0100 Subject: [PATCH 109/362] Update changelog for upcoming 1.3.3 --- docs/news.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 31f4d3026..f9a900771 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,13 +3,25 @@ Release notes ============= +Scrapy 1.3.3 (2017-03-XX) +------------------------- + +Bug fixes +~~~~~~~~~ + +- Make ``SpiderLoader`` raise ``ImportError`` again by default for missing + dependencies and wrong :setting:`SPIDER_MODULES`. + These exceptions were silenced as warnings since 1.3.0. + A new setting is introduced to toggle between warning or exception if needed ; + see :setting:`SPIDER_LOADER_WARN_ONLY` for details. + Scrapy 1.3.2 (2017-02-13) ------------------------- Bug fixes ~~~~~~~~~ -- Preserve crequest class when converting to/from dicts (utils.reqser) (:issue:`2510`). +- Preserve request class when converting to/from dicts (utils.reqser) (:issue:`2510`). - Use consistent selectors for author field in tutorial (:issue:`2551`). - Fix TLS compatibility in Twisted 17+ (:issue:`2558`) From b2c505d8ec71fd1226782c65edd06d901c3ef211 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 10 Mar 2017 16:29:44 +0100 Subject: [PATCH 110/362] Set release date in changelog for v1.3.3 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index f9a900771..da856d883 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,7 +3,7 @@ Release notes ============= -Scrapy 1.3.3 (2017-03-XX) +Scrapy 1.3.3 (2017-03-10) ------------------------- Bug fixes From a7f5207e9f2837040a3a08c3a8f4e76265b68bb2 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 10 Mar 2017 12:25:58 +0100 Subject: [PATCH 111/362] Update version added for SPIDER_LOADER_WARN_ONLY --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 569b71518..8367b1092 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1185,7 +1185,7 @@ The class that will be used for loading spiders, which must implement the SPIDER_LOADER_WARN_ONLY ----------------------- -.. versionadded:: 1.4 +.. versionadded:: 1.3.3 Default: ``False`` From 7dcc86e61adf37fdfb77b00375040fe25e7ad832 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 10 Mar 2017 21:35:25 +0100 Subject: [PATCH 112/362] Add file listing resource + redirecting resource to MockServer --- tests/mockserver.py | 16 ++++++++++++++++ .../python-logo-master-v3-TM-flattened.png | Bin 0 -> 11155 bytes .../files/images/python-powered-h-50x65.png | Bin 0 -> 3243 bytes .../test_site/files/images/scrapy.png | Bin 0 -> 2710 bytes 4 files changed, 16 insertions(+) create mode 100644 tests/sample_data/test_site/files/images/python-logo-master-v3-TM-flattened.png create mode 100644 tests/sample_data/test_site/files/images/python-powered-h-50x65.png create mode 100644 tests/sample_data/test_site/files/images/scrapy.png diff --git a/tests/mockserver.py b/tests/mockserver.py index e611cc3ec..26ab51183 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -5,13 +5,17 @@ from subprocess import Popen, PIPE from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource +from twisted.web.static import File from twisted.web.test.test_webclient import PayloadResource from twisted.web.server import GzipEncoderFactory from twisted.web.resource import EncodingResourceWrapper +from twisted.web.util import redirectTo from twisted.internet import reactor, ssl from twisted.internet.task import deferLater + from scrapy.utils.python import to_bytes, to_unicode +from tests import tests_datadir def getarg(request, name, default=None, type=None): @@ -120,6 +124,16 @@ class Echo(LeafResource): return to_bytes(json.dumps(output)) +class RedirectTo(LeafResource): + + def render(self, request): + goto = getarg(request, b'goto', b'/') + # we force the body content, otherwise Twisted redirectTo() + # returns HTML with )mfY@00XQM~#e_i5LqDi%dgZMIQ?b+Z22sAjAis zmi_Vf!5>^-B@F{Y@Cqe#NCdx$ywpv7v9O2_{=Fee<>J}kB;6}jlUK?0TYDc*cSnbpw(L({`Zxx7`ndYC zYn!l(3JZ%T_?Hi3VX(%->L@DUDIuIY)C`>CCbyk_-T<}6^)I*C_H@QqdIY1*i>j@X(n9|*#px% zxcYZq+^^6(T%6O_J8ke+2j`4c&-7?jhPr8u#C44H+_G1AD~D4&%N_=2YL#@cHE7OK zpyP7k(<5fmZeax#@G6*@+Q+A)qM{xhK7TKgZV~f=$6MSsIlVkN0Et% zT?Hs9z6@e$he33Oa=hAq-Vxh^)RKw0WgQ9I&5eHcPSa z2n^f&Ex<7Up(}Uv?o?(o_s;+zEAc6Yn)s0qzWkEcM?pE^Ycjuc%sa<=g3WEPM~DsL zn}#<-$ug0df`6)CExWk39F2J*f0a+R3HXsLNCZo>w(frpJPX)YT+hAxu2mj9E-srs zraMl*E0(s{ZkRHR1#PyJa5f2b1tA06p-rs&qqFs6jc?mAIMdd&0JN=s3MJZ%5=r(#P zs14&N0d&CYtM&Z9aW6V<`Ij+I!6=n~qpBjRrm0uy?4AV7#U~|%6MPumooWuE&J=uj zaN;Ypa-!9&9F!I57(sTB`<~WINrn{1`g&6L@L+$hYxAx-PBH=C6l;zcYD}nnodc#+Y2{I zi)?l+dgfhM8&mAb$=Qc~xX>~+%&)USwNpvbLsPiGFb*CRM-hh(E9dHv#6&8h=Te)R zJEsx_<_x?#?5&2uM)ub!Rb9H{$=q*;hKBrha3Y(U9{)#^d$W1Z3^_N)vcA49Ge9bJ z>cHLCl!9D(NQ!aZvqDBXe(YOJEuk~o_oj1J3Q0Y#XBl|JmKW9zgtxdd?NRvkI}-KG zFyoPXv7eKhTe*9SgL7y z8+TP=+-z+jMfypQ6!kRdKpGd($6~+M;!-~1I3+eSTEX~ouER#{-+#v(3;6higzmLQ zI!gYBUM9SLCj%e2xzS)lUYh(;m)UlgdScT|YU%0uWDb7s^7oT(CS>t-a>FMueeX+# zIZY6vEWcWwCSr(9SY4e3+B4^f-l(Yj`D7tTU}&gUg*3l3FhsX*3u`dcr1#mpkk2`s9_OmVhs%LJ~HlzRG={VIOP~j&< zF9x=8(crl#Cv!X~h815%PqH)-N3_BU8~ky!Bb0Rj-Y|57Hp4u}^57Wvt9aI^NZMba z0{bguPoSKIenRj2v780vg17%XV37-L92Ov^P&WF@klFfoVc=DfH;xli7K^p3TwrYe z2zF*jwmkxbi18>K06-^jV^O|XI&LGa!L`90WaP~M+7nK z3KxW(l2EzSwY9YZlZMV@SyQ6F%&;qL9dpK#BS<`Ti_C(T3jr8Z#FH1Q?;OFG;R8ka zhw2MEa3>*~4RCxuu~e~&Y()0JKp5&y;^0qeCX@^#G0CiNotu3EoNXwOd2tm)YxV3j z(lIczeo`Bpv;Zo(D#q5o1l~VNkzNR&oQO631~ek_-)t!yDLFxC8zLc5 zzA5u31+Ya2zcit^Nxv}OJ*_AGdSw$c9`klgjEsz>d8(Gk>ekk|>WH)mHd3*8=+W}S z&V}WA4{SOyBldbTGH^Hz=c-m1#Z;n3{BAGhi^pH~)sU*{r46iI92MyQx55{c@`-C} zT^TB)l4KD$G1mp|3m;X->n1xGC~LJiwfC;AIXZvL{I-aP>c{V1 zsz32=_SwbP=?G;_*$}kfQu?w>C;6mTtb{}AvcWhtZTPESx6$O$RUA^tuIQ$CpebjY zj>zP9@9pN|?fd%qEFCPQ^~T(-i{wF1-!(k z`-`a-g7lhU9)mW9|81MIRw-f*z@rpc2!>=bva$6}O=Wvpj*pMOEx7kIxzNVPNB>*7 zddORr$75!Ovs!Ido+LaeDJku}eKoJ2-j4~pJVndC`d+$yRy+9y=s>6(g{q#Cv+4fe zzBY)>aizk%$z#D)G;*g7J*y6D5;B(qi%Zkirt6P`<`RTYG*I4)_jyJR>5vg*t>+}> zh?)u8FgKZwhBh=NN?H!jt+}>#A$C-pth6+qseA|R`Vn7MPgq2R>LS5Do6Vm_po$&Z zwe!Ht#3Ztimw}l%MK=3eX|hifi`juN%D}MRbHASkxgYJ*^xCdkQn#--cQ6dGBXtd!;Q{7Z#=SG+NOv)u}7RIr5kUmrY_hhczpV+bhX^-HpX+upA9rm9`+ z_gv)u?CrH@e)9SU?+lr^1P8NSV0x2V{SiC`-S!fa}4 zs!R>|7`fxmok4%BoTca*I5a$L@8KbsX`^2@AuKCKq4fB{H_$u1?(f( z{O8ZAMfyac<>uziWG;H)`;*M^DBAUp!^N9joe@QNNv^s#>Q8#`fgNli2%_RmL7|3Y~F($YK0$_>JM_V5$!&47OT*GWU= zkHrNBd_>x$f>{H74x%|&C_6j5F7J*$+Siw-yC|(lS~BjfSIvGeJD>x1*_P_fDkL~8 zPMnO##i3BB&F#&?h6X;7Ns|TnC1xLU^;oB5ydvTrI`}RL2}yQd9(}MUP>wXmrV@p6 zRIux}g;XvLUK(O>s2AyXZGansg8`WN)m7{M@VAW8(!Lk9@2ulw2Eo3>FT1T*P6pN! zOiUmHU*KgMk+j0NMDq!-Pr^tmZD$Axz8BgvN^2Mk&bBFIoQF$favsVGG9TG0$1l9Z z#l^+o!3+HHHR8(m8mL!lr8mtK5fO_fnG#`< zde!&H#d7Fl97^-AhZf4@cn=0!u zrM=~{%x9PM7wAUHcOVbEbTcco@grS@u#i?i@+r^bETDAvnxZw*xj+av*`(jOx(fY< zA|G9PMuVsJmSyYl5Be0i-4@%-i=7gK7mIIG&j2h%Et|xDl;CBK>o8Be-1_7|6hZ^QY0%k>W2r15CN)sSO|r zG<_?7tDQhyp}Y0UEYRwiP#cAKAeV+xwaJgw)vqJO2b{@0g# z$t%Qi-#lSPKl8alO$V|;w88(?i9>~lwH%(=b6%R*8zm#dj5bqcpjpdP)%FF8a@-l3 zO;)5f$e|-#&STis{l);TbgpI0LuqN|i}Nq{FBTYTntRh|qu*y3f6|S`&a6JoGbVf~ z+Zf9b#UZ!ftcsnf97yC9I_ulpz|^Vjnb&qJ-g(LdNss8{dH;vxc;5VcL#)mnp0ZK6?0&wiA0N{x)J z>Q3Y+Q!}odH?gd-6`;gzrznt(#X8vGCU#V6ZQ7-UE%~-0&*xDM;*{n^L(a#iw6E}? zuF~a@fdZYXAoHCemwPi`sMdZh(WFXx%t58}WYmYh^?9L{DP(ER4Pk{Gf=To#2fsR< zMI6wZAhfwdNTNX~iN)IGC{E#!+i0F^kWQuV(0;2tJJZnCPOgoatFCZoMq2w5nl!MM zf%FE;VtG}YUn4!Xl2?pgg8PV5%7TZfMqHbkFJwvKWT5uz_{)L_{z7+FE(;1FGg4Do zsoGaT=nqToKCMcLrax(>X%ny^19B`|0ih#J4rQk=_AF<;$V}u?Cyzs9t*%Xb&ymuj z*0A}(q5?E`-yKR$mEdQe>7Kqxb{_T|i${K8QYyY5U(WhD{9I-dJCl2*!q~qkQFjDlMf{X*VdnFNCEjvo>D@MGW zpG$Ss=W&5;|JQ5tBzxWa4lWxCeI%r;tzvoMz8cg<_UPu) z8}8TW!D5>$c{~_m|MK`)KcNEYj?(*zm0Vq}JKHYxOt%%!NR*8z zK*{-0H#Q`HiCRfT>Gkt-c1T5SMJ~MGyNPcE?vWRXheaTNbW{HZz5uhhrq@H*>1N&= zUBI&XsqIa*OP1V1A1~WMLjxxI4Yho75E)k=7KLL(0jUrWmdJbL>MNCHKF_&7QPeur zncB)0f|;vrSh=#Un7B9genJ}|ie+K#<31kDJ#z`b8*i#MTc0`1Bn_G06LwYn98V?l(+Liy>2KV`-1>fAPS}9mjZr$ahJ2L@%t8nnEBKM zm$iW)j^4ZaR_82~Jx91mbWeH|+?7_5#Q*`=w<-55-1cJWdHzzE)3J5o6N=ORC?+R8IT=}^ZmrK(L9fB^a4pJ zcQev*xVhg}(uN^UWVbn=un)5Mj%D6$De2v|L_qK^+m1RIo&l6w!0y@q#~wn&W>e90%rI>trS>cphP^w%6^PSM@YwmOM$xBW1rW#EqezS>??OK=FYUQ-aQ$p-?# z%qZrLEdd5RBD37ygtMD?tc2Ud#KO+B3F>!|T%xNReyz`gaAx)4f5q4?Te1VsWcP+b zP*58SKO)-rjp7x>_+ixg9y}apE)eJ2En(oX>a7i}Kd=NN_Ewj)L|!IR)3)s(os8!} z`z!DwkDkMBww~M&|Ct$mg32(qz~f8Ao7zpO$XGfD+_(E$2)mJ*ziaC%%k~r^5a5OS z=i26%iM!b${DBI-q$W@9ev6=PJdIgD$Z2c;hMkT)v_(?;d)+IH;tD2+R=izd&(K7- z5Q$?k)C9vM{06qg;E@jJ3ZUMTEnh-0 z7+|Zsh(hDF!s6B1*hzFkOBBUKMgN)?v&qAoD@ii!`#As2B>F{~lGmm_qaF|1 z{eZWQuZv#pkYnIQ+jI=AA%eI}4$kf{zaQAjF-7j`T*F3z35IVdxT=KI(5NOga;s<; z%DZ|KRDc4sz~_GqP|>%M(uE4N9QCaSvig-OR?=`Q}IiswKD>u5B~)1baHyi zj#ZbCkc(qNQvNH zx}BztxAiVn*Kf4ct3NB(mGmtLR#~wl4^|9*vhL-)^?sR9AC|j3V)pZ%gINcKK7L)X zQ5lgg`)`M?>ltrBhM@kns|554xhdy{y%3!#uq|HS;oTvg1`at)BwoUi2aropUIixq z@VWsxn@h0wpr(F6qHk&&P3$t+3&YmWKJN9GBQ?8s!NIMezj^#Lu=eE<=h&wrRIr<^ zCDN}Tw_K@!QrVUF0gz1{sBrIMr>_NpaP;zz6n&ZBCzcW{yId`>><2bYvQl`^sz@%g zssV3B)S>jkm6SW{S`G+KY1~qw)Ulgfc4m1h!8KS*f4)Dbf`9$4|NSgeqhvhiM8_RW z{-?Ezl9I%v0frrj4`sj?#hAqtFIt@vvMYupLsP>QdDYp8@4%vQzgo@WVM((=Dn|Y8 zPPiC;+X0{@_JqKZA~oh21-BMohLT(!MxdnBg|vc7dKiF0cFsiHM9M~~dkYVy<4N1F zfj5NzGPkP5gSxt7MZ&}1^kJ*ZV;h<5Qj@vtv*1)v?LyQ`#-ldulga}Hm=2t;nb_Dc z0^`o&oD^msi_gimG}F0gh+l?*v{3OfjbldstomY;9_H3&8D9?_F@3m$_192W9tTV~ zD?{uyjpI^^ME$FC_-2Q!H#m)rCs63AU=Dn|1oarOd!_DpkZz8JFlPP(W#g2hF#KQ1 zNliHdPdYCrAMSKV008Op>%FnrH5S-N&{Gr04^-f2KHWc)3Hy>)7r}w7*ZIren8;&MRmC4z?$sRwM5WBBG{JH-E?^~&h!!l@TEEhBB4M+Rh@7b=h`%(<&BQ|=Y0^C&70UFZF79n%8TxV7h54L7t zl(|;9k)5bxhW~NjuufZXv-z$5_t>qiEt9S{P*6aRZlk3(3jJ)|+@5aQCG+G4a@NV~ zfOEwqC1D|opu8%A9xk{@RS#ANh~T1#;1`XN9o@@}s-L>@gtW;I!H zYFRpej0cZ|!A#7;!IX~p24FGPH%D15R-7v1;s8LnJeAZ;kF~4r>)%^_`MiT!_^A)Y zT#}l6puEn_YR~#x&@mZLEAZ=Zg}tPGMIm~6ZH3OBmA$(^NuGe&W>{xAm7cVpUZ^8$ z^t{tuIO=TU;X&@}>zn4t_~9u&0OK%w^X|sS6DYqwE)wAQR>~Ck4VpKc;Q!!aYKj45 z>^F!qB&q_FJ@f*AeGaDnEXm{IWX+YzT7>uN;J+j(8uB-GunubX2q5f5rUvJG75 z4yLsCW@;3_1TzVM>X1O+llKZ2-RgX*dUn=SvdNKn;yj&fk9}}~ZK3QkFa~^apppmo zAwgGuFP(W90R60U@6*sxvIGTKPde8>*e2}F&?WgP)65Wu7C!V&&tp@nso+@w$?!8Q za=EKq8^}PU@)XQFXLyxH!49MZ%F;PA^u~zh{X7VUCnVb|G?eA}^XFYCZS?k1XUscX z=zB()zwJSlO}~-V_PutpRnP$-?VVC5O{8Dz#daB0ol7IYqMmAN_s|7`hDZ6vmt{n5 zd7@j}uF=)FZYX7HtLLNKD}UOOVQA-TlOY0g&kPBbDP`49tT^gv3e;a06u)hp5f?!tnqxQo$>h$Y_Yd!!E>Ra;(`;-kG z5N$sx5?bCW$+HUJpY7%TtOFnq;?hz(fB%Q+K|$WP%mvF`X|&aII04IN%c;wA=lo|K z^71Kp?XcL43%{Ho?Ox3lpozlRMpKISOPA=@4`(rza&m-ohqg9auuBxTD$upnRW49e zt#527e^{;H)%-?17Xu(FN&g)N1H;`5O{JDsQe>)n`MjFQPQK*xXWtEod+ezGr`?7l zrWi4`dVD^KM*xcRg7TEi;f-p~#dZW{m#b#RhQULu{N<9xjVEHQ<1LTP?v1ZPuyx(O z)8~)5xu?4sQa-DR8$uYD`k7`I-4h(u%rzs5VfJf7k2YfcuW+4zK>eSOeb9lLX9~eP z)x9#-n9Gj|1y-o`48Sv({Ko|tb*^i7`dDlm6f`a3z?U4|uTjn6`A%cD2HHFJkg?ytyVyh~-vOPTAG9!ry7V7c+>fCh(1f+Hk)@?!8Z?fwQQ;3^ zo&hN4Qx$~yYyFWsTC%Rho%uJq%fHohvE|wmp-CfAIj5~+Yu~4tE}1~gEOcU}o9|DB zMzWJFQjUTkp!snmI zUqF*PkB(|KP&+-Gg}k8mqX|Z^wnwx6bAHS6y@pG7PpMT0_wR%+eEpDEh)B!ey4$8$ zc1yxMV4b{Qh ziUs%fkiuHl<1=%$o`pwx^xNoWPhBIytd#E(Ds>A$GW|aVhSa_Q2z=pv#>~p%UVxkH zsq^D{)Wdja^6w;S9 zHXR-wGMFi=plFqi#@uE*F1WttLWhG%Z!dliUBH93wtfc>t&FqL1&*uy{k2x(g|HK% zqf$-kj?<`Jp@`HPF846$O14)JP4Q!fEm6W=Pu?fRX$&V1+~P-ETfg$ypHN+zxZRwq zljwvEZ7PmQVrm^q1tx02N zwy`Fxkl~of5w)I6d0`Z7Htr2wdkR842O(8*TNeAp?ufnN(~}~0?2pSX9v;5~I`!T| z2P8w!yna-0a@zH27N5vtSKg2Y9v3ZO>l;G`tcJHw*T#o#=eDcm7v%e`pxYmdUw$*- z`({+u4j-&ecVqzy=fT^%y6uhcHErX`vt*Qu$snB>HZXbx9~xx)m+E*swOy99z)+P>B% zknhO+hM>B~ouAu`x!7M{5`t6(EKaEa!OfKTtqL$&dZZh$FszWhtCbfcW;gqYS^iOs zL(_!sWej4oq>(%`MyUWxBR$mko^>kdet`_gEWT4G(~*W44}fl`7F44OfXDI>J~u&X z7H}itbxz~8MKE-mgH2EAfoQ%4k1OF*XRXRflYBI@oe)=LW5b?1x5 znf6DO0iyqTW$B40Xo>V*?W1xKMfu*C)tmJU4nCc22j21BJ1Wqv1fLF~jD0n8C3JVF zl5G3TsutRb#ea*_leM+~A=TwCF>f01dBBdHrGaX2DjszP z9HjbqasZqp|1VBetD^Y-!J;Z7LB>o_EjRYYJ;Fl7lZoVI0ADB!?D?~8Vq8gf%p`uN zuSg=`1i*kM5?2XxQ`Y@$Z|_97L>|E)k=3>_j^7DqQKZ&@>taJ{it|f?rU{o1eq^=G zH@LJ$llW1~g|V`#WpY9$|4Y~o`X6g(`3`wZ*8>?Me7+LTf=#)qYtGiL_v5aoVRvVT zKDS?9{RWNlKc1wf*IW$ctx6fzOD?mgj6S$OHr&q9d_i#!7V{kiP+qLPVWbwIqi(}c zbCJKIwMSj9qHKg}YLlC7FrJ^^lv2P8TbNsJ-fc)tO#H^iP-C~sT#4J2=J~BgCP3m+Qyf&L&|26GI@^z zBpadt^~i(NRPT7J!e=|EMvk7y!o%^pVYgS(y@hHws%GDJwgLYuyW-%Dg~KN>7F2Om z)SnYsxt>mAlH$wQ3kGkRXEUNbAnM{4)9`E7>r=?iD3D1f!mp-qRhXd6ef*FTgRb6j zyu!{yg>&&cQt=nfBi=dJ7NjPCjR#yH?=;Rk_t^)p9^(my1eN4Fu)2DfhkVY7 z-z(n`n{N$z9=Ml28w!OZV>c{_2K#-acx0lPR%ofH&^SEpYD3`?9MRh0aJl_#m`9_~ zJiCvCE!>zPS+Ig~)kXPT5liE196*-?wfxho=-18&pScanKhd$@t-^7yGW8E;dG1tN z3`jhhJiT!-QN{-Ep}gP-jQdxWbq19ylM6a+l{a=eUej z`3{TZ`gC4IM%eEE_=(KQlUUwYMuf3~aQhhia&~#*aM>sA#yHJ8zR>Z|&Hy%GM4*wx zWT3PB%T<0(Ieif>S)R17_^UCFtLorKfdZhQ-T6lhA~eaZ!h1k{rz5v`lHVih*G3>v z5yjp#2~w5r_d!LrvD$|ux_8$YzXD5D2k4$H%r@qrbbWq-SvWCY5l|-pP)&ayO?{nYtKJPj2y@Vu5-S{Jb>W5EBoYbo#EBCnf*=rg@;-N* zh}Y|-PoF+r&hz};MI;uBk;jf5tKvBBjv77}1t$^=2IUtoUNnoM_@xq|vR#NqqhxPy zuh!@D(GG_rR$E&e2Y^Tv2WRk3}ua8~1azzgS=g*%ns;#Y! z&75<3tkda?YBU;Qx;+pG$h*6{4XIR0=5#tEcDp?}eN7ZaT)uo+?{>SD6h%p!Hf@?P z7!3Sedoz0Ewbx!-_R1@-*h8TZ#q&Jz^2;x;?C9ty27s=vF8%4#r^{yc@9gX}wzs!i z0f1#$82}`cNiv;If9{o+UV3T8n{U2Z)z#HyeD1mDoC5;`N&x8V>r=h(!V7gnLqjTx zqNG<}eYN_`nKLE;xOwxY>ctmdbbS2r$9lb9&)v9jL-+ji&##WfVq{((xq9`g@tJ3y zxmj0N7Y6{Erp0sT&RMo^-yX=+e`R z05mo>h7KJ%)Ntj>75&DI8z*ymWG-wrTf*n_DZ}A#)}@#W#l^+k^#6*AibNm~pgE2s zdV6~fD_5?JPG769ua89{5rxa;QqP@3tJMkygCRZB(~BZfR8+(R!0_-evospBK@d}R z#N%<2Wm$6OnpY?kB1KW+~fcWN>g$Ns^>kQ&W=wfOI-t5VbQH417gJMSNgj zKsDXwcDort5M&Jv4UxsnmDM1Z%cV^u5@a|WraC)2P1V)aQJSVjr_&h?g+hw1t}Z<= zr8r|_W6J4qMMXs%MNy*1<(s6h(ME9;MZ4WtU{ZOk4GpQbZ{I$)Zr!>?Da@Tf#u&L9vE-$d4S`6K z>b~XGjD_R!iD*(DiY6&aMo0!V!yB~>Z#AgXTh=U#P&s06u_zdkes75W@$rkcUiY{b zk_1Q+fX{wE|1ScP1dBnH`sNm=?~w=VCkjQVAWKUWfro#3s@@ZbF{^DR#FHru_(I4= zWV(IV`j!44d~3__@67@QI#>;ig=o)2oLOVH;O7Tg@b00nqgb!XIqp{E5>rNAOG_5a#2E#;LK+D zBJ-aAqy<7MZz0H=$jM8CMwSx_qWLqSQPKj$47r34%Q1e!lM@*jLR^d}~X?NOMC; z#H1LfAo0XJvJwFBK;pI%;eRn>EI{jQ#mR3!pWP6S;wS3lktoNIxQpTe{o>5 ztAhJ?3p~HIK@x~*56sRlGo?9?>`xVO`L6!FW2D5ON@csY|7-UQSES#uBJx`cP{7v- zOj+Iqjl7AJWg$Wkkajc_M-lm*Ip4YR22)jdBFzou!A$LDy)v!B5S=ScE(ylwyel(# z6Zwk68go_|=T%Ecf{>;HyTDM@MN#xXGn?A79>7CUW`$)UsAeAHbx!g|gil<2!w}n8L<$f4BQ1BCCM)=>U%$P?c z2~gG~3qj<||AeDx8({uZfGF%nx-cFg^L!~omjhLYrv9frKWR^x?)41r%-5p1_Wj8hqj5DX7v%Q&oi$3W=|mXq$>YlW0gf%G5^c7y?;5J8ux zz*MCmH|Ji0OTbWpa+TAT)eK~F$&?9~5SgVC7$ zpa1ro{UiRM;^(K%SuF-#dhd?)0i$jzKeheOMbn3$^y#WfOl&Y3&scHE=#g*z@D0bN zwYAZ|-~X^X9B0X+@3xl$!1rJM&#HmZapu{*+r96czF>LnKR&SNwMzc&Uv-v>qKE;{ zIDPy~cge5Me_R9r|9#<_vDBpF{_gdYmG*KA`|X`y@gF?ewn7jD;?~d@bKuB3)jJ<> z246b3cR(hS;n|&pufbV=l7E%j!AM-b-nk;Cph)TdRm)=~Mm^_pduJ6gT!UVQQBdOg)%F+w+*h}}kEBS+7YNZ0 zuB{2@vNLC}U%x$~TxBndnT6cdxF2addnb_$s~rB zOHHd+gpd4Nd*zn<>cSc&BkDCuE|ubB4qGYfu$8iUt!iE(bl6H+|728AWi_U2mzA=$ z%Su^VPD$$=Rnf~XwNz;}0V^6vl1+ci6yoD=7- z_301)<(?q`C^H*4vq76)ZgE_1@axcrdEi z+p@tw_kN|tdhRd2v1#=2AHMF`d~a=Vcx;mX-Vb){@kMM6wO~|A>q+qEXU!zQ9mmWb_Ef^MstD3i^q~okm8F+-)R) dKcc`unPw&?X(v;o(_+&^9j6^TNhdm!bdrCJ#A%1=M8%mVjhQAz zMQEpt+<>-36!8aE0rdjN9X}3^-*9-^_j&i-^S$4D+}_^O=9$^qTlQ{upAX;Xd7t;) zcNgF*^J)>{zJPyy_?nKa@;vNnUA^s~_5TJA0Z~L&xO|H&Sze&HK_ZKhESJb)Aixa& zGjIryQnK1a7F_FG_s1lpKult)+wW$^EHE=Zhd{ExsHqc1%^h^x1yng%4um2EBSHa1qm6y*u5HSp4E@R? zNXh&r{T!^Jnufl-45q6FURfc56d@xRRLWbTQj+vJur$Afsw7l~YU;jNOV%Acqa7#soyp&%Tpl&n=hR$X;#Z2FukiCHB9 zv?b(lcrd7zujM?g+VX#(;|r>yxdMs`5X@dyzo;IJP*-KZg2WWk^k2cvysaCbjyQDU zoI!Tw@Hu8afbpRq$vG%eA{gugy^o)lk``$`fkDIT#p`DFKJ!N&IAU{+Rc=Op$-EqX z!0X2AXGQ7}S>UidlZVapBP5*Wvpw>l+{j~o%{}L^>i0H&&fW03qE&AwtLe6y55&!E zUsZIZgm9vAoG)M|to>lTcpBR3kAc;6+igGk!NLptg;#{1fvI}@T(EhsDymIlTStou4{wTZTp;HX+fYdeA>C5ShQb zq=r?gkcb~wB^Xo+D#jIzTj#UO3>ja~hvdbchhsVaN+pFAs~)qEESJ?)lV8q#evsxE zwDrH-58r)hkK6a`_InhJR;P!T-#F;?c-+u!mZz%OQ&<&=V>D{ZFK0;@@y&BxQ%P+3YcCXn2c1;4j@az4MyK1eZ zW22u84*Gq-ADlcjDF^clhEZ(z| z!1UBKOc59p6BBfea2y@Pm~>ch?-oQy-gip22xW2hO4w5N9xQwODOs)s{bd94{*|MN zi46%1gfLDPTw?Vsb((CIG)*Y-#<0~2gBFWlbx2CgsViXbyYB^@XjKIV z!=|slpDIPRX1u)}^1i(~EMTMzc1eQLedi8bztJu$7_{bd&G7xg?}e?L2ue8^KD{Qh zQ!68vUAcS(($mu*Gcz-6aGaD=V;L0sG0qc&yxPdK z(~OO-#feA$Y-xHrL`6l(ez9j7&p=Mja`?uI6^g(S{T2Q92=%lKkLa)Hzqf>JG}&eX3azB1M7qdf*md~qub;v0-Mb(zE>6*qVtyNa z!nrs$CeSo@NIrpF;dQF|6l~h~tYYEB`^o(F4W~tfW9pQA0y)B~y1E+lOO`-M$uo+! zNTQ!De&JL!85^;ECWbRMr6%TJ!>hTu8B%n~P*(PHMZutd=qGI*(Qo3v&mwTlez zIxN&N?1!NYFRW)~Nh_f=7~Vhh7yXt}As=rL#KTV;3rTn(m@!f-g#!%lAM^LP6gY;O z!9f@&#T{nyKyptY;Z^s~I!H;@g$)e)h5n(R=r0J$$O5ulsdO92a9PkCWh8I@8moHs{T|0M#wKK{4gZ_;ReZxxWaD*nU3aMh{ zh=b2PL4?<}wl>g4N5i(QuP9dedNcF`{XxH&YN}kDa4F+C_k;e>vmpygg)7~#o`B9qOQl@O(Yl?e&7E_i6g8ihil%*%#&bLvo$+1X-31+!lk^1FYCZ_4a#n-! z*NuLlKfHf|D@Ra|m8b`&#ua@*inSjuS_Bc(<1jta4~v!+%o{iz^+#z)VHo|Y{e~D$ z`-p0)<_<>yWFUY_$pM!%jgG*&lvJ>f4$!2A_n89;v2ZBn*4a`dG*G1_)@q< z)Jnj-=308fN76Vx1HVkqHtieT{ifjx&MsX>u+rt>{!q8==IN{41= zK~xZi82(pKLE5+^)qufn0fLz#;qL+lZ97rT-{pfN0BR+GM#!gUL_jO!6JX)}yA)8Q zCK?hKXN4UMb|!DseF%1zeD~b!&AAT4%q%LxN z1tH;NXekfZ&qXO1LLwA2yzg|Nt4nOEA2cQvO_g&wM{ti$m#uO<9W4Gy9}@#x)6$$E zi8E6XRxqFt2*SXjP5xC52<1?3FPLU#_^(?T80DW1<^W{@_gpy->G{PE(ZVr-8{5HY zW^t(*u#mp8A%(uS0WFR|!+qEUPqN!3_?iZUFhUBrNlTAY*?{UeH8=>L_V>9$5eKfe zcAKr#BMthB>J)SqJ%~pQYEv%)KQdJ!!m!(lBgw%_+QEu<}*M8wp$hd3?c# ztD^K;EdiB874@i(A&hp?1fxPOj*Sv9HctxSq0o0(5HRX{mqnnEWU`Pz2`wD=r0we8}@g0u@Ln5&aO~|%2y_n{a1hi00-6?7X^q( Q0RR9107*qoM6N<$g8M56QUCw| literal 0 HcmV?d00001 From 708f1b009b9b23971d73bfc7bc09163969ab6e00 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 10 Mar 2017 21:36:33 +0100 Subject: [PATCH 113/362] Add integration tests for MEDIA_ALLOW_REDIRECTS --- tests/test_pipeline_crawl.py | 163 +++++++++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 tests/test_pipeline_crawl.py diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py new file mode 100644 index 000000000..1f5b80954 --- /dev/null +++ b/tests/test_pipeline_crawl.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +import os +import shutil + +from testfixtures import LogCapture +from twisted.internet import defer +from twisted.trial.unittest import TestCase +from w3lib.url import add_or_replace_parameter + +from scrapy.crawler import CrawlerRunner +from scrapy import signals +from tests.mockserver import MockServer +from tests.spiders import SimpleSpider + + +class MediaDownloadSpider(SimpleSpider): + name = 'mediadownload' + + def _process_url(self, url): + return url + + def parse(self, response): + self.logger.info(response.headers) + self.logger.info(response.text) + item = { + 'images': [], + 'image_urls': [ + self._process_url(response.urljoin(href)) + for href in response.xpath(''' + //table[thead/tr/th="Filename"] + /tbody//a/@href + ''').extract()], + } + yield item + + +class BrokenLinksMediaDownloadSpider(MediaDownloadSpider): + name = 'brokenmedia' + + def _process_url(self, url): + return url + '.foo' + + +class RedirectedMediaDownloadSpider(MediaDownloadSpider): + name = 'redirectedmedia' + + def _process_url(self, url): + return add_or_replace_parameter( + 'http://localhost:8998/redirect-to', + 'goto', url) + + +class MediaDownloadCrawlTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + # prepare a directory for storing files + self.tmpmediastore = self.mktemp() + os.mkdir(self.tmpmediastore) + self.settings = { + 'ITEM_PIPELINES': {'scrapy.pipelines.images.ImagesPipeline': 1}, + 'IMAGES_STORE': self.tmpmediastore, + } + self.runner = CrawlerRunner(self.settings) + self.items = [] + # these are the checksums for images in test_site/files/images + # - scrapy.png + # - python-powered-h-50x65.png + # - python-logo-master-v3-TM-flattened.png + self.expected_checksums = set([ + 'a7020c30837f971084834e603625af58', + 'acac52d42b63cf2c3b05832641f3a53c', + '195672ac5888feb400fbf7b352553afe']) + + def tearDown(self): + shutil.rmtree(self.tmpmediastore) + self.items = [] + self.mockserver.__exit__(None, None, None) + + def _on_item_scraped(self, item): + self.items.append(item) + + def _create_crawler(self, spider_class): + crawler = self.runner.create_crawler(spider_class) + crawler.signals.connect(self._on_item_scraped, signals.item_scraped) + return crawler + + def _assert_files_downloaded(self, items, logs): + self.assertEqual(len(items), 1) + self.assertIn('images', items[0]) + + # check that logs show the expected number of successful file downloads + file_dl_success = 'File (downloaded): Downloaded file from' + self.assertEqual(logs.count(file_dl_success), 3) + + # check that the images checksums are what we know they should be + checksums = set( + i['checksum'] + for item in items + for i in item['images']) + self.assertEqual(checksums, self.expected_checksums) + + # check that the image files where actually written to the media store + for item in items: + for i in item['images']: + self.assertTrue( + os.path.exists( + os.path.join(self.tmpmediastore, i['path']))) + + def _assert_files_download_failure(self, crawler, items, code, logs): + + # check that the item does NOT have the "images" field populated + self.assertEqual(len(items), 1) + self.assertIn('images', items[0]) + self.assertFalse(items[0]['images']) + + # check that there was 1 successful fetch and 3 other responses with non-200 code + self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4) + self.assertEqual(crawler.stats.get_value('downloader/response_count'), 4) + self.assertEqual(crawler.stats.get_value('downloader/response_status_count/200'), 1) + self.assertEqual(crawler.stats.get_value('downloader/response_status_count/%d' % code), 3) + + # check that logs do show the failure on the file downloads + file_dl_failure = 'File (code: %d): Error downloading file from' % code + self.assertEqual(logs.count(file_dl_failure), 3) + + # check that no files were written to the media store + self.assertEqual(os.listdir(self.tmpmediastore), []) + + @defer.inlineCallbacks + def test_download_media(self): + crawler = self._create_crawler(MediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl("http://localhost:8998/files/images/") + self._assert_files_downloaded(self.items, str(log)) + + @defer.inlineCallbacks + def test_download_media_wrong_urls(self): + crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl("http://localhost:8998/files/images/") + self._assert_files_download_failure(crawler, self.items, 404, str(log)) + + @defer.inlineCallbacks + def test_download_media_redirected_default_failure(self): + crawler = self._create_crawler(RedirectedMediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl("http://localhost:8998/files/images/") + self._assert_files_download_failure(crawler, self.items, 302, str(log)) + + @defer.inlineCallbacks + def test_download_media_redirected_allowed(self): + settings = dict(self.settings) + settings.update({'MEDIA_ALLOW_REDIRECTS': True}) + self.runner = CrawlerRunner(settings) + + crawler = self._create_crawler(RedirectedMediaDownloadSpider) + with LogCapture() as log: + yield crawler.crawl("http://localhost:8998/files/images/") + self._assert_files_downloaded(self.items, str(log)) + self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3) From 810658bcc5b1897d57c0882a0f7ab4a0d264a778 Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Sun, 12 Mar 2017 05:06:12 +0530 Subject: [PATCH 114/362] Add feature to set RETRY_TIMES per request (#2642) --- scrapy/downloadermiddlewares/retry.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 549d74f46..a5342995f 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -63,6 +63,9 @@ class RetryMiddleware(object): def _retry(self, request, reason, spider): retries = request.meta.get('retry_times', 0) + 1 + if 'max_retry_times' in request.meta: + self.max_retry_times = request.meta['max_retry_times'] + stats = spider.crawler.stats if retries <= self.max_retry_times: logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s", From 871134ee22653f7f074fbfb8f3c393ba3555a6d4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Sun, 12 Mar 2017 17:30:24 +0100 Subject: [PATCH 115/362] Refactor to also test FilesPipeline --- tests/test_pipeline_crawl.py | 79 ++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 1f5b80954..9b81f827d 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -23,8 +23,8 @@ class MediaDownloadSpider(SimpleSpider): self.logger.info(response.headers) self.logger.info(response.text) item = { - 'images': [], - 'image_urls': [ + self.media_key: [], + self.media_urls_key: [ self._process_url(response.urljoin(href)) for href in response.xpath(''' //table[thead/tr/th="Filename"] @@ -50,7 +50,15 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): 'goto', url) -class MediaDownloadCrawlTestCase(TestCase): +class FileDownloadCrawlTestCase(TestCase): + pipeline_class = 'scrapy.pipelines.files.FilesPipeline' + store_setting_key = 'FILES_STORE' + media_key = 'files' + media_urls_key = 'file_urls' + expected_checksums = set([ + '5547178b89448faf0015a13f904c936e', + 'c2281c83670e31d8aaab7cb642b824db', + 'ed3f6538dc15d4d9179dae57319edc5f']) def setUp(self): self.mockserver = MockServer() @@ -60,19 +68,11 @@ class MediaDownloadCrawlTestCase(TestCase): self.tmpmediastore = self.mktemp() os.mkdir(self.tmpmediastore) self.settings = { - 'ITEM_PIPELINES': {'scrapy.pipelines.images.ImagesPipeline': 1}, - 'IMAGES_STORE': self.tmpmediastore, + 'ITEM_PIPELINES': {self.pipeline_class: 1}, + self.store_setting_key: self.tmpmediastore, } self.runner = CrawlerRunner(self.settings) self.items = [] - # these are the checksums for images in test_site/files/images - # - scrapy.png - # - python-powered-h-50x65.png - # - python-logo-master-v3-TM-flattened.png - self.expected_checksums = set([ - 'a7020c30837f971084834e603625af58', - 'acac52d42b63cf2c3b05832641f3a53c', - '195672ac5888feb400fbf7b352553afe']) def tearDown(self): shutil.rmtree(self.tmpmediastore) @@ -82,39 +82,40 @@ class MediaDownloadCrawlTestCase(TestCase): def _on_item_scraped(self, item): self.items.append(item) - def _create_crawler(self, spider_class): - crawler = self.runner.create_crawler(spider_class) + def _create_crawler(self, spider_class, **kwargs): + crawler = self.runner.create_crawler(spider_class, **kwargs) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) return crawler def _assert_files_downloaded(self, items, logs): self.assertEqual(len(items), 1) - self.assertIn('images', items[0]) + self.assertIn(self.media_key, items[0]) # check that logs show the expected number of successful file downloads file_dl_success = 'File (downloaded): Downloaded file from' self.assertEqual(logs.count(file_dl_success), 3) - # check that the images checksums are what we know they should be - checksums = set( - i['checksum'] - for item in items - for i in item['images']) - self.assertEqual(checksums, self.expected_checksums) + # check that the images/files checksums are what we know they should be + if self.expected_checksums is not None: + checksums = set( + i['checksum'] + for item in items + for i in item[self.media_key]) + self.assertEqual(checksums, self.expected_checksums) # check that the image files where actually written to the media store for item in items: - for i in item['images']: + for i in item[self.media_key]: self.assertTrue( os.path.exists( os.path.join(self.tmpmediastore, i['path']))) def _assert_files_download_failure(self, crawler, items, code, logs): - # check that the item does NOT have the "images" field populated + # check that the item does NOT have the "images/files" field populated self.assertEqual(len(items), 1) - self.assertIn('images', items[0]) - self.assertFalse(items[0]['images']) + self.assertIn(self.media_key, items[0]) + self.assertFalse(items[0][self.media_key]) # check that there was 1 successful fetch and 3 other responses with non-200 code self.assertEqual(crawler.stats.get_value('downloader/request_method_count/GET'), 4) @@ -133,21 +134,27 @@ class MediaDownloadCrawlTestCase(TestCase): def test_download_media(self): crawler = self._create_crawler(MediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/") + yield crawler.crawl("http://localhost:8998/files/images/", + media_key=self.media_key, + media_urls_key=self.media_urls_key) self._assert_files_downloaded(self.items, str(log)) @defer.inlineCallbacks def test_download_media_wrong_urls(self): crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/") + yield crawler.crawl("http://localhost:8998/files/images/", + media_key=self.media_key, + media_urls_key=self.media_urls_key) self._assert_files_download_failure(crawler, self.items, 404, str(log)) @defer.inlineCallbacks def test_download_media_redirected_default_failure(self): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/") + yield crawler.crawl("http://localhost:8998/files/images/", + media_key=self.media_key, + media_urls_key=self.media_urls_key) self._assert_files_download_failure(crawler, self.items, 302, str(log)) @defer.inlineCallbacks @@ -158,6 +165,18 @@ class MediaDownloadCrawlTestCase(TestCase): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/") + yield crawler.crawl("http://localhost:8998/files/images/", + media_key=self.media_key, + media_urls_key=self.media_urls_key) self._assert_files_downloaded(self.items, str(log)) self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3) + + +class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase): + pipeline_class = 'scrapy.pipelines.images.ImagesPipeline' + store_setting_key = 'IMAGES_STORE' + media_key = 'images' + media_urls_key = 'image_urls' + + # somehow checksums for images are different for Python 3.3 + expected_checksums = None From 0d57b5cd43343a335fcf2e923b29e76d09dd0b51 Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Mon, 13 Mar 2017 02:10:23 +0530 Subject: [PATCH 116/362] Prevent max_retry_times override --- scrapy/downloadermiddlewares/retry.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index a5342995f..07e979628 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -63,11 +63,13 @@ class RetryMiddleware(object): def _retry(self, request, reason, spider): retries = request.meta.get('retry_times', 0) + 1 + retry_times = self.max_retry_times + if 'max_retry_times' in request.meta: - self.max_retry_times = request.meta['max_retry_times'] + retry_times = request.meta['max_retry_times'] stats = spider.crawler.stats - if retries <= self.max_retry_times: + if retries <= retry_times: logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s", {'request': request, 'retries': retries, 'reason': reason}, extra={'spider': spider}) From 3cd9185aa12430708b17eb8c02a6bdc3709ed5f9 Mon Sep 17 00:00:00 2001 From: jorenham Date: Wed, 8 Mar 2017 20:44:39 +0100 Subject: [PATCH 117/362] Fixed the FIXME; more specific exception catching --- scrapy/pipelines/files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 843b4d3ec..bdc0f24e5 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -58,7 +58,7 @@ class FSFilesStore(object): absolute_path = self._get_filesystem_path(path) try: last_modified = os.path.getmtime(absolute_path) - except: # FIXME: catching everything! + except os.error: return {} with open(absolute_path, 'rb') as f: From fbb411a805724fec50b786f369be79dc221c798e Mon Sep 17 00:00:00 2001 From: woxcab Date: Mon, 13 Mar 2017 14:16:39 +0300 Subject: [PATCH 118/362] Allowed passing objects of Mapping class or its subclass to the CaselessDict initializer --- scrapy/utils/datatypes.py | 4 ++-- tests/test_utils_datatypes.py | 43 +++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index e516185bd..eb373c501 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -8,7 +8,7 @@ This module must not depend on any module outside the Standard Library. import copy import six import warnings -from collections import OrderedDict +from collections import OrderedDict, Mapping from scrapy.exceptions import ScrapyDeprecationWarning @@ -224,7 +224,7 @@ class CaselessDict(dict): return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) def update(self, seq): - seq = seq.items() if isinstance(seq, dict) else seq + seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super(CaselessDict, self).update(iseq) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 80f797227..3a4137942 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,5 +1,6 @@ import copy import unittest +from collections import Mapping, MutableMapping from scrapy.utils.datatypes import CaselessDict, SequenceExclude @@ -18,6 +19,48 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) + class MyMapping(Mapping): + def __init__(self, **kwargs): + self._d = kwargs + + def __getitem__(self, key): + return self._d[key] + + def __iter__(self): + return iter(self._d) + + def __len__(self): + return len(self._d) + + seq = MyMapping(red=1, black=3) + d = CaselessDict(seq) + self.assertEqual(d['red'], 1) + self.assertEqual(d['black'], 3) + + class MyMutableMapping(MutableMapping): + def __init__(self, **kwargs): + self._d = kwargs + + def __getitem__(self, key): + return self._d[key] + + def __setitem__(self, key, value): + self._d[key] = value + + def __delitem__(self, key): + del self._d[key] + + def __iter__(self): + return iter(self._d) + + def __len__(self): + return len(self._d) + + seq = MyMutableMapping(red=1, black=3) + d = CaselessDict(seq) + self.assertEqual(d['red'], 1) + self.assertEqual(d['black'], 3) + def test_caseless(self): d = CaselessDict() d['key_Lower'] = 1 From 0f2a5cdb8edb3d4d6ce542f13b5d7e5858905bae Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Mon, 13 Mar 2017 15:13:24 +0100 Subject: [PATCH 119/362] [logformatter] 'flags' format spec backward compatibility pass 'flags' kwarg to logger so that it is compatible with old format of CRAWLEDMSG. --- scrapy/logformatter.py | 1 + tests/test_logformatter.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index e7bf7942e..2a89c00c5 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -43,6 +43,7 @@ class LogFormatter(object): 'request_flags' : request_flags, 'referer': referer_str(request), 'response_flags': response_flags, + 'flags': response_flags } } diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 11fe7b653..52646ec1b 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -64,5 +64,32 @@ class LoggingContribTest(unittest.TestCase): assert all(isinstance(x, six.text_type) for x in lines) self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3']) + +class LogFormatterSubclass(LogFormatter): + # Formatter with format spec that is same as in Scrapy before 1.3 version. + def crawled(self, request, response, spider): + kwargs = super(LogFormatterSubclass, self).crawled( + request, response, spider) + CRAWLEDMSG = ( + u"Crawled (%(status)s) %(request)s (referer: " + u"%(referer)s)%(flags)s" + ) + return { + 'level': kwargs['level'], + 'msg': CRAWLEDMSG, + 'args': kwargs['args'] + } + + +class LogformatterSubclassTest(LoggingContribTest): + # Test if old crawledmsg format string still works fine + def setUp(self): + self.formatter = LogFormatterSubclass() + self.spider = Spider('default') + + def test_flags_in_request(self): + pass + + if __name__ == "__main__": unittest.main() From 694c6d3d7460ab80ace979c4563af8f310614d37 Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Tue, 14 Mar 2017 16:14:40 +0530 Subject: [PATCH 120/362] Simplify retry_times assignment statement --- scrapy/downloadermiddlewares/retry.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 07e979628..c22437ff1 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -63,10 +63,7 @@ class RetryMiddleware(object): def _retry(self, request, reason, spider): retries = request.meta.get('retry_times', 0) + 1 - retry_times = self.max_retry_times - - if 'max_retry_times' in request.meta: - retry_times = request.meta['max_retry_times'] + retry_times = request.meta.get('max_retry_times') or self.max_retry_times stats = spider.crawler.stats if retries <= retry_times: From 966bd49c421fdf40a8b21b49c76cd54ded06fe50 Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Tue, 14 Mar 2017 16:23:47 +0530 Subject: [PATCH 121/362] Update unittest for meta['max_retry_times'] --- tests/test_downloadermiddleware_retry.py | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index b833cb448..cc3d37075 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -103,6 +103,34 @@ class RetryTest(unittest.TestCase): req = self.mw.process_exception(req, exception, self.spider) self.assertEqual(req, None) + def test_different_retry(self): + + req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 1}) + self._test_retry(req, DNSLookupError('foo')) + req2 = Request('http://www.scrapytest.org/invalid_url') + self._test_retry(req2, DNSLookupError('foo')) + + stats = self.crawler.stats + assert stats.get_value('retry/max_reached') == 2 + assert stats.get_value('retry/count') == 3 + + def _test_retry(self, req, exception): + + req = self.mw.process_exception(req, exception, self.spider) + assert isinstance(req, Request) + + retry_times = req.meta.get('max_retry_times') or self.mw.max_retry_times + + while req.meta['retry_times'] != retry_times: + req = self.mw.process_exception(req, exception, self.spider) + assert isinstance(req, Request) + + self.assertEqual(req.meta['retry_times'], retry_times) + + # discard it + req = self.mw.process_exception(req, exception, self.spider) + self.assertEqual(req, None) + if __name__ == "__main__": unittest.main() From e321ac9931007360a38dbd6a5933794af415274b Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Wed, 15 Mar 2017 04:12:32 +0530 Subject: [PATCH 122/362] Update unittests for max_retry_times --- tests/test_downloadermiddleware_retry.py | 37 ++++++++++++++---------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index cc3d37075..064c740c9 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -105,27 +105,34 @@ class RetryTest(unittest.TestCase): def test_different_retry(self): - req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 1}) - self._test_retry(req, DNSLookupError('foo')) + req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3}) + + # SETINGS: meta(max_retry_times) = 3, RETRY_TIMES = 2 + self._test_retry(req, DNSLookupError('foo'), 3) + req2 = Request('http://www.scrapytest.org/invalid_url') - self._test_retry(req2, DNSLookupError('foo')) - - stats = self.crawler.stats - assert stats.get_value('retry/max_reached') == 2 - assert stats.get_value('retry/count') == 3 - - def _test_retry(self, req, exception): - req = self.mw.process_exception(req, exception, self.spider) - assert isinstance(req, Request) + # SETINGS: RETRY_TIMES < meta(max_retry_times) + self._test_retry(req2, DNSLookupError('foo'), 2) - retry_times = req.meta.get('max_retry_times') or self.mw.max_retry_times + # SETINGS: RETRY_TIMES = 0 + self.mw.max_retry_times = 0 + self._test_retry(req2, DNSLookupError('foo'), 0) + + # SETINGS: RETRY_TIMES > meta(max_retry_times) + self.mw.max_retry_times = 4 + self._test_retry(req2, DNSLookupError('foo'), 4) - while req.meta['retry_times'] != retry_times: + # RESET RETRY_TIMES SETTINGS + self.mw.max_retry_times = 2 + + def _test_retry(self, req, exception, max_retry_times): + + while max_retry_times > 0: req = self.mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) - - self.assertEqual(req.meta['retry_times'], retry_times) + if req.meta['retry_times'] == max_retry_times: + break # discard it req = self.mw.process_exception(req, exception, self.spider) From 9d97d788c06c2e8fbbdfbcfbee65321ac2dfb517 Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Wed, 15 Mar 2017 04:13:47 +0530 Subject: [PATCH 123/362] Update docs for meta key --- docs/topics/downloader-middleware.rst | 5 +++++ docs/topics/request-response.rst | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index c3a454279..b808a6448 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -852,6 +852,11 @@ Default: ``2`` Maximum number of times to retry, in addition to the first download. +.. reqmeta:: max_retry_times + +If :attr:`Request.meta ` has ``max_retry_times`` key +set to some value, this setting will be ignored by this middleware for the corresponding request. + .. setting:: RETRY_HTTP_CODES RETRY_HTTP_CODES diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 67f8ec285..64a1e55fa 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -308,6 +308,7 @@ Those are: * ``ftp_user`` (See :setting:`FTP_USER` for more info) * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * :reqmeta:`referrer_policy` +* :reqmeta:`max_retry_times` .. reqmeta:: bindaddress @@ -342,6 +343,13 @@ download_fail_on_dataloss Whether or not to fail on broken responses. See: :setting:`DOWNLOAD_FAIL_ON_DATALOSS`. +.. reqmeta:: max_retry_times + +max_retry_times +--------------- + +The meta key is used set retry times per request. When initialized, the :setting:`RETRY_TIMES` setting will be ignored by the downloader middleware. + .. _topics-request-response-ref-request-subclasses: Request subclasses From 7ba4b0a21b0bb2efe0523bec0c1db07771816347 Mon Sep 17 00:00:00 2001 From: Bernardas Date: Wed, 15 Mar 2017 07:50:31 +0000 Subject: [PATCH 124/362] add support for embeded ptpython shell --- scrapy/utils/console.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 1888d9599..a9d73aada 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -31,6 +31,15 @@ def _embed_bpython_shell(namespace={}, banner=''): bpython.embed(locals_=namespace, banner=banner) return wrapper +def _embed_ptpython_shell(namespace={}, banner=''): + """Start a ptpython shell""" + import ptpython.repl + @wraps(_embed_ptpython_shell) + def wrapper(namespace=namespace, banner=''): + print(banner) + ptpython.repl.embed(locals=namespace) + return wrapper + def _embed_standard_shell(namespace={}, banner=''): """Start a standard python shell""" import code @@ -49,7 +58,8 @@ def _embed_standard_shell(namespace={}, banner=''): DEFAULT_PYTHON_SHELLS = OrderedDict([ ('ipython', _embed_ipython_shell), ('bpython', _embed_bpython_shell), - ( 'python', _embed_standard_shell), + ('ptpython', _embed_ptpython_shell), + ('python', _embed_standard_shell), ]) def get_shell_embed_func(shells=None, known_shells=None): From a84652e775fda1135fe959465b27cf6ed2c25e1d Mon Sep 17 00:00:00 2001 From: woxcab Date: Wed, 15 Mar 2017 12:39:48 +0300 Subject: [PATCH 125/362] Init tests are split by initializer' input --- tests/test_utils_datatypes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 3a4137942..49323f0ff 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -8,17 +8,19 @@ __doctests__ = ['scrapy.utils.datatypes'] class CaselessDictTest(unittest.TestCase): - def test_init(self): + def test_init_dict(self): seq = {'red': 1, 'black': 3} d = CaselessDict(seq) self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) + def test_init_pair_sequence(self): seq = (('red', 1), ('black', 3)) d = CaselessDict(seq) self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) + def test_init_mapping(self): class MyMapping(Mapping): def __init__(self, **kwargs): self._d = kwargs @@ -37,6 +39,7 @@ class CaselessDictTest(unittest.TestCase): self.assertEqual(d['red'], 1) self.assertEqual(d['black'], 3) + def test_init_mutable_mapping(self): class MyMutableMapping(MutableMapping): def __init__(self, **kwargs): self._d = kwargs From 4345eaf1b640bfecb2340a0e27649b1ab1079da6 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Fri, 17 Mar 2017 08:11:20 +0100 Subject: [PATCH 126/362] [logformatter] backward compat comments --- scrapy/logformatter.py | 1 + tests/test_logformatter.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 2a89c00c5..075a6d862 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -43,6 +43,7 @@ class LogFormatter(object): 'request_flags' : request_flags, 'referer': referer_str(request), 'response_flags': response_flags, + # backward compatibility with Scrapy logformatter below 1.4 version 'flags': response_flags } } diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 52646ec1b..94e6c9fde 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -66,7 +66,6 @@ class LoggingContribTest(unittest.TestCase): class LogFormatterSubclass(LogFormatter): - # Formatter with format spec that is same as in Scrapy before 1.3 version. def crawled(self, request, response, spider): kwargs = super(LogFormatterSubclass, self).crawled( request, response, spider) @@ -82,7 +81,6 @@ class LogFormatterSubclass(LogFormatter): class LogformatterSubclassTest(LoggingContribTest): - # Test if old crawledmsg format string still works fine def setUp(self): self.formatter = LogFormatterSubclass() self.spider = Spider('default') From 49c5afc5ff6c810c78678ea1c86beb19ca3487ad Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Sun, 19 Mar 2017 06:08:35 +0530 Subject: [PATCH 127/362] Fix bug involving OR condition --- scrapy/downloadermiddlewares/retry.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index c22437ff1..07e979628 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -63,7 +63,10 @@ class RetryMiddleware(object): def _retry(self, request, reason, spider): retries = request.meta.get('retry_times', 0) + 1 - retry_times = request.meta.get('max_retry_times') or self.max_retry_times + retry_times = self.max_retry_times + + if 'max_retry_times' in request.meta: + retry_times = request.meta['max_retry_times'] stats = spider.crawler.stats if retries <= retry_times: From 0d9ebd6e1ed58654ea1996d5a236b4d0240590df Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Sun, 19 Mar 2017 06:15:46 +0530 Subject: [PATCH 128/362] Update tests for max_retry_times --- tests/test_downloadermiddleware_retry.py | 76 +++++++++++++++++++----- 1 file changed, 61 insertions(+), 15 deletions(-) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 064c740c9..5f1760fd1 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -103,29 +103,75 @@ class RetryTest(unittest.TestCase): req = self.mw.process_exception(req, exception, self.spider) self.assertEqual(req, None) - def test_different_retry(self): + +class MaxRetryTimesTest(unittest.TestCase): + def setUp(self): + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('foo') + self.mw = RetryMiddleware.from_crawler(self.crawler) + self.mw.max_retry_times = 2 + + def test_without_metakey(self): - req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3}) + req = Request('http://www.scrapytest.org/invalid_url') - # SETINGS: meta(max_retry_times) = 3, RETRY_TIMES = 2 - self._test_retry(req, DNSLookupError('foo'), 3) + # SETTINGS: RETRY_TIMES is NON-ZERO + self.mw.max_retry_times = 5 + self._test_retry(req, DNSLookupError('foo'), 5) - req2 = Request('http://www.scrapytest.org/invalid_url') - - # SETINGS: RETRY_TIMES < meta(max_retry_times) - self._test_retry(req2, DNSLookupError('foo'), 2) - - # SETINGS: RETRY_TIMES = 0 + # SETTINGS: RETRY_TIMES = 0 self.mw.max_retry_times = 0 - self._test_retry(req2, DNSLookupError('foo'), 0) - - # SETINGS: RETRY_TIMES > meta(max_retry_times) - self.mw.max_retry_times = 4 - self._test_retry(req2, DNSLookupError('foo'), 4) + self._test_retry(req, DNSLookupError('foo'), 0) # RESET RETRY_TIMES SETTINGS self.mw.max_retry_times = 2 + def test_with_metakey_preceding(self): + # request with meta(max_retry_times) is called first + + req1 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3}) + req2 = Request('http://www.scrapytest.org/invalid_url') + req3 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 4}) + + # SETINGS: RETRY_TIMES < meta(max_retry_times) + self.mw.max_retry_times = 2 + self._test_retry(req1, DNSLookupError('foo'), 3) + self._test_retry(req2, DNSLookupError('foo'), 2) + + # SETINGS: RETRY_TIMES > meta(max_retry_times) + self.mw.max_retry_times = 5 + self._test_retry(req3, DNSLookupError('foo'), 4) + self._test_retry(req2, DNSLookupError('foo'), 5) + + # RESET RETRY_TIMES SETTINGS + self.mw.max_retry_times = 2 + + def test_with_metakey_succeeding(self): + # request with meta(max_retry_times) is called second + + req1 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3}) + req2 = Request('http://www.scrapytest.org/invalid_url') + req3 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 4}) + + # SETINGS: RETRY_TIMES < meta(max_retry_times) + self.mw.max_retry_times = 2 + self._test_retry(req2, DNSLookupError('foo'), 2) + self._test_retry(req1, DNSLookupError('foo'), 3) + + # SETINGS: RETRY_TIMES > meta(max_retry_times) + self.mw.max_retry_times = 5 + self._test_retry(req2, DNSLookupError('foo'), 5) + self._test_retry(req3, DNSLookupError('foo'), 4) + + # RESET RETRY_TIMES SETTINGS + self.mw.max_retry_times = 2 + + def test_with_metakey_zero(self): + + req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 0}) + self._test_retry(req, DNSLookupError('foo'), 0) + + def _test_retry(self, req, exception, max_retry_times): while max_retry_times > 0: From 10741aca720293a12dedda4d1872cf0604b49f0b Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Sun, 19 Mar 2017 06:17:28 +0530 Subject: [PATCH 129/362] Update docs - improve clarity --- docs/topics/downloader-middleware.rst | 8 ++++---- docs/topics/request-response.rst | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index b808a6448..0d168017f 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -852,10 +852,10 @@ Default: ``2`` Maximum number of times to retry, in addition to the first download. -.. reqmeta:: max_retry_times - -If :attr:`Request.meta ` has ``max_retry_times`` key -set to some value, this setting will be ignored by this middleware for the corresponding request. +Maximum number of retries can also be specified per-request using +:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. +When initialized, the :reqmeta:`max_retry_times` meta key takes higher +precedence over the :setting:`RETRY_TIMES` setting. .. setting:: RETRY_HTTP_CODES diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 64a1e55fa..03918fd2d 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -348,7 +348,9 @@ Whether or not to fail on broken responses. See: max_retry_times --------------- -The meta key is used set retry times per request. When initialized, the :setting:`RETRY_TIMES` setting will be ignored by the downloader middleware. +The meta key is used set retry times per request. When initialized, the +:reqmeta:`max_retry_times` meta key takes higher precedence over the +:setting:`RETRY_TIMES` setting. .. _topics-request-response-ref-request-subclasses: From 605691792f5198d75524a1ad952489193970d4a7 Mon Sep 17 00:00:00 2001 From: Oto Brglez Date: Sun, 19 Mar 2017 12:35:39 +0100 Subject: [PATCH 130/362] Updating media-pipeline docs for S3-like storage. --- docs/topics/media-pipeline.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 82c0aaa88..733a7fe2b 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -169,6 +169,18 @@ policy:: For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. +Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like +self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings:: + + AWS_ENDPOINT_URL = 'http://minio.example.com:9000' + +For self-hosting you also might feel the need not to use SSL and not to verify SSL connection:: + + AWS_USE_SSL = False # or True (None by default) + AWS_VERIFY = False # or True (None by default) + +.. _Minio: https://github.com/minio/minio +.. _s3.scality: https://s3.scality.com/ .. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl Usage example From 11cf6ad4258250b8a2410029eb794ed892dc4cea Mon Sep 17 00:00:00 2001 From: Oto Brglez Date: Sun, 19 Mar 2017 12:48:06 +0100 Subject: [PATCH 131/362] Comments for AWS_ENDPOINT_URL setting. --- docs/topics/settings.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ccdd02c4e..3ac3fd5af 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -180,6 +180,34 @@ such as the :ref:`S3 feed storage backend `. .. setting:: BOT_NAME +AWS_ENDPOINT_URL +---------------- + +Default: ``None`` + +Endpoint URL used for S3-like self-hosted storage. Storage like Minio or s3.scality. + +.. setting:: AWS_ENDPOINT_URL + +AWS_USE_SSL +----------- + +Default: ``None`` + +Use this option if you want to disable SSL connection for communication with S3 or S3-like storage. +By default SSL will be used. + +.. setting:: AWS_USE_SSL + +AWS_VERIFY +---------- + +Default: ``None`` + +Verify SSL connection between Scrapy and S3 or S3-like storage. By default SSL verification will occur. + +.. setting:: AWS_VERIFY + BOT_NAME -------- From a57e49d55b7d703c8e5153811d6f5de3eacd2272 Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Mon, 20 Mar 2017 19:49:31 +0530 Subject: [PATCH 132/362] Add sphinx_rtd_theme to docs setup readme --- docs/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README.rst b/docs/README.rst index 733af2af4..af8bf4297 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -15,7 +15,7 @@ and all its dependencies run :: - pip install 'Sphinx >= 1.3' + pip install 'Sphinx >= 1.3' sphinx_rtd_theme Compile the documentation From 4ec07ae7640df6fdb13a24a4265232fd11e53755 Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Mon, 20 Mar 2017 22:21:08 +0530 Subject: [PATCH 133/362] Create docs/requirements.txt --- docs/README.rst | 2 +- docs/requirements.txt | 2 ++ tox.ini | 3 +-- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 docs/requirements.txt diff --git a/docs/README.rst b/docs/README.rst index af8bf4297..f6011b2c6 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -15,7 +15,7 @@ and all its dependencies run :: - pip install 'Sphinx >= 1.3' sphinx_rtd_theme + pip install -r requirements.txt Compile the documentation diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..44e97ceb1 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +'Sphinx >= 1.3' +sphinx_rtd_theme \ No newline at end of file diff --git a/tox.ini b/tox.ini index bbf50b733..6987847f8 100644 --- a/tox.ini +++ b/tox.ini @@ -82,8 +82,7 @@ deps = {[testenv:py33]deps} [docs] changedir = docs deps = - Sphinx - sphinx_rtd_theme + -rdocs/requirements.txt [testenv:docs] changedir = {[docs]changedir} From 83aa0c5e1a971c07d0beab89fde91ce1184f098c Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Mon, 20 Mar 2017 22:36:09 +0530 Subject: [PATCH 134/362] Clarify docs readme --- docs/README.rst | 2 +- docs/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/README.rst b/docs/README.rst index f6011b2c6..0a343cd19 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -11,7 +11,7 @@ Setup the environment --------------------- To compile the documentation you need Sphinx Python library. To install it -and all its dependencies run +and all its dependencies run the following command from this dir :: diff --git a/docs/requirements.txt b/docs/requirements.txt index 44e97ceb1..d3dcb97be 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -'Sphinx >= 1.3' +Sphinx>=1.3 sphinx_rtd_theme \ No newline at end of file From c5f74f7d1a5245b65e5c7863b02a379667bee0dd Mon Sep 17 00:00:00 2001 From: Qiwei Huang Date: Mon, 20 Mar 2017 18:52:33 -0700 Subject: [PATCH 135/362] Update spiders.rst Added a note to allowed_domains attribute, reminding users not to add urls into the list. --- docs/topics/spiders.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 5e69055d1..9e27614d8 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -80,6 +80,8 @@ scrapy.Spider allowed to crawl. Requests for URLs not belonging to the domain names specified in this list (or their subdomains) won't be followed if :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled. + .. note:: If you are scraping an url ``https://www.example.com/1.html`` + you should add ``'example.com'`` to allowed_domains list. .. attribute:: start_urls From 8ecc307e8f04b058f2fc8a1a47759203ff6ebca0 Mon Sep 17 00:00:00 2001 From: Qiwei Huang Date: Mon, 20 Mar 2017 19:37:07 -0700 Subject: [PATCH 136/362] Update spiders.rst Added note to allowed_domain attribute with an example explaining what goes in the list --- docs/topics/spiders.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 9e27614d8..49c0cefb5 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -80,8 +80,9 @@ scrapy.Spider allowed to crawl. Requests for URLs not belonging to the domain names specified in this list (or their subdomains) won't be followed if :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled. - .. note:: If you are scraping an url ``https://www.example.com/1.html`` - you should add ``'example.com'`` to allowed_domains list. + + Let's say your target url is ``https://www.example.com/1.html``, + then add ``'example.com'`` to the list. .. attribute:: start_urls From 21d794d35ae1347918e6bf5b2ffe13515ea28795 Mon Sep 17 00:00:00 2001 From: Simon Diviani Gartz Date: Tue, 21 Mar 2017 17:04:30 +0100 Subject: [PATCH 137/362] Fixes conversion of transparent PNG with palette images to jpg #2452 --- scrapy/pipelines/images.py | 5 +++++ tests/test_pipeline_images.py | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 5796bfb80..bc449431f 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -132,6 +132,11 @@ class ImagesPipeline(FilesPipeline): background = Image.new('RGBA', image.size, (255, 255, 255)) background.paste(image, image) image = background.convert('RGB') + elif image.mode == 'P': + image = image.convert("RGBA") + background = Image.new('RGBA', image.size, (255, 255, 255)) + background.paste(image, image) + image = background.convert('RGB') elif image.mode != 'RGB': image = image.convert('RGB') diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 342f25ea9..0f3047602 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -96,6 +96,14 @@ class ImagesPipelineTestCase(unittest.TestCase): self.assertEquals(converted.mode, 'RGB') self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))]) + # transparency case with palette: P and PNG + COLOUR = (0, 127, 255, 50) + im = _create_image('PNG', 'RGBA', SIZE, COLOUR) + im = im.convert('P') + converted, _ = self.pipeline.convert_image(im) + self.assertEquals(converted.mode, 'RGB') + self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))]) + class DeprecatedImagesPipeline(ImagesPipeline): def file_key(self, url): From 99e3c0d653e23d1af3e4236b88a747317e1c8a8a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 23 Mar 2017 11:52:01 +0100 Subject: [PATCH 138/362] Set bodyproducer with empty content for POST --- scrapy/core/downloader/handlers/http11.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 37e836809..bff4a30c9 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -280,19 +280,7 @@ class ScrapyAgent(object): if request.body: bodyproducer = _RequestBodyProducer(request.body) else: - bodyproducer = None - # Setting Content-Length: 0 even for POST requests is not a - # MUST per HTTP RFCs, but it's common behavior, and some - # servers require this, otherwise returning HTTP 411 Length required - # - # RFC 7230#section-3.3.2: - # "a Content-Length header field is normally sent in a POST - # request even when the value is 0 (indicating an empty payload body)." - # - # Twisted Agent will not add "Content-Length: 0" by itself - if method == b'POST': - headers.addRawHeader(b'Content-Length', b'0') - + bodyproducer = _RequestBodyProducer(b'') if method == b'POST' else None start_time = time() d = agent.request( method, to_bytes(url, encoding='ascii'), headers, bodyproducer) From 38e6857c957ef023533128f054688152be223c87 Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Thu, 23 Mar 2017 19:45:04 +0530 Subject: [PATCH 139/362] Improvise the clarity of test cases --- tests/test_downloadermiddleware_retry.py | 105 +++++++++++------------ 1 file changed, 51 insertions(+), 54 deletions(-) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 5f1760fd1..51b79b6c3 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -110,75 +110,72 @@ class MaxRetryTimesTest(unittest.TestCase): self.spider = self.crawler._create_spider('foo') self.mw = RetryMiddleware.from_crawler(self.crawler) self.mw.max_retry_times = 2 + self.invalid_url = 'http://www.scrapytest.org/invalid_url' - def test_without_metakey(self): - - req = Request('http://www.scrapytest.org/invalid_url') - - # SETTINGS: RETRY_TIMES is NON-ZERO - self.mw.max_retry_times = 5 - self._test_retry(req, DNSLookupError('foo'), 5) + def test_with_settings_zero(self): # SETTINGS: RETRY_TIMES = 0 self.mw.max_retry_times = 0 - self._test_retry(req, DNSLookupError('foo'), 0) - # RESET RETRY_TIMES SETTINGS - self.mw.max_retry_times = 2 - - def test_with_metakey_preceding(self): - # request with meta(max_retry_times) is called first - - req1 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3}) - req2 = Request('http://www.scrapytest.org/invalid_url') - req3 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 4}) - - # SETINGS: RETRY_TIMES < meta(max_retry_times) - self.mw.max_retry_times = 2 - self._test_retry(req1, DNSLookupError('foo'), 3) - self._test_retry(req2, DNSLookupError('foo'), 2) - - # SETINGS: RETRY_TIMES > meta(max_retry_times) - self.mw.max_retry_times = 5 - self._test_retry(req3, DNSLookupError('foo'), 4) - self._test_retry(req2, DNSLookupError('foo'), 5) - - # RESET RETRY_TIMES SETTINGS - self.mw.max_retry_times = 2 - - def test_with_metakey_succeeding(self): - # request with meta(max_retry_times) is called second - - req1 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 3}) - req2 = Request('http://www.scrapytest.org/invalid_url') - req3 = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 4}) - - # SETINGS: RETRY_TIMES < meta(max_retry_times) - self.mw.max_retry_times = 2 - self._test_retry(req2, DNSLookupError('foo'), 2) - self._test_retry(req1, DNSLookupError('foo'), 3) - - # SETINGS: RETRY_TIMES > meta(max_retry_times) - self.mw.max_retry_times = 5 - self._test_retry(req2, DNSLookupError('foo'), 5) - self._test_retry(req3, DNSLookupError('foo'), 4) - - # RESET RETRY_TIMES SETTINGS - self.mw.max_retry_times = 2 + req = Request(self.invalid_url) + self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times) def test_with_metakey_zero(self): + + # SETTINGS: meta(max_retry_times) = 0 + meta_max_retry_times = 0 - req = Request('http://www.scrapytest.org/invalid_url', meta={'max_retry_times': 0}) + req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) + self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times) + + def test_without_metakey(self): + + # SETTINGS: RETRY_TIMES is NON-ZERO + self.mw.max_retry_times = 5 + + req = Request(self.invalid_url) + self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times) + + def test_with_metakey_greater(self): + + # SETINGS: RETRY_TIMES < meta(max_retry_times) + self.mw.max_retry_times = 2 + meta_max_retry_times = 3 + + req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) + req2 = Request(self.invalid_url) + + self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times) + self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) + + def test_with_metakey_lesser(self): + + # SETINGS: RETRY_TIMES > meta(max_retry_times) + self.mw.max_retry_times = 5 + meta_max_retry_times = 4 + + req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) + req2 = Request(self.invalid_url) + + self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times) + self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) + + def test_with_dont_retry(self): + + # SETTINGS: meta(max_retry_times) = 4 + meta_max_retry_times = 4 + + req = Request(self.invalid_url, meta= \ + {'max_retry_times': meta_max_retry_times, 'dont_retry': True}) + self._test_retry(req, DNSLookupError('foo'), 0) def _test_retry(self, req, exception, max_retry_times): - while max_retry_times > 0: + for i in range(0, max_retry_times): req = self.mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) - if req.meta['retry_times'] == max_retry_times: - break # discard it req = self.mw.process_exception(req, exception, self.spider) From 0298bcbe79ec9c010282eb59d79848f17bada7ed Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Fri, 24 Mar 2017 18:13:08 +0530 Subject: [PATCH 140/362] Update Makefile to open webbrowser in MacOS (#2661) --- docs/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index eaba3ba2b..a3d1611f9 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -82,7 +82,8 @@ pydoc-topics: build "into the Lib/ directory" htmlview: html - $(PYTHON) -c "import webbrowser; webbrowser.open('build/html/index.html')" + $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \ + os.path.realpath('build/html/index.html'))" clean: -rm -rf build/* From 2ff6b0572318742f1b16ee7f9f0c3b836b020633 Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Fri, 24 Mar 2017 20:43:28 +0530 Subject: [PATCH 141/362] Remove __nonzero__ from SelectorList docs --- docs/topics/selectors.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 8a5d44aac..61206a193 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -714,10 +714,6 @@ SelectorList objects Call the ``.re()`` method for each element in this list and return their results flattened, as a list of unicode strings. - .. method:: __nonzero__() - - returns True if the list is not empty, False otherwise. - Selector examples on HTML response ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 163618c9b792ded60e451e929341bb502d9cc19a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 29 Mar 2017 12:02:44 +0200 Subject: [PATCH 142/362] FAQ Rewrite note on Python 3 support on Windows --- docs/faq.rst | 3 ++- docs/intro/install.rst | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index ad11b071b..f0ee20b5e 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -74,7 +74,8 @@ Python 2.6 support was dropped starting at Scrapy 0.20. Python 3 support was added in Scrapy 1.1. .. note:: - Python 3 is not yet supported on Windows. + For Python 3 support on Windows, it is recommended to use + Anaconda/Miniconda as :ref:`outlined in the installation guide `. Did Scrapy "steal" X from Django? --------------------------------- diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 86387ef5e..9cec2eaee 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -116,6 +116,8 @@ Python virtualenvs can be created to use Python 2 by default, or Python 3 by def Platform specific installation notes ==================================== +.. _intro-install-windows: + Windows ------- @@ -128,6 +130,8 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: conda install -c conda-forge scrapy +.. _intro-install-ubuntu: + Ubuntu 12.04 or above --------------------- @@ -163,6 +167,8 @@ you can install Scrapy with ``pip`` after that:: Wheezy (7.0) and above. +.. _intro-install-macos: + Mac OS X -------- From 6352c2e9b2028473acdbd58175bbc5638258e29d Mon Sep 17 00:00:00 2001 From: LMKight Date: Sun, 2 Apr 2017 15:11:13 +0200 Subject: [PATCH 143/362] fixed command list --- scrapy/cmdline.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index cb7bbd64d..05b0a12e0 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -27,8 +27,9 @@ def _get_commands_from_module(module, inproject): d = {} for cmd in _iter_command_classes(module): if inproject or not cmd.requires_project: - cmdname = cmd.__module__.split('.')[-1] - d[cmdname] = cmd() + if not cmd.__module__ == module: + cmdname = cmd.__module__.split('.')[-1] + d[cmdname] = cmd() return d def _get_commands_from_entry_points(inproject, group='scrapy.commands'): From 05ce1296c6a60f23e81af7ec38ac1855e78be79f Mon Sep 17 00:00:00 2001 From: LMKight Date: Mon, 3 Apr 2017 19:47:01 +0200 Subject: [PATCH 144/362] changed code according to request --- scrapy/cmdline.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 05b0a12e0..8edc1ad2d 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -20,16 +20,16 @@ def _iter_command_classes(module_name): for obj in vars(module).values(): if inspect.isclass(obj) and \ issubclass(obj, ScrapyCommand) and \ - obj.__module__ == module.__name__: + obj.__module__ == module.__name__ and \ + not obj == ScrapyCommand: yield obj def _get_commands_from_module(module, inproject): d = {} for cmd in _iter_command_classes(module): if inproject or not cmd.requires_project: - if not cmd.__module__ == module: - cmdname = cmd.__module__.split('.')[-1] - d[cmdname] = cmd() + cmdname = cmd.__module__.split('.')[-1] + d[cmdname] = cmd() return d def _get_commands_from_entry_points(inproject, group='scrapy.commands'): From 422b38f65ccbbe4479b6249c89bc57bd2d22d092 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 11 Apr 2017 16:55:43 +0200 Subject: [PATCH 145/362] DOC Rearrange selector sections --- docs/topics/selectors.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 8a5d44aac..68d8120a8 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -593,6 +593,9 @@ Built-in Selectors reference .. module:: scrapy.selector :synopsis: Selector class +Selector objects +---------------- + .. class:: Selector(response=None, text=None, type=None) An instance of :class:`Selector` is a wrapper over response to select @@ -720,7 +723,7 @@ SelectorList objects Selector examples on HTML response -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------- Here's a couple of :class:`Selector` examples to illustrate several concepts. In all cases, we assume there is already a :class:`Selector` instantiated with @@ -745,7 +748,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this:: print node.xpath("@class").extract() Selector examples on XML response -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------------- Here's a couple of examples to illustrate several concepts. In both cases we assume there is already a :class:`Selector` instantiated with an @@ -767,7 +770,7 @@ assume there is already a :class:`Selector` instantiated with an .. _removing-namespaces: Removing namespaces -~~~~~~~~~~~~~~~~~~~ +------------------- When dealing with scraping projects, it is often quite convenient to get rid of namespaces altogether and just work with element names, to write more From f3f7a4186150119377891069a3155cea8cabb142 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 12 Apr 2017 16:32:21 +0200 Subject: [PATCH 146/362] Travis CI: use portable pypy for Linux --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2df02ea43..906115096 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,8 +33,8 @@ install: else rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT" fi - # get latest PyPy from pyenv directly (thanks to natural version sort option -V) - export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-[0-9][\.0-9]*$' |sort -V |tail -1` + # get latest portable PyPy from pyenv directly (thanks to natural version sort option -V) + export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-portable-[0-9][\.0-9]*$' |sort -V |tail -1` "$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION" virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" From 30eec559104649653d4c0e182fb2d3e2252dca1e Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Sat, 22 Apr 2017 00:24:18 +0200 Subject: [PATCH 147/362] [PEDANTIC] FIX trailing whitespaces in LICENSE. --- LICENSE | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/LICENSE b/LICENSE index 68ccf9762..6ead05ece 100644 --- a/LICENSE +++ b/LICENSE @@ -4,10 +4,10 @@ All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright + + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. From 00ee9eaeafcd3b3af43132c4513754627f97a886 Mon Sep 17 00:00:00 2001 From: Tiago Cardoso Date: Sat, 22 Apr 2017 14:36:44 +0100 Subject: [PATCH 148/362] Mention how to disable request filtering in documentation of DUPEFILTER_CLASS setting --- docs/topics/settings.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8367b1092..9bf07588b 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -646,6 +646,13 @@ override its ``request_fingerprint`` method. This method should accept scrapy :class:`~scrapy.http.Request` object and return its fingerprint (a string). +You can disable filtering of duplicate requests by setting +:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``. +Be very careful about this however, because you can get into crawling loops. +It's usually a better idea to set the ``dont_filter`` parameter to +``True`` on the specific :class:`~scrapy.http.Request` that should not be +filtered. + .. setting:: DUPEFILTER_DEBUG DUPEFILTER_DEBUG From 97fc68fa1699b1c782c6e6d888e21d995bdf071d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 24 Apr 2017 22:08:39 +0200 Subject: [PATCH 149/362] Refactor conditions on body producer --- scrapy/core/downloader/handlers/http11.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index bff4a30c9..46493f87f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -279,8 +279,10 @@ class ScrapyAgent(object): headers.removeHeader(b'Proxy-Authorization') if request.body: bodyproducer = _RequestBodyProducer(request.body) + elif method == b'POST': + bodyproducer = _RequestBodyProducer(b'') else: - bodyproducer = _RequestBodyProducer(b'') if method == b'POST' else None + bodyproducer = None start_time = time() d = agent.request( method, to_bytes(url, encoding='ascii'), headers, bodyproducer) From b1a0a6e25810353f46d314cea1fd34bd37109b0c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 25 Apr 2017 17:01:54 +0200 Subject: [PATCH 150/362] Make mockserver runnable outside of tox Add POST support for Echo resource --- tests/mockserver.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 26ab51183..b95a6c3c4 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -15,7 +15,6 @@ from twisted.internet.task import deferLater from scrapy.utils.python import to_bytes, to_unicode -from tests import tests_datadir def getarg(request, name, default=None, type=None): @@ -122,6 +121,7 @@ class Echo(LeafResource): 'body': to_unicode(request.content.read()), } return to_bytes(json.dumps(output)) + render_POST = render_GET class RedirectTo(LeafResource): @@ -174,7 +174,11 @@ class Root(Resource): self.putChild(b"echo", Echo()) self.putChild(b"payload", PayloadResource()) self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) - self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/'))) + try: + from tests import tests_datadir + self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/'))) + except: + pass self.putChild(b"redirect-to", RedirectTo()) def getChild(self, name, request): From a63d9f502f50fbd948154fc65c8a94ffd4722d11 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 25 Apr 2017 17:03:03 +0200 Subject: [PATCH 151/362] Restore comments on why POST needs `Content-Length: 0` --- scrapy/core/downloader/handlers/http11.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 46493f87f..55bd31303 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -280,6 +280,18 @@ class ScrapyAgent(object): if request.body: bodyproducer = _RequestBodyProducer(request.body) elif method == b'POST': + # Setting Content-Length: 0 even for POST requests is not a + # MUST per HTTP RFCs, but it's common behavior, and some + # servers require this, otherwise returning HTTP 411 Length required + # + # RFC 7230#section-3.3.2: + # "a Content-Length header field is normally sent in a POST + # request even when the value is 0 (indicating an empty payload body)." + # + # Twisted < 17 will not add "Content-Length: 0" by itself; + # Twisted >= 17 fixes this; + # Using a producer with an empty-string sends `0` as Content-Length + # for all versions of Twisted. bodyproducer = _RequestBodyProducer(b'') else: bodyproducer = None From c3d0f9b6c10b68e436054ce9421d2ddadfc47087 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 25 Apr 2017 17:03:41 +0200 Subject: [PATCH 152/362] Add test for non-duplicated `Content-Length: 0` for bodyless POST --- tests/test_downloader_handlers.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 3efcf6e9c..0f28037ba 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -29,7 +29,7 @@ from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spiders import Spider -from scrapy.http import Request +from scrapy.http import Headers, Request from scrapy.http.response.text import TextResponse from scrapy.responsetypes import responsetypes from scrapy.settings import Settings @@ -37,7 +37,7 @@ from scrapy.utils.test import get_crawler, skip_if_no_boto from scrapy.utils.python import to_bytes from scrapy.exceptions import NotConfigured -from tests.mockserver import MockServer, ssl_context_factory +from tests.mockserver import MockServer, ssl_context_factory, Echo from tests.spiders import SingleRequestSpider class DummyDH(object): @@ -202,6 +202,7 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"broken-chunked", BrokenChunkedResource()) r.putChild(b"contentlength", ContentLengthHeaderResource()) r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) + r.putChild(b"echo", Echo()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.host = 'localhost' @@ -310,6 +311,17 @@ class HttpTestCase(unittest.TestCase): request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'}) return self.download_request(request, Spider('foo')).addCallback(_test) + def test_content_length_zero_bodyless_post_only_one(self): + def _test(response): + import json + headers = Headers(json.loads(response.text)['headers']) + contentlengths = headers.getlist('Content-Length') + self.assertEquals(len(contentlengths), 1) + self.assertEquals(contentlengths, [b"0"]) + + request = Request(self.getURL('echo'), method='POST') + return self.download_request(request, Spider('foo')).addCallback(_test) + def test_payload(self): body = b'1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) From 4bc0c6b0f4c5dae033bd83549b6d0a573fcf4805 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Gait=C3=A1n?= Date: Tue, 25 Apr 2017 22:33:22 -0300 Subject: [PATCH 153/362] Update practices.rst fix a typo --- docs/topics/practices.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 25ae4b5ba..63913d3c4 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -238,7 +238,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * if possible, use `Google cache`_ to fetch pages, instead of hitting the sites directly * use a pool of rotating IPs. For example, the free `Tor project`_ or paid - services like `ProxyMesh`_. An open source alterantive is `scrapoxy`_, a + services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. * use a highly distributed downloader that circumvents bans internally, so you can just focus on parsing clean pages. One example of such downloaders is From 2b34c6edffcf14d81b5f11369648f4661e067ccc Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Sat, 4 Mar 2017 21:29:24 -0300 Subject: [PATCH 154/362] Abort connection earlier and avoid to buffer data A symptom of this issue was having the log message "Received (X) bytes larger than download max size (Y)" several times printed, with increased X values. --- scrapy/core/downloader/handlers/http11.py | 11 ++++++++- tests/test_downloader_handlers.py | 30 +++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 55bd31303..9bfdd803c 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -348,7 +348,8 @@ class ScrapyAgent(object): {'size': expected_size, 'warnsize': warnsize}) def _cancel(_): - txresponse._transport._producer.loseConnection() + # Abort connection inmediately. + txresponse._transport._producer.abortConnection() d = defer.Deferred(_cancel) txresponse.deliverBody(_ResponseReader( @@ -401,6 +402,11 @@ class _ResponseReader(protocol.Protocol): self._bytes_received = 0 def dataReceived(self, bodyBytes): + # This maybe called several times after cancel was called with buffered + # data. + if self._finished.called: + return + self._bodybuf.write(bodyBytes) self._bytes_received += len(bodyBytes) @@ -409,6 +415,9 @@ class _ResponseReader(protocol.Protocol): "max size (%(maxsize)s).", {'bytes': self._bytes_received, 'maxsize': self._maxsize}) + # Clear buffer earlier to avoid keeping data in memory for a long + # time. + self._bodybuf.truncate(0) self._finished.cancel() if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize: diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0f28037ba..b52dac499 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -177,6 +177,16 @@ class EmptyContentTypeHeaderResource(resource.Resource): return request.content.read() +class LargeChunkedFileResource(resource.Resource): + def render(self, request): + def response(): + for i in range(1024): + request.write(b"x" * 1024) + request.finish() + reactor.callLater(0, response) + return server.NOT_DONE_YET + + class HttpTestCase(unittest.TestCase): scheme = 'http' @@ -202,6 +212,7 @@ class HttpTestCase(unittest.TestCase): r.putChild(b"broken-chunked", BrokenChunkedResource()) r.putChild(b"contentlength", ContentLengthHeaderResource()) r.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) + r.putChild(b"largechunkedfile", LargeChunkedFileResource()) r.putChild(b"echo", Echo()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) @@ -384,6 +395,25 @@ class Http11TestCase(HttpTestCase): d = self.download_request(request, Spider('foo', download_maxsize=9)) yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + @defer.inlineCallbacks + def test_download_with_maxsize_very_large_file(self): + with mock.patch('scrapy.core.downloader.handlers.http11.logger') as logger: + request = Request(self.getURL('largechunkedfile')) + + def check(logger): + logger.error.assert_called_once_with(mock.ANY, mock.ANY) + + d = self.download_request(request, Spider('foo', download_maxsize=1500)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + + # As the error message is logged in the dataReceived callback, we + # have to give a bit of time to the reactor to process the queue + # after closing the connection. + d = defer.Deferred() + d.addCallback(check) + reactor.callLater(.1, d.callback, logger) + yield d + @defer.inlineCallbacks def test_download_with_maxsize_per_req(self): meta = {'download_maxsize': 2} From e6ab8bc9a5460c5af29a330598f7afa5a6efb7e2 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 27 Apr 2017 23:10:52 +0200 Subject: [PATCH 155/362] Change "localhost" test server certificate --- tests/keys/localhost.crt | 20 ++++++++++++++++++++ tests/keys/localhost.gen.README | 21 +++++++++++++++++++++ tests/keys/localhost.key | 28 ++++++++++++++++++++++++++++ tests/test_downloader_handlers.py | 4 ++-- 4 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 tests/keys/localhost.crt create mode 100644 tests/keys/localhost.gen.README create mode 100644 tests/keys/localhost.key diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt new file mode 100644 index 000000000..13c5b5bd6 --- /dev/null +++ b/tests/keys/localhost.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x +NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K +o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y +njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY +ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/ +8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52 +gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O +BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j +xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES +9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i +jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+ +TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own +tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM +am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U +Mjybr/hQK3mnVdE= +-----END CERTIFICATE----- diff --git a/tests/keys/localhost.gen.README b/tests/keys/localhost.gen.README new file mode 100644 index 000000000..19c29a725 --- /dev/null +++ b/tests/keys/localhost.gen.README @@ -0,0 +1,21 @@ +$ openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt +Generating a 2048 bit RSA private key +...................................................................................................+++ +.....+++ +writing new private key to 'localhost.key' +----- +You are about to be asked to enter information that will be incorporated +into your certificate request. +What you are about to enter is what is called a Distinguished Name or a DN. +There are quite a few fields but you can leave some blank +For some fields there will be a default value, +If you enter '.', the field will be left blank. +----- +Country Name (2 letter code) [AU]:IE +State or Province Name (full name) [Some-State]:. +Locality Name (eg, city) []:. +Organization Name (eg, company) [Internet Widgits Pty Ltd]:Scrapy +Organizational Unit Name (eg, section) []:. +Common Name (e.g. server FQDN or YOUR name) []:localhost +Email Address []:. + diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key new file mode 100644 index 000000000..da975e6d3 --- /dev/null +++ b/tests/keys/localhost.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5 +ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4 +JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz +vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2 +RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY +b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/ +XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0 +gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk +i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu +Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss +ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4 +hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD +7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa +EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf +uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG +HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR +UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO +j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri +fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ +6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI +xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g +DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f +npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM +8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6 +J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj +Z+pMCFLbwAbFZ9Qb/NFlUv0= +-----END PRIVATE KEY----- diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0f28037ba..c138bec80 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -183,8 +183,8 @@ class HttpTestCase(unittest.TestCase): download_handler_cls = HTTPDownloadHandler # only used for HTTPS tests - keyfile = 'keys/cert.pem' - certfile = 'keys/cert.pem' + keyfile = 'keys/localhost.key' + certfile = 'keys/localhost.crt' def setUp(self): self.tmpname = self.mktemp() From 6d14e392f1096d94669c00456855b64164dba6bd Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 27 Apr 2017 23:35:01 +0200 Subject: [PATCH 156/362] Remove old test certificate+key --- tests/keys/cert.pem | 36 ------------------------------------ tests/mockserver.py | 2 +- 2 files changed, 1 insertion(+), 37 deletions(-) delete mode 100644 tests/keys/cert.pem diff --git a/tests/keys/cert.pem b/tests/keys/cert.pem deleted file mode 100644 index 65478765e..000000000 --- a/tests/keys/cert.pem +++ /dev/null @@ -1,36 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDBjCCAm+gAwIBAgIBATANBgkqhkiG9w0BAQQFADB7MQswCQYDVQQGEwJTRzER -MA8GA1UEChMITTJDcnlwdG8xFDASBgNVBAsTC00yQ3J5cHRvIENBMSQwIgYDVQQD -ExtNMkNyeXB0byBDZXJ0aWZpY2F0ZSBNYXN0ZXIxHTAbBgkqhkiG9w0BCQEWDm5n -cHNAcG9zdDEuY29tMB4XDTAwMDkxMDA5NTEzMFoXDTAyMDkxMDA5NTEzMFowUzEL -MAkGA1UEBhMCU0cxETAPBgNVBAoTCE0yQ3J5cHRvMRIwEAYDVQQDEwlsb2NhbGhv -c3QxHTAbBgkqhkiG9w0BCQEWDm5ncHNAcG9zdDEuY29tMFwwDQYJKoZIhvcNAQEB -BQADSwAwSAJBAKy+e3dulvXzV7zoTZWc5TzgApr8DmeQHTYC8ydfzH7EECe4R1Xh -5kwIzOuuFfn178FBiS84gngaNcrFi0Z5fAkCAwEAAaOCAQQwggEAMAkGA1UdEwQC -MAAwLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRpZmljYXRl -MB0GA1UdDgQWBBTPhIKSvnsmYsBVNWjj0m3M2z0qVTCBpQYDVR0jBIGdMIGagBT7 -hyNp65w6kxXlxb8pUU/+7Sg4AaF/pH0wezELMAkGA1UEBhMCU0cxETAPBgNVBAoT -CE0yQ3J5cHRvMRQwEgYDVQQLEwtNMkNyeXB0byBDQTEkMCIGA1UEAxMbTTJDcnlw -dG8gQ2VydGlmaWNhdGUgTWFzdGVyMR0wGwYJKoZIhvcNAQkBFg5uZ3BzQHBvc3Qx -LmNvbYIBADANBgkqhkiG9w0BAQQFAAOBgQA7/CqT6PoHycTdhEStWNZde7M/2Yc6 -BoJuVwnW8YxGO8Sn6UJ4FeffZNcYZddSDKosw8LtPOeWoK3JINjAk5jiPQ2cww++ -7QGG/g5NDjxFZNDJP1dGiLAxPW6JXwov4v0FmdzfLOZ01jDcgQQZqEpYlgpuI5JE -WUQ9Ho4EzbYCOQ== ------END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIIBPAIBAAJBAKy+e3dulvXzV7zoTZWc5TzgApr8DmeQHTYC8ydfzH7EECe4R1Xh -5kwIzOuuFfn178FBiS84gngaNcrFi0Z5fAkCAwEAAQJBAIqm/bz4NA1H++Vx5Ewx -OcKp3w19QSaZAwlGRtsUxrP7436QjnREM3Bm8ygU11BjkPVmtrKm6AayQfCHqJoT -ZIECIQDW0BoMoL0HOYM/mrTLhaykYAVqgIeJsPjvkEhTFXWBuQIhAM3deFAvWNu4 -nklUQ37XsCT2c9tmNt1LAT+slG2JOTTRAiAuXDtC/m3NYVwyHfFm+zKHRzHkClk2 -HjubeEgjpj32AQIhAJqMGTaZVOwevTXvvHwNEH+vRWsAYU/gbx+OQB+7VOcBAiEA -oolb6NMg/R3enNPvS1O4UU1H8wpaF77L4yiSWlE0p4w= ------END RSA PRIVATE KEY----- ------BEGIN CERTIFICATE REQUEST----- -MIIBDTCBuAIBADBTMQswCQYDVQQGEwJTRzERMA8GA1UEChMITTJDcnlwdG8xEjAQ -BgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3DQEJARYObmdwc0Bwb3N0MS5jb20w -XDANBgkqhkiG9w0BAQEFAANLADBIAkEArL57d26W9fNXvOhNlZzlPOACmvwOZ5Ad -NgLzJ1/MfsQQJ7hHVeHmTAjM664V+fXvwUGJLziCeBo1ysWLRnl8CQIDAQABoAAw -DQYJKoZIhvcNAQEEBQADQQA7uqbrNTjVWpF6By5ZNPvhZ4YdFgkeXFVWi5ao/TaP -Vq4BG021fJ9nlHRtr4rotpgHDX1rr+iWeHKsx4+5DRSy ------END CERTIFICATE REQUEST----- \ No newline at end of file diff --git a/tests/mockserver.py b/tests/mockserver.py index b95a6c3c4..98723846e 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -203,7 +203,7 @@ class MockServer(): time.sleep(0.2) -def ssl_context_factory(keyfile='keys/cert.pem', certfile='keys/cert.pem'): +def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt'): return ssl.DefaultOpenSSLContextFactory( os.path.join(os.path.dirname(__file__), keyfile), os.path.join(os.path.dirname(__file__), certfile), From 6c1cacb5d5f0e5523d429cb89808dabfd824dcd5 Mon Sep 17 00:00:00 2001 From: Liu Siyuan Date: Sat, 6 May 2017 05:47:06 +0800 Subject: [PATCH 157/362] [MRG+1] doc: fix documentation error in link-extractor.rst (#2676) * fix doc error in link-extractor.rst * remove the import clause * update based on suggestion --- docs/topics/link-extractors.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 01d7f0b97..f40a36d31 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -8,7 +8,7 @@ Link extractors are objects whose only purpose is to extract links from web pages (:class:`scrapy.http.Response` objects) which will be eventually followed. -There is ``scrapy.linkextractors import LinkExtractor`` available +There is ``scrapy.linkextractors.LinkExtractor`` available in Scrapy, but you can create your own custom Link Extractors to suit your needs by implementing a simple interface. From 4966dd7a7fa544f6c5b3bbeb1f09426d9adcdb16 Mon Sep 17 00:00:00 2001 From: yandongxu Date: Mon, 8 May 2017 18:50:30 +0800 Subject: [PATCH 158/362] Fix doc: open file with "wb" mode will get an error in python 3 --- docs/topics/item-pipeline.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 8c7aa361f..33e4d7429 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -107,7 +107,7 @@ format:: class JsonWriterPipeline(object): def open_spider(self, spider): - self.file = open('items.jl', 'wb') + self.file = open('items.jl', 'w') def close_spider(self, spider): self.file.close() @@ -134,7 +134,7 @@ method and how to clean up the resources properly.:: import pymongo class MongoPipeline(object): - + collection_name = 'scrapy_items' def __init__(self, mongo_uri, mongo_db): @@ -248,4 +248,3 @@ To activate an Item Pipeline component you must add its class to the The integer values you assign to classes in this setting determine the order in which they run: items go through from lower valued to higher valued classes. It's customary to define these numbers in the 0-1000 range. - From 63b8caf5debf84e8da7f299782d15d0a41bf8a14 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 9 May 2017 11:58:53 -0300 Subject: [PATCH 159/362] Feed exports: rewrite indentation test without .strip() --- tests/test_feedexport.py | 41 ++++++++++++++-------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c66c470a8..f55927121 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -466,28 +466,23 @@ class FeedExportTest(unittest.TestCase): { 'format': 'json', 'indent': -1, - 'expected': b""" -[ + 'expected': b"""[ {"foo": ["bar"]}, {"key": "value"} -] -""", +]""", }, { 'format': 'json', 'indent': 0, - 'expected': b""" -[ + 'expected': b"""[ {"foo": ["bar"]}, {"key": "value"} -] -""", +]""", }, { 'format': 'json', 'indent': 2, - 'expected': b""" -[ + 'expected': b"""[ { "foo": [ "bar" @@ -501,8 +496,7 @@ class FeedExportTest(unittest.TestCase): { 'format': 'json', 'indent': 4, - 'expected': b""" -[ + 'expected': b"""[ { "foo": [ "bar" @@ -516,8 +510,7 @@ class FeedExportTest(unittest.TestCase): { 'format': 'json', 'indent': 5, - 'expected': b""" -[ + 'expected': b"""[ { "foo": [ "bar" @@ -533,15 +526,13 @@ class FeedExportTest(unittest.TestCase): { 'format': 'xml', 'indent': None, - 'expected': b""" - + 'expected': b""" barvalue""", }, { 'format': 'xml', 'indent': -1, - 'expected': b""" - + 'expected': b""" bar value @@ -550,8 +541,7 @@ class FeedExportTest(unittest.TestCase): { 'format': 'xml', 'indent': 0, - 'expected': b""" - + 'expected': b""" bar value @@ -560,8 +550,7 @@ class FeedExportTest(unittest.TestCase): { 'format': 'xml', 'indent': 2, - 'expected': b""" - + 'expected': b""" @@ -576,8 +565,7 @@ class FeedExportTest(unittest.TestCase): { 'format': 'xml', 'indent': 4, - 'expected': b""" - + 'expected': b""" @@ -592,8 +580,7 @@ class FeedExportTest(unittest.TestCase): { 'format': 'xml', 'indent': 5, - 'expected': b""" - + 'expected': b""" @@ -611,4 +598,4 @@ class FeedExportTest(unittest.TestCase): settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']} data = yield self.exported_data(items, settings) print(row['format'], row['indent']) - self.assertEqual(row['expected'].strip(), data) + self.assertEqual(row['expected'], data) From 548a432951ef48f142d6091ecffd8e54eaab3fc4 Mon Sep 17 00:00:00 2001 From: Kurt Peek Date: Fri, 5 May 2017 13:03:56 +0200 Subject: [PATCH 160/362] Minor grammatical changes --- docs/topics/logging.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index ac3b614fc..a3281dd6b 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -27,7 +27,7 @@ Scrapy from scripts as described in :ref:`run-from-script`. Log levels ========== -Python's builtin logging defines 5 different levels to indicate severity on a +Python's builtin logging defines 5 different levels to indicate the severity of a given log message. Here are the standard ones, listed in decreasing order: 1. ``logging.CRITICAL`` - for critical errors (highest severity) @@ -47,20 +47,20 @@ level:: There are shortcuts for issuing log messages on any of the standard 5 levels, and there's also a general ``logging.log`` method which takes a given level as -argument. If you need so, last example could be rewrote as:: +argument. If needed, the last example could be rewritten as:: import logging logging.log(logging.WARNING, "This is a warning") -On top of that, you can create different "loggers" to encapsulate messages (For -example, a common practice it's to create different loggers for every module). +On top of that, you can create different "loggers" to encapsulate messages. (For +example, a common practice is to create different loggers for every module). These loggers can be configured independently, and they allow hierarchical constructions. -Last examples use the root logger behind the scenes, which is a top level +The previous examples use the root logger behind the scenes, which is a top level logger where all messages are propagated to (unless otherwise specified). Using ``logging`` helpers is merely a shortcut for getting the root logger -explicitly, so this is also an equivalent of last snippets:: +explicitly, so this is also an equivalent of the last snippets:: import logging logger = logging.getLogger() @@ -95,7 +95,7 @@ Logging from Spiders ==================== Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider -instance, that can be accessed and used like this:: +instance, which can be accessed and used like this:: import scrapy From 25535dba9ca7e6f6f3c2279dd77240a21b1cc672 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 10 May 2017 16:45:15 -0300 Subject: [PATCH 161/362] Feed exports: edit note, fix typos --- docs/topics/exporters.rst | 4 ++-- docs/topics/feed-exports.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 2ad77c905..b6139af92 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -222,8 +222,8 @@ BaseItemExporter * ``indent=None`` selects the most compact representation, all items in the same line with no indentation - * ``indent<=0`` each item on it's own line, no indentation - * ``indent>0`` each item on it's own line, indentated with the provided numeric value + * ``indent<=0`` each item on its own line, no indentation + * ``indent>0`` each item on its own line, indented with the provided numeric value .. highlight:: none diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index e57a4e776..d760b1a28 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -279,7 +279,7 @@ is a non-negative integer, then array elements and object members will be pretty with that indent level. An indent level of ``0``, or negative, will put each item on a new line. ``None`` selects the most compact representation -Currently used by :class:`~scrapy.exporters.JsonItemExporter` +Currently implemented only by :class:`~scrapy.exporters.JsonItemExporter` and :class:`~scrapy.exporters.XmlItemExporter` .. setting:: FEED_STORE_EMPTY From 3a0a86ed31df1d22fea3b5b05e853f212adc40c8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 12 May 2017 17:26:17 +0200 Subject: [PATCH 162/362] Clarify FEED_EXPORT_INDENT section --- docs/topics/feed-exports.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index d760b1a28..135d05c93 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -276,11 +276,12 @@ Default: ``0`` Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT`` is a non-negative integer, then array elements and object members will be pretty-printed -with that indent level. An indent level of ``0``, or negative, will put each item on a new line. -``None`` selects the most compact representation +with that indent level. An indent level of ``0`` (the default), or negative, +will put each item on a new line. ``None`` selects the most compact representation. Currently implemented only by :class:`~scrapy.exporters.JsonItemExporter` -and :class:`~scrapy.exporters.XmlItemExporter` +and :class:`~scrapy.exporters.XmlItemExporter`, i.e. when you are exporting +to ``.json`` or ``.xml``. .. setting:: FEED_STORE_EMPTY From 26f723e4e63185b35b2f11285f50cfc3ac52b8fc Mon Sep 17 00:00:00 2001 From: Harrison Gregg Date: Sun, 30 Apr 2017 19:07:29 -0400 Subject: [PATCH 163/362] Allow formdata value to be None to drop field generated from response --- scrapy/http/request/form.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 905d8412f..d9d178a3e 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -135,7 +135,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): if clickable and clickable[0] not in formdata and not clickable[0] is None: values.append(clickable) - values.extend(formdata.items()) + values.extend((k, v) for k, v in formdata.items() if v is not None) return values From 45a323024c7a7008df3e319a8a1437fae53826f9 Mon Sep 17 00:00:00 2001 From: Harrison Gregg Date: Sun, 30 Apr 2017 19:14:47 -0400 Subject: [PATCH 164/362] Add documentation for dropping fields in from_response request body --- docs/topics/request-response.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 67f8ec285..5410654ef 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -417,7 +417,9 @@ fields with form data from :class:`Response` objects. :param formdata: fields to override in the form data. If a field was already present in the response ``
`` element, its value is - overridden by the one passed in this parameter. + overridden by the one passed in this parameter. If a value passed in + this parameter is ``None``, the field will not be included in the + request, even if it was present in the response ```` element. :type formdata: dict :param clickdata: attributes to lookup the control clicked. If it's not From ffef828a8deb86520e3bd6a50d76b2a4ecf3ae71 Mon Sep 17 00:00:00 2001 From: Harrison Gregg Date: Sun, 30 Apr 2017 19:33:51 -0400 Subject: [PATCH 165/362] Add test for dropping fields in from_response request body --- tests/test_http_request.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 7eadb874f..bbce537f4 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -425,6 +425,17 @@ class FormRequestTest(RequestTest): self.assertEqual(fs[b'one'], [b'1']) self.assertEqual(fs[b'two'], [b'2']) + def test_from_response_drop_params(self): + response = _buildresponse( + """ + + + """) + req = self.request_class.from_response(response, formdata={'two': None}) + fs = _qs(req) + self.assertEqual(fs[b'one'], [b'1']) + self.assertNotIn(b'two', fs) + def test_from_response_override_method(self): response = _buildresponse( ''' From df7a5c4aa4f5898de3c70cef17c3c5031f7e05a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A1bio=20C=2E=20Barrionuevo=20da=20Luz?= Date: Mon, 15 May 2017 22:52:23 -0300 Subject: [PATCH 166/362] Add support for executing scrapy using -m option of python python -m scrapy --- scrapy/__main__.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 scrapy/__main__.py diff --git a/scrapy/__main__.py b/scrapy/__main__.py new file mode 100644 index 000000000..e467e057f --- /dev/null +++ b/scrapy/__main__.py @@ -0,0 +1,4 @@ +from scrapy.cmdline import execute + +if __name__ == '__main__': + execute() From b74b98fa3e7149c27bd3a541940457864a28d5d1 Mon Sep 17 00:00:00 2001 From: Eli Atzaba Date: Tue, 16 May 2017 13:59:58 +0300 Subject: [PATCH 167/362] cleanup: removed unused MEMUSAGE_REPORT Signed-off-by: Eli Atzaba --- docs/topics/extensions.rst | 1 - docs/topics/settings.rst | 13 ------------- scrapy/extensions/memusage.py | 1 - scrapy/settings/default_settings.py | 1 - 4 files changed, 16 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 7f2952f4c..6036db0f5 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -220,7 +220,6 @@ can be configured with the following settings: * :setting:`MEMUSAGE_LIMIT_MB` * :setting:`MEMUSAGE_WARNING_MB` * :setting:`MEMUSAGE_NOTIFY_MAIL` -* :setting:`MEMUSAGE_REPORT` * :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` Memory debugger extension diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 9bf07588b..2cf6ffe75 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -959,19 +959,6 @@ Example:: See :ref:`topics-extensions-ref-memusage`. -.. setting:: MEMUSAGE_REPORT - -MEMUSAGE_REPORT ---------------- - -Default: ``False`` - -Scope: ``scrapy.extensions.memusage`` - -Whether to send a memory usage report after each spider has been closed. - -See :ref:`topics-extensions-ref-memusage`. - .. setting:: MEMUSAGE_WARNING_MB MEMUSAGE_WARNING_MB diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 322213cf0..c0570567e 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -35,7 +35,6 @@ class MemoryUsage(object): self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL') self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 - self.report = crawler.settings.getbool('MEMUSAGE_REPORT') self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS') self.mail = MailSender.from_settings(crawler.settings) crawler.signals.connect(self.engine_started, signal=signals.engine_started) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 26ff4257e..03c36a0f9 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -219,7 +219,6 @@ MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0 MEMUSAGE_ENABLED = True MEMUSAGE_LIMIT_MB = 0 MEMUSAGE_NOTIFY_MAIL = [] -MEMUSAGE_REPORT = False MEMUSAGE_WARNING_MB = 0 METAREFRESH_ENABLED = True From 1a452c038cc0547924051a7cd0786215ec7c2104 Mon Sep 17 00:00:00 2001 From: Bernardas Date: Thu, 18 May 2017 16:57:13 +0000 Subject: [PATCH 168/362] increase ptpython priority since it can use other shells as backend --- scrapy/utils/console.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index a9d73aada..2e9981556 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -56,9 +56,9 @@ def _embed_standard_shell(namespace={}, banner=''): return wrapper DEFAULT_PYTHON_SHELLS = OrderedDict([ + ('ptpython', _embed_ptpython_shell), ('ipython', _embed_ipython_shell), ('bpython', _embed_bpython_shell), - ('ptpython', _embed_ptpython_shell), ('python', _embed_standard_shell), ]) From 9ce03d096d7ed569071a1951d9e327d7294e1a83 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 19 May 2017 00:01:27 +0500 Subject: [PATCH 169/362] codecov config: disable project check, tweak PR comments --- codecov.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..d8aa6b984 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,6 @@ +comment: + layout: "header, diff, tree" + +coverage: + status: + project: false From 851adcedf2220a783cdfb2d6b5872580e18b8d88 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 6 Mar 2017 23:36:21 +0100 Subject: [PATCH 170/362] List merged pull requests since 1.3.3 --- docs/news.rst | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index da856d883..675a6d595 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,78 @@ Release notes ============= +Scrapy 1.4.0 (2017-XX-XX) +------------------------- + +New Features +~~~~~~~~~~~~ + +- Use credentials from request.meta['proxy'] #2530 +- [httpcompression] add support for br - brotli content encoding #2535 +- Enable memusage extension by default. #2539 +- response.follow #2540 +- add flags to request #2082 +- Support Anonymous FTP #2343 +- HttpErrorMiddleware stats #2566 +- Retry stats #2543 +- Set canonicalize=False for LinkExtractor #2537 +- Referrer policies in RefererMiddleware #2306 +- Fix referrer policy from response headers and support explicit empty string #2627 +- Data URI download handler. #2334 +- HttpCacheMiddleware: log cache directory at instantiation #2611 +- Add warning on duplicate spider name #2612 +- Allowed passing objects of Mapping class or its subclass to the CaselessDict initializer #2646 +- Allow redirections in media files downloads #2616 +- Travis CI: use portable pypy for Linux #2710 + + +Bug fixes +~~~~~~~~~ + +- LinkExtractors: strip whitespaces #2547 +- FormRequest: handle whitespaces in action attribute properly #2548 +- Buffer CONNECT response bytes from proxy until all HTTP headers are received #2495 +- Fix FTP downloader and re-enable FTP tests on Python 3 #2599 +- Handle data loss gracefully. #2590 +- Use body to choose response type after decompression content #2393 +- Always decompress Content-Encoding: gzip at HttpCompression stage #2391 +- Respect custom log level (#2581, fixes #1612) +- [logformatter] 'flags' format spec backward compatibility #2649 +- 'make htmlview' does not open the webbrowser #2661 +- Remove "commands" from the command list #2695 + +Cleanups +~~~~~~~~ + +- TST remove temp files and folders #2570 +- TST fixed ProjectUtilsTest on OS X #2569 +- Separate building request from _requests_to_follow in CrawlSpider #2562 +- remove “Python 3 progress” badge #2567 +- add a couple more lines to gitignore #2557 +- deprecate Spider.make_requests_from_url. #1728 +- Remove bumpversion prerelease configuration #2159 +- Set context factory implementation based on Twisted version #2577 +- Add omitted "self" arguments #2595 +- Remove redundant slot.add_request() call in ExecutionEngine #2617 +- Removed contrib section in contribution documentation #2636 +- More specific exception catching: os.path.getmtime can only raise os.error in FSFilesStore #2644 + +Documentation +~~~~~~~~~~~~~ + +- Doc: binary mode is required for exporters #2564 +- document issue with FormRequest.from_response due to bug in lxml #2572 +- Use single quotes uniformly #2596 +- Document ftp_user and ftp_password meta keys #2587 +- Update release notes for 1.0.7, 1.1.4 and 1.2.3 #2625 +- DOC Mention brotli support in HttpCompressionMiddleware section #2628 +- Removed contrib section in contribution documentation #2636 +- docs: installation instructions, mention conda in the beginning (closes #2475) #2477 +- FAQ Rewrite note on Python 3 support on Windows #2690 +- DOC Rearrange selector sections #2705 +- Remove __nonzero__ from SelectorList docs #2683 + + Scrapy 1.3.3 (2017-03-10) ------------------------- @@ -15,6 +87,7 @@ Bug fixes A new setting is introduced to toggle between warning or exception if needed ; see :setting:`SPIDER_LOADER_WARN_ONLY` for details. + Scrapy 1.3.2 (2017-02-13) ------------------------- From 8729a91f7a136decb66b00ada858a67c41796e11 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 19 Apr 2017 17:22:37 +0200 Subject: [PATCH 171/362] Rephrase "New features" section --- docs/news.rst | 50 +++++++++++++++++++++----------- docs/topics/media-pipeline.rst | 5 ++-- docs/topics/request-response.rst | 2 +- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 675a6d595..979e9a953 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -9,24 +9,40 @@ Scrapy 1.4.0 (2017-XX-XX) New Features ~~~~~~~~~~~~ -- Use credentials from request.meta['proxy'] #2530 -- [httpcompression] add support for br - brotli content encoding #2535 -- Enable memusage extension by default. #2539 -- response.follow #2540 -- add flags to request #2082 -- Support Anonymous FTP #2343 -- HttpErrorMiddleware stats #2566 -- Retry stats #2543 -- Set canonicalize=False for LinkExtractor #2537 -- Referrer policies in RefererMiddleware #2306 -- Fix referrer policy from response headers and support explicit empty string #2627 -- Data URI download handler. #2334 -- HttpCacheMiddleware: log cache directory at instantiation #2611 -- Add warning on duplicate spider name #2612 -- Allowed passing objects of Mapping class or its subclass to the CaselessDict initializer #2646 -- Allow redirections in media files downloads #2616 -- Travis CI: use portable pypy for Linux #2710 +- Accept proxy credentials in request.meta['proxy'] (:issue:`2526`) +- Support `brotli`_-compressed content; requires optional `brotlipy`_ + (:issue:`2535`) +- Enable memusage extension by default (:issue:`2187`) ; + **this is technically backwards-incompatible** so please check if you have + any non-default ``MEMUSAGE_***`` settings set. +- New :meth:`Response.follow ` shortcur + for creating requests (:issue:`1940`) +- Added ``flags`` argument and attribute to :class:`Request ` + (:issue:`2047`) +- Support Anonymous FTP (:issue:`2342`) +- Added ``retry/count``, ``retry/max_reached`` and ``retry/reason_count/***`` + stats to :class:`RetryMiddleware ` + (:issue:`2543`) +- Added ``httperror/response_ignored_count`` and ``httperror/response_ignored_status_count/***`` + stats to :class:`HttpErrorMiddleware ` + (:issue:`2566`) +- Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor` + (:issue:`2537`, fixes :issue:`1941` and :issue:`1982`): + **warning, this istechnically backwards-incompatible** +- Customizable :setting:`Referrer policy ` in + :class:`RefererMiddleware ` + (:issue:`2306`) +- New ``data:`` URI download handler (:issue:`2334`, fixes :issue:`2156`) +- Log cache directory when HTTP Cache is used (:issue:`2611`, fixes :issue:`2604`) +- Warn users when project contains duplicate spider names (fixes :issue:`2181`) +- :class:`CaselessDict` now accepts ``Mapping`` instances and not only dicts (:issue:`2646`) +- :ref:`Media downloads `, with :class:`FilesPipelines` + or :class:`ImagesPipelines`, can now optionally handle HTTP redirects + using the new :setting:`MEDIA_ALLOW_REDIRECTS` (:issue:`2616`, fixes :issue:`2004`) +- Use portable pypy for Linux on Travis CI (:issue:`2710`) +.. _brotli: https://github.com/google/brotli +.. _brotlipy: https://github.com/python-hyper/brotlipy/ Bug fixes ~~~~~~~~~ diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index f258ff748..e948913a4 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -320,8 +320,6 @@ all be dropped because at least one dimension is shorter than the constraint. By default, there are no size constraints, so all images are processed. -.. _topics-media-pipeline-override: - Allowing redirections --------------------- @@ -330,10 +328,11 @@ Allowing redirections By default media pipelines ignore redirects, i.e. an HTTP redirection to a media file URL request will mean the media download is considered failed. -To handle media redirections, set this settings to ``True``: +To handle media redirections, set this setting to ``True``:: MEDIA_ALLOW_REDIRECTS = True +.. _topics-media-pipeline-override: Extending the Media Pipelines ============================= diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f1552572a..6ca37b7c9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and Request objects =============== -.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback]) +.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags]) A :class:`Request` object represents an HTTP request, which is usually generated in the Spider and executed by the Downloader, and thus generating From cba55cd190c573a03b8458cda7ba2a20da5651e6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 21 Apr 2017 16:52:32 +0200 Subject: [PATCH 172/362] Rephrase other sections --- docs/news.rst | 96 ++++++++++++++++++++++++++++----------------------- 1 file changed, 52 insertions(+), 44 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 979e9a953..a19d7a93a 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -9,26 +9,26 @@ Scrapy 1.4.0 (2017-XX-XX) New Features ~~~~~~~~~~~~ -- Accept proxy credentials in request.meta['proxy'] (:issue:`2526`) +- Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`) - Support `brotli`_-compressed content; requires optional `brotlipy`_ (:issue:`2535`) - Enable memusage extension by default (:issue:`2187`) ; **this is technically backwards-incompatible** so please check if you have any non-default ``MEMUSAGE_***`` settings set. -- New :meth:`Response.follow ` shortcur +- New :meth:`Response.follow ` shortcut for creating requests (:issue:`1940`) - Added ``flags`` argument and attribute to :class:`Request ` - (:issue:`2047`) + objects (:issue:`2047`) - Support Anonymous FTP (:issue:`2342`) -- Added ``retry/count``, ``retry/max_reached`` and ``retry/reason_count/***`` +- Added ``retry/count``, ``retry/max_reached`` and ``retry/reason_count/`` stats to :class:`RetryMiddleware ` (:issue:`2543`) -- Added ``httperror/response_ignored_count`` and ``httperror/response_ignored_status_count/***`` +- Added ``httperror/response_ignored_count`` and ``httperror/response_ignored_status_count/`` stats to :class:`HttpErrorMiddleware ` (:issue:`2566`) - Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor` (:issue:`2537`, fixes :issue:`1941` and :issue:`1982`): - **warning, this istechnically backwards-incompatible** + **warning, this is technically backwards-incompatible** - Customizable :setting:`Referrer policy ` in :class:`RefererMiddleware ` (:issue:`2306`) @@ -38,8 +38,9 @@ New Features - :class:`CaselessDict` now accepts ``Mapping`` instances and not only dicts (:issue:`2646`) - :ref:`Media downloads `, with :class:`FilesPipelines` or :class:`ImagesPipelines`, can now optionally handle HTTP redirects - using the new :setting:`MEDIA_ALLOW_REDIRECTS` (:issue:`2616`, fixes :issue:`2004`) -- Use portable pypy for Linux on Travis CI (:issue:`2710`) + using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting (:issue:`2616`, fixes :issue:`2004`) +- Accept non-complete responses from websites using a new + :setting:`DOWNLOAD_FAIL_ON_DATALOSS` setting (:issue:`2590`, fixes :issue:`2586`) .. _brotli: https://github.com/google/brotli .. _brotlipy: https://github.com/python-hyper/brotlipy/ @@ -47,48 +48,55 @@ New Features Bug fixes ~~~~~~~~~ -- LinkExtractors: strip whitespaces #2547 -- FormRequest: handle whitespaces in action attribute properly #2548 -- Buffer CONNECT response bytes from proxy until all HTTP headers are received #2495 -- Fix FTP downloader and re-enable FTP tests on Python 3 #2599 -- Handle data loss gracefully. #2590 -- Use body to choose response type after decompression content #2393 -- Always decompress Content-Encoding: gzip at HttpCompression stage #2391 -- Respect custom log level (#2581, fixes #1612) -- [logformatter] 'flags' format spec backward compatibility #2649 -- 'make htmlview' does not open the webbrowser #2661 -- Remove "commands" from the command list #2695 +- LinkExtractor now strips leading and trailing whitespaces from attributes + (:issue:`2547`, fixes :issue:`1614`) +- Properly handle whitespaces in action attribute in :class:`FormRequest` + (:issue:`2548`) +- Buffer CONNECT response bytes from proxy until all HTTP headers are received + (:issue:`2495`, fixes :issue:`2491`) +- FTP downloader now works on Python 3, provided you use Twisted>=17.1 + (:issue:`2599`) +- Use body to choose response type after decompressing content (:issue:`2393`, + fixes :issue:`2145`) +- Always decompress ``Content-Encoding: gzip`` at :class:`HttpCompressionMiddleware + ` stage (:issue:`2391`) +- Respect custom log level in ``Spider.custom_settings`` (:issue:`2581`, + fixes :issue:`1612`) +- 'make htmlview' fix for macOS (:issue:`2661`) +- Remove "commands" from the command list (:issue:`2695`) -Cleanups -~~~~~~~~ +Cleanups & Refactoring +~~~~~~~~~~~~~~~~~~~~~~ -- TST remove temp files and folders #2570 -- TST fixed ProjectUtilsTest on OS X #2569 -- Separate building request from _requests_to_follow in CrawlSpider #2562 -- remove “Python 3 progress” badge #2567 -- add a couple more lines to gitignore #2557 -- deprecate Spider.make_requests_from_url. #1728 -- Remove bumpversion prerelease configuration #2159 -- Set context factory implementation based on Twisted version #2577 -- Add omitted "self" arguments #2595 -- Remove redundant slot.add_request() call in ExecutionEngine #2617 -- Removed contrib section in contribution documentation #2636 -- More specific exception catching: os.path.getmtime can only raise os.error in FSFilesStore #2644 +- Tests: remove temp files and folders (:issue:`2570`), + fixed ProjectUtilsTest on OS X (:issue:`2569`), + use portable pypy for Linux on Travis CI (:issue:`2710`) + +- Separate building request from ``_requests_to_follow`` in CrawlSpider (:issue:`2562`) +- Remove “Python 3 progress” badge (:issue:`2567`) +- Add a couple more lines to ``.gitignore`` (:issue:`2557`) +- Deprecate ``Spider.make_requests_from_url`` (:issue:`1728`) +- Remove bumpversion prerelease configuration (:issue:`2159`) +- Set context factory implementation based on Twisted version (:issue:`2577`, + fixes :issue:`2560`) +- Add omitted ``self`` arguments in default project middleware template (:issue:`2595`) +- Remove redundant ``slot.add_request()`` call in ExecutionEngine (:issue:`2617`) +- Catch more specific ``os.error`` exception in :class:`FSFilesStore` (:issue:`2644`) Documentation ~~~~~~~~~~~~~ -- Doc: binary mode is required for exporters #2564 -- document issue with FormRequest.from_response due to bug in lxml #2572 -- Use single quotes uniformly #2596 -- Document ftp_user and ftp_password meta keys #2587 -- Update release notes for 1.0.7, 1.1.4 and 1.2.3 #2625 -- DOC Mention brotli support in HttpCompressionMiddleware section #2628 -- Removed contrib section in contribution documentation #2636 -- docs: installation instructions, mention conda in the beginning (closes #2475) #2477 -- FAQ Rewrite note on Python 3 support on Windows #2690 -- DOC Rearrange selector sections #2705 -- Remove __nonzero__ from SelectorList docs #2683 +- Binary mode is required for exporters (:issue:`2564`, fixes :issue:`2553`) +- Mention issue with :meth:`FormRequest.from_response + ` due to bug in lxml (:issue:`2572`) +- Use single quotes uniformly in templates (:issue:`2596`) +- Document :reqmeta:`ftp_user` and :reqmeta:`ftp_password` meta keys (:issue:`2587`) +- Removed section on deprecated ``contrib/`` (:issue:`2636`) +- Recommend Anaconda when installing Scrapy on Windows + (:issue:`2477`, fixes :issue:`2475`) +- FAQ: rewrite note on Python 3 support on Windows (:issue:`2690`) +- Rearrange selector sections (:issue:`2705`) +- Remove ``__nonzero__`` from :class:`SelectorList` docs (:issue:`2683`) Scrapy 1.3.3 (2017-03-10) From e139d990fca1e1e2144992bb2e05665d17c3f989 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 21 Apr 2017 16:53:20 +0200 Subject: [PATCH 173/362] Fix sphinx-build warning on deprecated latex_paper_size --- docs/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index a3d1611f9..187f03c4c 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -10,7 +10,8 @@ PAPER = SOURCES = SHELL = /bin/bash -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \ +ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \ + -D latex_elements.papersize=$(PAPER) \ $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) .PHONY: help update build html htmlhelp clean From c6464cc4f5a152c2d23a79457b7e01ee90b18069 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 9 May 2017 20:16:41 +0200 Subject: [PATCH 174/362] Add verbose introduction to new features --- docs/news.rst | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index a19d7a93a..f225fef1d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,6 +6,52 @@ Release notes Scrapy 1.4.0 (2017-XX-XX) ------------------------- +Scrapy 1.4 does not bring that many breathtaking new features +but quite a few handy improvements nonetheless. + +Scrapy now supports anonymous FTP sessions with customizable user and +password via the new :setting:`FTP_USER` and :setting:`FTP_PASSWORD` settings. +**And if you're using Twisted version 17.1.0 or above, FTP is now available +with Python 3.** + +Link extractors now work similarly to what a regular modern browser would +do. Especially, leading and trailing whitespace are removed from attributes +(think ``href=" http://example.com"``) when building ``Link`` objects. +This whitespace-stripping also happens for ``action`` attributes with +``FormRequest``. +**Please also note that link extractors do not canonicalize URLs by default +anymore.** This was puzzling users every now and then, and it's not what +browsers do in fact, so we removed that extra transformation on extractred +links. + +There's a new ``response.follow()`` shortcut for creating URLs directly +from the response instance in callbacks. +For example, instead of:: + + scrapy.Request(response.urljoin(somehrefvalue)) + +you can now use the simpler:: + + response.follow(somehrefvalue) + + +For those of you wanting more control on the ``Referer:`` header that Scrapy +sends when following links, you can set your own ``Referrer Policy``. +Prior to Scrapy 1.4, the default ``RefererMiddleware`` would simply and +blindly set it to the URL of the response that generated the HTTP request +(which could leak information on your URL seeds). +By default, Scrapy now behaves much like your regular browser does. +And this policy is fully customizable with W3C standard values +(or with something really custom of your own if you wish). +See :setting:`REFERRER_POLICY` for details. + +Last but not least, Scrapy now has the option to make JSON and XML items +more human-readable, with newlines between items and even custom indenting +offset, using the new :setting:`FEED_EXPORT_INDENT` setting. + +Enjoy! (Or read on for the rest of changes in this release.) + + New Features ~~~~~~~~~~~~ From 7d72394794bea63fdaa08d1264a50b0b83998424 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 10 May 2017 18:55:09 +0200 Subject: [PATCH 175/362] Reword mention of new response.follow() shortcut --- docs/news.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index f225fef1d..27baff754 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -24,8 +24,8 @@ anymore.** This was puzzling users every now and then, and it's not what browsers do in fact, so we removed that extra transformation on extractred links. -There's a new ``response.follow()`` shortcut for creating URLs directly -from the response instance in callbacks. +There's a new ``response.follow()`` shortcut for creating requests directly +from a response instance and a relative URL. For example, instead of:: scrapy.Request(response.urljoin(somehrefvalue)) @@ -34,7 +34,6 @@ you can now use the simpler:: response.follow(somehrefvalue) - For those of you wanting more control on the ``Referer:`` header that Scrapy sends when following links, you can set your own ``Referrer Policy``. Prior to Scrapy 1.4, the default ``RefererMiddleware`` would simply and From 55d10823603e8033a4c4c441b8e78da217a951bf Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 11 May 2017 14:25:11 +0200 Subject: [PATCH 176/362] Reference recent fixes and commits --- docs/news.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 27baff754..64995dd10 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -109,6 +109,8 @@ Bug fixes fixes :issue:`1612`) - 'make htmlview' fix for macOS (:issue:`2661`) - Remove "commands" from the command list (:issue:`2695`) +- Fix duplicate Content-Length header for POST requests with empty body (:issue:`2677`) +- Properly cancel large downloads, i.e. above :setting:`DOWNLOAD_MAXSIZE` (:issue:`1616`) Cleanups & Refactoring ~~~~~~~~~~~~~~~~~~~~~~ @@ -116,7 +118,6 @@ Cleanups & Refactoring - Tests: remove temp files and folders (:issue:`2570`), fixed ProjectUtilsTest on OS X (:issue:`2569`), use portable pypy for Linux on Travis CI (:issue:`2710`) - - Separate building request from ``_requests_to_follow`` in CrawlSpider (:issue:`2562`) - Remove “Python 3 progress” badge (:issue:`2567`) - Add a couple more lines to ``.gitignore`` (:issue:`2557`) @@ -127,6 +128,7 @@ Cleanups & Refactoring - Add omitted ``self`` arguments in default project middleware template (:issue:`2595`) - Remove redundant ``slot.add_request()`` call in ExecutionEngine (:issue:`2617`) - Catch more specific ``os.error`` exception in :class:`FSFilesStore` (:issue:`2644`) +- Change "localhost" test server certificate (:issue:`2720`) Documentation ~~~~~~~~~~~~~ @@ -142,6 +144,10 @@ Documentation - FAQ: rewrite note on Python 3 support on Windows (:issue:`2690`) - Rearrange selector sections (:issue:`2705`) - Remove ``__nonzero__`` from :class:`SelectorList` docs (:issue:`2683`) +- Mention how to disable request filtering in documentation of + :setting:`DUPEFILTER_CLASS` setting (:issue:`2714`) +- Add sphinx_rtd_theme to docs setup readme (:issue:`2668`) +- Open file in text mode in JSON item writer example (:issue:`2729`) Scrapy 1.3.3 (2017-03-10) From 896c30a8eb1ea5d03898b27f238243ea629a3e42 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 12 May 2017 19:39:31 +0200 Subject: [PATCH 177/362] Reference items pretty-printing issue number --- docs/news.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 64995dd10..c249c038a 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -86,6 +86,8 @@ New Features using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting (:issue:`2616`, fixes :issue:`2004`) - Accept non-complete responses from websites using a new :setting:`DOWNLOAD_FAIL_ON_DATALOSS` setting (:issue:`2590`, fixes :issue:`2586`) +- Optional pretty-printing of JSON and XML items via + :setting:`FEED_EXPORT_INDENT` setting (:issue:`2456`, fixes :issue:`1327`) .. _brotli: https://github.com/google/brotli .. _brotlipy: https://github.com/python-hyper/brotlipy/ From 432668acf7b8236229fca3b7726139acdc0c05f2 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 16 May 2017 16:00:26 +0200 Subject: [PATCH 178/362] Mention implementation of #667 --- docs/news.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index c249c038a..2bd6640b3 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -88,6 +88,8 @@ New Features :setting:`DOWNLOAD_FAIL_ON_DATALOSS` setting (:issue:`2590`, fixes :issue:`2586`) - Optional pretty-printing of JSON and XML items via :setting:`FEED_EXPORT_INDENT` setting (:issue:`2456`, fixes :issue:`1327`) +- Allow dropping fields in ``FormRequest.from_response`` formdata when + ``None`` value is passed (:issue:`667`) .. _brotli: https://github.com/google/brotli .. _brotlipy: https://github.com/python-hyper/brotlipy/ From a3d3cd4cb7d256a0b876019279ec88c47d9957ff Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 17 May 2017 19:52:18 +0200 Subject: [PATCH 179/362] Update with latest merges --- docs/news.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 2bd6640b3..7debe176b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -90,6 +90,8 @@ New Features :setting:`FEED_EXPORT_INDENT` setting (:issue:`2456`, fixes :issue:`1327`) - Allow dropping fields in ``FormRequest.from_response`` formdata when ``None`` value is passed (:issue:`667`) +- Per-request retry times with the new :reqmeta:`max_retry_times` meta key + (:issue:`2642`) .. _brotli: https://github.com/google/brotli .. _brotlipy: https://github.com/python-hyper/brotlipy/ @@ -133,6 +135,7 @@ Cleanups & Refactoring - Remove redundant ``slot.add_request()`` call in ExecutionEngine (:issue:`2617`) - Catch more specific ``os.error`` exception in :class:`FSFilesStore` (:issue:`2644`) - Change "localhost" test server certificate (:issue:`2720`) +- Remove unused ``MEMUSAGE_REPORT`` setting (:issue:`2576`) Documentation ~~~~~~~~~~~~~ From edcde7a2cf1b7abbc0b27d51d83f40fe6fe8f5fa Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 18 May 2017 23:11:34 +0500 Subject: [PATCH 180/362] DOC tweak release notes: promote response.follow, mention logging/stats changes --- docs/news.rst | 53 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 7debe176b..b5e31e444 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -11,29 +11,42 @@ but quite a few handy improvements nonetheless. Scrapy now supports anonymous FTP sessions with customizable user and password via the new :setting:`FTP_USER` and :setting:`FTP_PASSWORD` settings. -**And if you're using Twisted version 17.1.0 or above, FTP is now available -with Python 3.** +And if you're using Twisted version 17.1.0 or above, FTP is now available +with Python 3. + +There's a new :meth:`response.follow ` method +for creating requests; **it is now a recommended way to create Requests +in Scrapy spiders**. This method makes it easier to write correct +spiders; ``response.follow`` has several advantages over creating +``scrapy.Request`` objects directly: + +* it handles relative URLs; +* it works properly with non-ascii URLs on non-UTF8 pages; +* in addition to absolute and relative URLs it supports Selectors; + for ``
`` elements it can also extract their href values. + +For example, instead of this:: + + for href in response.css('li.page a::attr(href)').extract(): + url = response.urljoin(href) + yield scrapy.Request(url, self.parse, encoding=response.encoding) + +One can now write this:: + + for a in response.css('li.page a'): + yield response.follow(a, self.parse) + +Link extractors are also improved. They work similarly to what a regular +modern browser would do: leading and trailing whitespace are removed +from attributes (think ``href=" http://example.com"``) when building +``Link`` objects. This whitespace-stripping also happens for ``action`` +attributes with ``FormRequest``. -Link extractors now work similarly to what a regular modern browser would -do. Especially, leading and trailing whitespace are removed from attributes -(think ``href=" http://example.com"``) when building ``Link`` objects. -This whitespace-stripping also happens for ``action`` attributes with -``FormRequest``. **Please also note that link extractors do not canonicalize URLs by default anymore.** This was puzzling users every now and then, and it's not what browsers do in fact, so we removed that extra transformation on extractred links. -There's a new ``response.follow()`` shortcut for creating requests directly -from a response instance and a relative URL. -For example, instead of:: - - scrapy.Request(response.urljoin(somehrefvalue)) - -you can now use the simpler:: - - response.follow(somehrefvalue) - For those of you wanting more control on the ``Referer:`` header that Scrapy sends when following links, you can set your own ``Referrer Policy``. Prior to Scrapy 1.4, the default ``RefererMiddleware`` would simply and @@ -44,6 +57,10 @@ And this policy is fully customizable with W3C standard values (or with something really custom of your own if you wish). See :setting:`REFERRER_POLICY` for details. +To make Scrapy spiders easier to debug, Scrapy logs more stats by default +in 1.4: memory usage stats, detailed retry stats, detailed HTTP error code +stats. A similar change is that HTTP cache path is also visible in logs now. + Last but not least, Scrapy now has the option to make JSON and XML items more human-readable, with newlines between items and even custom indenting offset, using the new :setting:`FEED_EXPORT_INDENT` setting. @@ -60,7 +77,7 @@ New Features - Enable memusage extension by default (:issue:`2187`) ; **this is technically backwards-incompatible** so please check if you have any non-default ``MEMUSAGE_***`` settings set. -- New :meth:`Response.follow ` shortcut +- New :ref:`response.follow ` shortcut for creating requests (:issue:`1940`) - Added ``flags`` argument and attribute to :class:`Request ` objects (:issue:`2047`) From 76e5b0f65c256373411c8fa6e005a85c48cad6ee Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 19 May 2017 01:13:32 +0500 Subject: [PATCH 181/362] DOC 1.4 deprecations and backwards incompatible changes, add recent commits to news. --- docs/news.rst | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index b5e31e444..3c641af44 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -67,6 +67,21 @@ offset, using the new :setting:`FEED_EXPORT_INDENT` setting. Enjoy! (Or read on for the rest of changes in this release.) +Deprecations and Backwards Incompatible Changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor` + (:issue:`2537`, fixes :issue:`1941` and :issue:`1982`): + **warning, this is technically backwards-incompatible** +- Enable memusage extension by default (:issue:`2539`, fixes :issue:`2187`); + **this is technically backwards-incompatible** so please check if you have + any non-default ``MEMUSAGE_***`` options set. +- ``EDITOR`` environment variable now takes precedence over ``EDITOR`` + option defined in settings.py (:issue:`1829`); Scrapy default settings + no longer depend on environment variables. **This is technically a backwards + incompatible change**. +- ``Spider.make_requests_from_url`` is deprecated + (:issue:`1728`, fixes :issue:`1495`). New Features ~~~~~~~~~~~~ @@ -74,9 +89,6 @@ New Features - Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`) - Support `brotli`_-compressed content; requires optional `brotlipy`_ (:issue:`2535`) -- Enable memusage extension by default (:issue:`2187`) ; - **this is technically backwards-incompatible** so please check if you have - any non-default ``MEMUSAGE_***`` settings set. - New :ref:`response.follow ` shortcut for creating requests (:issue:`1940`) - Added ``flags`` argument and attribute to :class:`Request ` @@ -88,9 +100,6 @@ New Features - Added ``httperror/response_ignored_count`` and ``httperror/response_ignored_status_count/`` stats to :class:`HttpErrorMiddleware ` (:issue:`2566`) -- Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor` - (:issue:`2537`, fixes :issue:`1941` and :issue:`1982`): - **warning, this is technically backwards-incompatible** - Customizable :setting:`Referrer policy ` in :class:`RefererMiddleware ` (:issue:`2306`) @@ -109,6 +118,8 @@ New Features ``None`` value is passed (:issue:`667`) - Per-request retry times with the new :reqmeta:`max_retry_times` meta key (:issue:`2642`) +- ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command + (:issue:`2740`) .. _brotli: https://github.com/google/brotli .. _brotlipy: https://github.com/python-hyper/brotlipy/ @@ -134,6 +145,8 @@ Bug fixes - Remove "commands" from the command list (:issue:`2695`) - Fix duplicate Content-Length header for POST requests with empty body (:issue:`2677`) - Properly cancel large downloads, i.e. above :setting:`DOWNLOAD_MAXSIZE` (:issue:`1616`) +- ImagesPipeline: fixed processing of transparent PNG images with palette + (:issue:`2675`) Cleanups & Refactoring ~~~~~~~~~~~~~~~~~~~~~~ @@ -144,8 +157,8 @@ Cleanups & Refactoring - Separate building request from ``_requests_to_follow`` in CrawlSpider (:issue:`2562`) - Remove “Python 3 progress” badge (:issue:`2567`) - Add a couple more lines to ``.gitignore`` (:issue:`2557`) -- Deprecate ``Spider.make_requests_from_url`` (:issue:`1728`) - Remove bumpversion prerelease configuration (:issue:`2159`) +- Add codecov.yml file (:issue:`2750`) - Set context factory implementation based on Twisted version (:issue:`2577`, fixes :issue:`2560`) - Add omitted ``self`` arguments in default project middleware template (:issue:`2595`) @@ -172,6 +185,7 @@ Documentation :setting:`DUPEFILTER_CLASS` setting (:issue:`2714`) - Add sphinx_rtd_theme to docs setup readme (:issue:`2668`) - Open file in text mode in JSON item writer example (:issue:`2729`) +- Clarify ``allowed_domains`` example (:issue:`2670`) Scrapy 1.3.3 (2017-03-10) From fc2846d637ca1ebc31e574284d9bc2537969bbe0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 18 May 2017 22:59:46 +0200 Subject: [PATCH 182/362] Set release date for v1.4.0 --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 3c641af44..e0f8eee0b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,7 +3,7 @@ Release notes ============= -Scrapy 1.4.0 (2017-XX-XX) +Scrapy 1.4.0 (2017-05-18) ------------------------- Scrapy 1.4 does not bring that many breathtaking new features From 5f69ec98f70e1e1e5f65fb36eb1cfb23d0be5b45 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 18 May 2017 23:01:05 +0200 Subject: [PATCH 183/362] =?UTF-8?q?Bump=20version:=201.3.2=20=E2=86=92=201?= =?UTF-8?q?.4.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 3 ++- scrapy/VERSION | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 36484c49f..21800f616 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,8 @@ [bumpversion] -current_version = 1.3.2 +current_version = 1.4.0 commit = True tag = True tag_name = {new_version} [bumpversion:file:scrapy/VERSION] + diff --git a/scrapy/VERSION b/scrapy/VERSION index 1892b9267..88c5fb891 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.3.2 +1.4.0 From af2963d0eb79b1241e04b9fc7972bc4a31adf67c Mon Sep 17 00:00:00 2001 From: Kurt Peek Date: Wed, 24 May 2017 15:50:47 +0200 Subject: [PATCH 184/362] Update autothrottle.rst Added missing bullet point for the AUTOTHROTTLE_TARGET_CONCURRENCY setting. --- docs/topics/autothrottle.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index b83946a58..c9bece753 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -88,6 +88,7 @@ The settings used to control the AutoThrottle extension are: * :setting:`AUTOTHROTTLE_ENABLED` * :setting:`AUTOTHROTTLE_START_DELAY` * :setting:`AUTOTHROTTLE_MAX_DELAY` +* :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` * :setting:`AUTOTHROTTLE_DEBUG` * :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` * :setting:`CONCURRENT_REQUESTS_PER_IP` From 80b160d0d739bc3667adf65817a43111f14b9599 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Mon, 29 May 2017 14:56:49 -0300 Subject: [PATCH 185/362] include references to scrapy subreddit in the docs --- docs/contributing.rst | 13 +++++++------ docs/index.rst | 10 +++++----- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index ab3779395..c969bd842 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -22,7 +22,7 @@ There are many ways to contribute to Scrapy. Here are some of them: `Writing patches`_ and `Submitting patches`_ below for details on how to write and submit a patch. -* Join the `scrapy-users`_ mailing list and share your ideas on how to +* Join the `Scrapy subreddit`_ and share your ideas on how to improve Scrapy. We're always open to suggestions. Reporting bugs @@ -44,9 +44,9 @@ guidelines when reporting a new bug. don't dismiss the report but check the ticket history and comments, you may find additional useful information to contribute. -* search the `scrapy-users`_ list to see if it has been discussed there, or - if you're not sure if what you're seeing is a bug. You can also ask in the - `#scrapy` IRC channel. +* search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has + been discussed there, or if you're not sure if what you're seeing is a bug. + You can also ask in the `#scrapy` IRC channel. * write **complete, reproducible, specific bug reports**. The smaller the test case, the better. Remember that other developers won't have your project to @@ -98,8 +98,8 @@ patch, but it's always good to have a patch ready to illustrate your arguments and show that you have put some additional thought into the subject. A good starting point is to send a pull request on GitHub. It can be simple enough to illustrate your idea, and leave documentation/tests for later, after the idea -has been validated and proven useful. Alternatively, you can send an email to -`scrapy-users`_ to discuss your idea first. +has been validated and proven useful. Alternatively, you can start a +conversation in the `Scrapy subreddit`_ to discuss your idea first. When writing GitHub pull requests, try to keep titles short but descriptive. E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests" prefer "Fix hanging when exception occurs in start_requests (#411)" @@ -188,6 +188,7 @@ And their unit-tests are in:: .. _issue tracker: https://github.com/scrapy/scrapy/issues .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users +.. _Scrapy subreddit: http://reddit.com/r/scrapy .. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests diff --git a/docs/index.rst b/docs/index.rst index 289fb2b1b..7e8c979c4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,14 +13,14 @@ Having trouble? We'd like to help! * Try the :doc:`FAQ ` -- it's got answers to some common questions. * Looking for specific information? Try the :ref:`genindex` or :ref:`modindex`. -* Ask or search questions in `StackOverflow using the scrapy tag`_, -* Search for information in the `archives of the scrapy-users mailing list`_, or - `post a question`_. +* Ask or search questions in `StackOverflow using the scrapy tag`_. +* Ask or search questions in the `Scrapy subreddit`_. +* Search for questions on the archives of the `scrapy-users mailing list`_. * Ask a question in the `#scrapy IRC channel`_, * Report bugs with Scrapy in our `issue tracker`_. -.. _archives of the scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users -.. _post a question: https://groups.google.com/forum/#!forum/scrapy-users +.. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users +.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/ .. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy .. _#scrapy IRC channel: irc://irc.freenode.net/scrapy .. _issue tracker: https://github.com/scrapy/scrapy/issues From 083880888b3d4916c3713aae251f42ae4c77e7f7 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 30 May 2017 00:44:11 +0500 Subject: [PATCH 186/362] DOC fixed rst syntax in DOWNLOAD_FAIL_ON_DATALOSS docs --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 9cd639db4..37e3828a4 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -627,7 +627,7 @@ Optionally, this can be set per-request basis by using the circumstances, from server misconfiguration to network errors to data corruption. It is up to the user to decide if it makes sense to process broken responses considering they may contain partial or incomplete content. - If setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``, + If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``, the ``ResponseFailed([_DataLoss])`` failure will be retried as usual. .. setting:: DUPEFILTER_CLASS From 5e1f7a9eadbb5e6bd1ef1f200e7422b77e3377c9 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 30 May 2017 00:50:32 +0500 Subject: [PATCH 187/362] DOC change "releases" section content --- README.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 4eb36b44a..27fab8e29 100644 --- a/README.rst +++ b/README.rst @@ -49,18 +49,17 @@ The quick way:: For more details see the install section in the documentation: http://doc.scrapy.org/en/latest/intro/install.html -Releases -======== - -You can download the latest stable and development releases from: -http://scrapy.org/download/ - Documentation ============= Documentation is available online at http://doc.scrapy.org/ and in the ``docs`` directory. +Releases +======== + +You can find release notes at https://doc.scrapy.org/en/latest/news.html + Community (blog, twitter, mail list, IRC) ========================================= From c8dc158697453b610f168f97ce94945892163674 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 30 May 2017 17:22:23 +0200 Subject: [PATCH 188/362] Use HTTP pool and proper endpoint key for ProxyAgent --- scrapy/core/downloader/handlers/http11.py | 32 +++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9bfdd803c..a4b077b57 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -14,7 +14,7 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from twisted.internet.error import TimeoutError from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool, ResponseFailed + HTTPConnectionPool, ResponseFailed, URI from twisted.internet.endpoints import TCP4ClientEndpoint from scrapy.http import Headers @@ -228,10 +228,33 @@ class TunnelingAgent(Agent): headers, bodyProducer, requestPath) +class ScrapyProxyAgent(Agent): + + def __init__(self, reactor, proxyURI, + connectTimeout=None, bindAddress=None, pool=None): + super(ScrapyProxyAgent, self).__init__(reactor, + connectTimeout=connectTimeout, + bindAddress=bindAddress, + pool=pool) + self._proxyURI = URI.fromBytes(proxyURI) + + def request(self, method, uri, headers=None, bodyProducer=None): + """ + Issue a new request via the configured proxy. + """ + # Cache *all* connections under the same key, since we are only + # connecting to a single destination, the proxy: + proxyEndpoint = self._getEndpoint(self._proxyURI) + key = ("http-proxy", self._proxyURI.host, self._proxyURI.port) + return self._requestWithEndpoint(key, proxyEndpoint, method, + URI.fromBytes(uri), headers, + bodyProducer, uri) + + class ScrapyAgent(object): _Agent = Agent - _ProxyAgent = ProxyAgent + _ProxyAgent = ScrapyProxyAgent _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, @@ -260,9 +283,8 @@ class ScrapyAgent(object): contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) else: - endpoint = TCP4ClientEndpoint(reactor, proxyHost, proxyPort, - timeout=timeout, bindAddress=bindaddress) - return self._ProxyAgent(endpoint) + return self._ProxyAgent(reactor, proxyURI=proxy, + connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) return self._Agent(reactor, contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) From 60727dedf605fad2ed4be844cb2ec44e305257f0 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Wed, 31 May 2017 15:00:38 -0300 Subject: [PATCH 189/362] verify if Request callback is callable --- scrapy/http/request/__init__.py | 4 ++++ tests/test_http_request.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 1435d91de..b9c5f8541 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -27,6 +27,10 @@ class Request(object_ref): assert isinstance(priority, int), "Request priority not an integer: %r" % priority self.priority = priority + if callback is not None and not callable(callback): + raise TypeError('callback must be a function, got %s' % type(callback).__name__) + if errback is not None and not callable(errback): + raise TypeError('errback must be a function, got %s' % type(errback).__name__) assert callback or not errback, "Cannot use errback without a callback" self.callback = callback self.errback = errback diff --git a/tests/test_http_request.py b/tests/test_http_request.py index bbce537f4..9b0ee63dc 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -235,6 +235,26 @@ class RequestTest(unittest.TestCase): self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com') self.assertRaises(AttributeError, setattr, r, 'body', 'xxx') + def test_callback_is_callable(self): + def a_function(): + pass + r = self.request_class('http://example.com') + self.assertIsNone(r.callback) + r = self.request_class('http://example.com', a_function) + self.assertIs(r.callback, a_function) + with self.assertRaises(TypeError): + self.request_class('http://example.com', 'a_function') + + def test_errback_is_callable(self): + def a_function(): + pass + r = self.request_class('http://example.com') + self.assertIsNone(r.errback) + r = self.request_class('http://example.com', a_function, errback=a_function) + self.assertIs(r.errback, a_function) + with self.assertRaises(TypeError): + self.request_class('http://example.com', a_function, errback='a_function') + class FormRequestTest(RequestTest): From e162c1ff40bf74802c1a9f7b93fdbb56a5ff4a13 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 1 Jun 2017 11:58:17 +0200 Subject: [PATCH 190/362] Pass proxy URI to ProxyAgent as bytes --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index a4b077b57..7b77d82da 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -283,7 +283,7 @@ class ScrapyAgent(object): contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) else: - return self._ProxyAgent(reactor, proxyURI=proxy, + return self._ProxyAgent(reactor, proxyURI=to_bytes(proxy, encoding='ascii'), connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) return self._Agent(reactor, contextFactory=self._contextFactory, From 6b092c66809ec11245d03063961ca64e488580e1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 1 Jun 2017 12:29:01 +0200 Subject: [PATCH 191/362] Handle Twisted versions before 15.0 --- scrapy/core/downloader/handlers/http11.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 7b77d82da..216671f82 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -14,7 +14,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from twisted.internet.error import TimeoutError from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool, ResponseFailed, URI + HTTPConnectionPool, ResponseFailed +try: + from twisted.web.client import URI +except ImportError: + from twisted.web.client import _URI as URI from twisted.internet.endpoints import TCP4ClientEndpoint from scrapy.http import Headers @@ -244,7 +248,12 @@ class ScrapyProxyAgent(Agent): """ # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: - proxyEndpoint = self._getEndpoint(self._proxyURI) + if twisted_version >= (15, 0, 0): + proxyEndpoint = self._getEndpoint(self._proxyURI) + else: + proxyEndpoint = self._getEndpoint(self._proxyURI.scheme, + self._proxyURI.host, + self._proxyURI.port) key = ("http-proxy", self._proxyURI.host, self._proxyURI.port) return self._requestWithEndpoint(key, proxyEndpoint, method, URI.fromBytes(uri), headers, From fad6b70d92825b7530e39cc66020273f7f6b836b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 1 Jun 2017 16:37:28 +0200 Subject: [PATCH 192/362] Use https:// for readthedocs links --- docs/intro/install.rst | 2 +- docs/news.rst | 4 ++-- docs/topics/commands.rst | 2 +- docs/topics/deploy.rst | 4 ++-- docs/topics/item-pipeline.rst | 2 +- docs/topics/scrapyd.rst | 2 +- docs/topics/spiders.rst | 4 ++-- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 9cec2eaee..47af8292e 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -107,7 +107,7 @@ Python virtualenvs can be created to use Python 2 by default, or Python 3 by def .. _virtualenv: https://virtualenv.pypa.io .. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/ -.. _virtualenvwrapper: http://virtualenvwrapper.readthedocs.io/en/latest/install.html +.. _virtualenvwrapper: https://virtualenvwrapper.readthedocs.io/en/latest/install.html .. _user guide: https://virtualenv.pypa.io/en/stable/userguide/ diff --git a/docs/news.rst b/docs/news.rst index e0f8eee0b..577c93b8e 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -12,7 +12,7 @@ but quite a few handy improvements nonetheless. Scrapy now supports anonymous FTP sessions with customizable user and password via the new :setting:`FTP_USER` and :setting:`FTP_PASSWORD` settings. And if you're using Twisted version 17.1.0 or above, FTP is now available -with Python 3. +with Python 3. There's a new :meth:`response.follow ` method for creating requests; **it is now a recommended way to create Requests @@ -407,7 +407,7 @@ Refactoring - ``canonicalize_url`` has been moved to `w3lib.url`_ (:issue:`2168`). -.. _w3lib.url: http://w3lib.readthedocs.io/en/latest/w3lib.html#w3lib.url.canonicalize_url +.. _w3lib.url: https://w3lib.readthedocs.io/en/latest/w3lib.html#w3lib.url.canonicalize_url Tests & Requirements ~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 3e69c4e6f..8de858f8a 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -543,7 +543,7 @@ Example:: COMMANDS_MODULE = 'mybot.commands' -.. _Deploying your project: http://scrapyd.readthedocs.org/en/latest/deploy.html +.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html Register commands via setup.py entry points ------------------------------------------- diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index bc48ddce7..f4186ea7a 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -50,10 +50,10 @@ them as needed - the configuration is read from the ``scrapy.cfg`` file just like ``scrapyd-deploy``. .. _Scrapyd: https://github.com/scrapy/scrapyd -.. _Deploying your project: https://scrapyd.readthedocs.org/en/latest/deploy.html +.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html .. _Scrapy Cloud: http://scrapinghub.com/scrapy-cloud/ .. _scrapyd-client: https://github.com/scrapy/scrapyd-client .. _shub: http://doc.scrapinghub.com/shub.html -.. _scrapyd-deploy documentation: http://scrapyd.readthedocs.org/en/latest/deploy.html +.. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html .. _Scrapy Cloud documentation: http://doc.scrapinghub.com/scrapy-cloud.html .. _Scrapinghub: http://scrapinghub.com/ diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 33e4d7429..ac0a5973b 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -208,7 +208,7 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite item["screenshot_filename"] = filename return item -.. _Splash: http://splash.readthedocs.io/en/stable/ +.. _Splash: https://splash.readthedocs.io/en/stable/ .. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html Duplicates filter diff --git a/docs/topics/scrapyd.rst b/docs/topics/scrapyd.rst index 57921b901..a3d6f7698 100644 --- a/docs/topics/scrapyd.rst +++ b/docs/topics/scrapyd.rst @@ -10,4 +10,4 @@ Scrapyd has been moved into a separate project. Its documentation is now hosted at: - http://scrapyd.readthedocs.org/en/latest/ + https://scrapyd.readthedocs.io/en/latest/ diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 49c0cefb5..6ac946003 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -80,7 +80,7 @@ scrapy.Spider allowed to crawl. Requests for URLs not belonging to the domain names specified in this list (or their subdomains) won't be followed if :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled. - + Let's say your target url is ``https://www.example.com/1.html``, then add ``'example.com'`` to the list. @@ -756,4 +756,4 @@ Combine SitemapSpider with other sources of urls:: .. _Sitemap index files: http://www.sitemaps.org/protocol.html#index .. _robots.txt: http://www.robotstxt.org/ .. _TLD: https://en.wikipedia.org/wiki/Top-level_domain -.. _Scrapyd documentation: http://scrapyd.readthedocs.org/en/latest/ +.. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/ From 0f6f486769d7761054274509de08fcf455d492de Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Mon, 5 Jun 2017 16:19:00 -0300 Subject: [PATCH 193/362] fix parse command issue with callback as a string --- scrapy/commands/parse.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 5264982b6..a90095146 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -142,7 +142,8 @@ class Command(ScrapyCommand): logger.error('Unable to find spider for: %(url)s', {'url': url}) - request = Request(url, opts.callback) + # Request requires callback argument as callable or None, not string + request = Request(url, None) _start_requests = lambda s: [self.prepare_request(s, request, opts)] self.spidercls.start_requests = _start_requests @@ -164,7 +165,9 @@ class Command(ScrapyCommand): # determine real callback cb = response.meta['_callback'] if not cb: - if opts.rules and self.first_response == response: + if opts.callback: + cb = opts.callback + elif opts.rules and self.first_response == response: cb = self.get_callback_from_rules(spider, response) if not cb: From 4b6f68b9ee2830534c25b824103b13d502005fe1 Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Mon, 5 Jun 2017 17:26:52 -0300 Subject: [PATCH 194/362] make reqser tests create Request with proper callback/errback --- tests/test_utils_reqser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 073baadc2..dcc070b8f 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -17,8 +17,8 @@ class RequestSerializationTest(unittest.TestCase): def test_all_attributes(self): r = Request("http://www.example.com", - callback='parse_item', - errback='handle_error', + callback=self.spider.parse_item, + errback=self.spider.handle_error, method="POST", body=b"some body", headers={'content-encoding': 'text/html; charset=latin-1'}, @@ -27,7 +27,7 @@ class RequestSerializationTest(unittest.TestCase): priority=20, meta={'a': 'b'}, flags=['testFlag']) - self._assert_serializes_ok(r) + self._assert_serializes_ok(r, spider=self.spider) def test_latin1_body(self): r = Request("http://www.example.com", body=b"\xa3") From 3f8542eb566dd06d35b17574d65955de2682a29e Mon Sep 17 00:00:00 2001 From: Chuan Jin Date: Mon, 5 Jun 2017 23:31:37 +0200 Subject: [PATCH 195/362] Update extensions.rst #2759 --- docs/topics/extensions.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 6036db0f5..d24f579ee 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -277,9 +277,10 @@ CLOSESPIDER_ITEMCOUNT Default: ``0`` An integer which specifies a number of items. If the spider scrapes more than -that amount if items and those items are passed by the item pipeline, the -spider will be closed with the reason ``closespider_itemcount``. If zero (or -non set), spiders won't be closed by number of passed items. +that amount and those items are passed by the item pipeline, the +spider will be closed with the reason ``closespider_itemcount``, requests which +are currently in the downloader queue (up to CONCURRENT_REQUEST requests) are still processed. +If zero (or non set), spiders won't be closed by number of passed items. .. setting:: CLOSESPIDER_PAGECOUNT From e7061f7a4193ea4bd7587c6cac5c9a52f847bb58 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 6 Jun 2017 10:47:43 +0200 Subject: [PATCH 196/362] Reformat a bit --- docs/topics/extensions.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index d24f579ee..03c5f2316 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -278,8 +278,9 @@ Default: ``0`` An integer which specifies a number of items. If the spider scrapes more than that amount and those items are passed by the item pipeline, the -spider will be closed with the reason ``closespider_itemcount``, requests which -are currently in the downloader queue (up to CONCURRENT_REQUEST requests) are still processed. +spider will be closed with the reason ``closespider_itemcount``. +Requests which are currently in the downloader queue (up to +:setting:`CONCURRENT_REQUEST` requests) are still processed. If zero (or non set), spiders won't be closed by number of passed items. .. setting:: CLOSESPIDER_PAGECOUNT From 39ad0d0bddbc3d15b4186598bc69386e7321e17b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 6 Jun 2017 10:48:30 +0200 Subject: [PATCH 197/362] Fix setting name reference --- docs/topics/extensions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 03c5f2316..7a67cf295 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -280,7 +280,7 @@ An integer which specifies a number of items. If the spider scrapes more than that amount and those items are passed by the item pipeline, the spider will be closed with the reason ``closespider_itemcount``. Requests which are currently in the downloader queue (up to -:setting:`CONCURRENT_REQUEST` requests) are still processed. +:setting:`CONCURRENT_REQUESTS` requests) are still processed. If zero (or non set), spiders won't be closed by number of passed items. .. setting:: CLOSESPIDER_PAGECOUNT From ae679f6499b7d63061f11ca11592dcaff5919a00 Mon Sep 17 00:00:00 2001 From: Casker <100347755@alumnos.uc3m.es> Date: Fri, 9 Jun 2017 16:12:20 +0800 Subject: [PATCH 198/362] Create item-pipeline.rst --- docs/topics/item-pipeline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index ac0a5973b..38265b474 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -156,7 +156,7 @@ method and how to clean up the resources properly.:: self.client.close() def process_item(self, item, spider): - self.db[self.collection_name].insert(dict(item)) + self.db[self.collection_name].insert_one(dict(item)) return item .. _MongoDB: https://www.mongodb.org/ From b33e0d5a54a90588e02396e7a2162d4ea12ae7dd Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Wed, 14 Jun 2017 12:17:20 +0800 Subject: [PATCH 199/362] Added: Now supporting tags in Response.follow --- scrapy/http/response/text.py | 9 +++++---- tests/test_http_response.py | 8 +++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6415e191a..74a042f2c 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -135,7 +135,7 @@ class TextResponse(Response): * an attribute Selector (not SelectorList) - e.g. ``response.css('a::attr(href)')[0]`` or ``response.xpath('//img/@src')[0]``. - * a Selector for ```` element, e.g. + * a Selector for ```` or ```` element, e.g. ``response.css('a.my_link')[0]``. See :ref:`response-follow-example` for usage examples. @@ -165,10 +165,11 @@ def _url_from_selector(sel): return strip_html5_whitespace(sel.root) if not hasattr(sel.root, 'tag'): raise ValueError("Unsupported selector: %s" % sel) - if sel.root.tag != 'a': - raise ValueError("Only elements are supported; got <%s>" % + if sel.root.tag not in ('a', 'link'): + raise ValueError("Only and elements are supported; got <%s>" % sel.root.tag) href = sel.root.get('href') if href is None: - raise ValueError(" element has no href attribute: %s" % sel) + raise ValueError("<%s> element has no href attribute: %s" % + (sel.root.tag, sel)) return strip_html5_whitespace(href) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 779f5a71c..a36ec3af6 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -162,7 +162,6 @@ class BaseResponseTest(unittest.TestCase): def test_follow_whitespace_link(self): self._assert_followed_url(Link('http://example.com/foo '), 'http://example.com/foo%20') - def _assert_followed_url(self, follow_obj, target_url, response=None): if response is None: response = self._links_response() @@ -402,6 +401,13 @@ class TextResponseTest(BaseResponseTest): for sel, url in zip(sellist, urls): self._assert_followed_url(sel, url, response=resp) + # select elements + self._assert_followed_url( + Selector(text='').css('link')[0], + 'http://example.com/foo', + response=resp + ) + # href attributes should work for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]: for sel, url in zip(sellist, urls): From f712513ed709e2e6d71a19b4e259934fd6f86955 Mon Sep 17 00:00:00 2001 From: Pengyu CHEN Date: Thu, 15 Jun 2017 10:41:02 +0800 Subject: [PATCH 200/362] Added doc for `scrapy.exceptions.DontCloseSpider`. Also fixes inaccurate doc for `scrapy.signals.spider_idle`. --- docs/topics/exceptions.rst | 8 ++++++++ docs/topics/signals.rst | 10 ++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index cc02369d4..09cb8ed66 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -39,6 +39,14 @@ For example:: if 'Bandwidth exceeded' in response.body: raise CloseSpider('bandwidth_exceeded') +DontCloseSpider +--------------- + +.. exception:: DontCloseSpider + +This exception can be raised in a :signal:`spider_idle` signal handler to +prevent the spider from being closed. + IgnoreRequest ------------- diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 0306ee4a5..cf1588df8 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -189,14 +189,20 @@ spider_idle the engine starts closing the spider. After the spider has finished closing, the :signal:`spider_closed` signal is sent. - You can, for example, schedule some requests in your :signal:`spider_idle` - handler to prevent the spider from being closed. + You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to + prevent the spider from being closed. This signal does not support returning deferreds from their handlers. :param spider: the spider which has gone idle :type spider: :class:`~scrapy.spiders.Spider` object +.. note:: Scheduling some requests in your :signal:`spider_idle` handler does + **not** guarantee that it can prevent the spider from being closed, + although it sometimes can. That's because the spider may still remain idle + if all the scheduled requests are rejected by the scheduler (e.g. filtered + due to duplication). + spider_error ------------ From 5a08cf3b9606bf77ef02a37dca1e2bc76f74558f Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 2 Sep 2016 00:22:22 +0300 Subject: [PATCH 201/362] Fix test_start_requests_errors for PyPy Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies --- scrapy/cmdline.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index e4dc7f2de..b546d030e 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -4,6 +4,7 @@ import optparse import cProfile import inspect import pkg_resources +import gc import scrapy from scrapy.crawler import CrawlerProcess @@ -165,4 +166,9 @@ def _run_command_profiled(cmd, args, opts): p.dump_stats(opts.profile) if __name__ == '__main__': - execute() + try: + execute() + finally: + # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() + # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies + gc.collect() From 6014856df5717496b57aaac6c8a2e64b125ac32b Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 1 Sep 2016 18:39:25 +0300 Subject: [PATCH 202/362] Fix test_output_processor_error undere PyPy For float(u'$10') PyPy includes "u'" in the error message, and it's more fair to check error message on input we are really passing. --- tests/test_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_loader.py b/tests/test_loader.py index 2693a18d9..9d07eb95b 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -290,7 +290,7 @@ class BasicItemLoaderTest(unittest.TestCase): il = TestItemLoader() il.add_value('name', [u'$10']) try: - float('$10') + float(u'$10') except Exception as e: expected_exc_str = str(e) From c3d17659b33432ba8cd2e5b2da57105c1cc0da22 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 1 Sep 2016 15:39:16 +0300 Subject: [PATCH 203/362] Fix queue serialization test on PyPy It is not affected by Twisted bug #7989 and is more permissive with pickling (especially with protocol=2). --- tests/test_squeues.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 48871ceeb..3a24348b4 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,3 +1,5 @@ +import pickle + from queuelib.tests import test_queue as t from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue from scrapy.item import Item, Field @@ -14,6 +16,22 @@ class TestLoader(ItemLoader): default_item_class = TestItem name_out = staticmethod(_test_procesor) +def nonserializable_object_test(self): + try: + pickle.dumps(lambda x: x) + except Exception: + # Trigger Twisted bug #7989 + import twisted.persisted.styles # NOQA + q = self.queue() + self.assertRaises(ValueError, q.push, lambda x: x) + else: + # Use a different unpickleable object + class A(object): pass + a = A() + a.__reduce__ = a.__reduce_ex__ = None + q = self.queue() + self.assertRaises(ValueError, q.push, a) + class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): chunksize = 100000 @@ -30,11 +48,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): self.assertEqual(q.pop(), 123) self.assertEqual(q.pop(), {'a': 'dict'}) - def test_nonserializable_object(self): - # Trigger Twisted bug #7989 - import twisted.persisted.styles # NOQA - q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + test_nonserializable_object = nonserializable_object_test class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 @@ -110,11 +124,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): self.assertEqual(q.pop(), 123) self.assertEqual(q.pop(), 'a') - def test_nonserializable_object(self): - # Trigger Twisted bug #7989 - import twisted.persisted.styles # NOQA - q = self.queue() - self.assertRaises(ValueError, q.push, lambda x: x) + test_nonserializable_object = nonserializable_object_test class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest): From 5abb70c8d71365adbf3a035b1f6d07f5fbbbf446 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 1 Sep 2016 15:43:57 +0300 Subject: [PATCH 204/362] Fix test_weakkeycache on PyPy: run gc.collect() One gc.collect() seems to be enough, but it's more reliable to run it several times (at most 100), until all objects are collected. --- tests/test_utils_python.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 9a0cc975d..e22bd8eb6 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,3 +1,4 @@ +import gc import functools import operator import unittest @@ -144,6 +145,9 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertNotEqual(v, wk[_Weakme()]) self.assertEqual(v, wk[k]) del k + for _ in range(100): + if wk._weakdict: + gc.collect() self.assertFalse(len(wk._weakdict)) @unittest.skipUnless(six.PY2, "deprecated function") From 7c67047e77914dfe0d6666e4dc535c684aa77090 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 1 Sep 2016 18:35:57 +0300 Subject: [PATCH 205/362] Fix get_func_args tests under PyPy On CPython get_func_args does not work correctly for built-in methods. --- tests/test_utils_python.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index e22bd8eb6..8becca0f1 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -3,6 +3,7 @@ import functools import operator import unittest from itertools import count +import platform import six from scrapy.utils.python import ( @@ -212,10 +213,16 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(cal), ['a', 'b', 'c']) self.assertEqual(get_func_args(object), []) - # TODO: how do we fix this to return the actual argument names? - self.assertEqual(get_func_args(six.text_type.split), []) - self.assertEqual(get_func_args(" ".join), []) - self.assertEqual(get_func_args(operator.itemgetter(2)), []) + if platform.python_implementation() == 'CPython': + # TODO: how do we fix this to return the actual argument names? + self.assertEqual(get_func_args(six.text_type.split), []) + self.assertEqual(get_func_args(" ".join), []) + self.assertEqual(get_func_args(operator.itemgetter(2)), []) + else: + self.assertEqual(get_func_args(six.text_type.split), ['sep', 'maxsplit']) + self.assertEqual(get_func_args(" ".join), ['list']) + self.assertEqual(get_func_args(operator.itemgetter(2)), ['obj']) + def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) From 19ca986aa1340b08658caaac0ce677aa22be9814 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 2 Sep 2016 00:32:33 +0300 Subject: [PATCH 206/362] Move garbage_collect to scrapy.utils.python --- scrapy/cmdline.py | 4 ++-- scrapy/utils/python.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index b546d030e..dc6b59fe0 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -4,7 +4,6 @@ import optparse import cProfile import inspect import pkg_resources -import gc import scrapy from scrapy.crawler import CrawlerProcess @@ -12,6 +11,7 @@ from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import inside_project, get_project_settings +from scrapy.utils.python import garbage_collect from scrapy.settings.deprecated import check_deprecated_settings def _iter_command_classes(module_name): @@ -171,4 +171,4 @@ if __name__ == '__main__': finally: # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() # on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies - gc.collect() + garbage_collect() diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 4c500abf4..d28d71bd3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,6 +1,7 @@ """ This module contains essential stuff that should've come with Python itself ;) """ +import gc import os import re import inspect @@ -8,6 +9,8 @@ import weakref import errno import six from functools import partial, wraps +import sys +import time from scrapy.utils.decorators import deprecated @@ -355,3 +358,20 @@ def global_object_name(obj): 'scrapy.http.request.Request' """ return "%s.%s" % (obj.__module__, obj.__name__) + + +if sys.platform.startswith('java'): + def garbage_collect(): + # Some JVM GCs will execute finalizers in a different thread, meaning + # we need to wait for that to complete before we go on looking for the + # effects of that. + gc.collect() + time.sleep(0.1) +elif hasattr(sys, "pypy_version_info"): + def garbage_collect(): + # Collecting weakreferences can take two collections on PyPy. + gc.collect() + gc.collect() +else: + def garbage_collect(): + gc.collect() From b4eb60e5270498a64200332587a065ec9978a333 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 15 Jun 2017 13:24:06 +0300 Subject: [PATCH 207/362] Install PyPyDispatcher for PyPy tests Using https://github.com/lopuhin/pydispatcher, pypy branch. This is executed as a separate step to avoid changing default requirements.txt and setup.py. If just added to "deps" in tox, this install command will be executed as one command and PyPyDispatcher will not override PyDispatcher. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 6987847f8..c559f1e47 100644 --- a/tox.ini +++ b/tox.ini @@ -57,6 +57,7 @@ commands = [testenv:pypy] basepython = pypy commands = + pip install PyPyDispatcher>=2.0.6 py.test {posargs:scrapy tests} [testenv:py33] From a8df0900713a1a56cffca8774bc137a6e781877e Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 15 Jun 2017 14:29:28 +0300 Subject: [PATCH 208/362] Fix httpcache leveldb tests: gc.collect after del LevelDB does not have "official" close method, so we have to rely on garbage collection to close it. --- scrapy/extensions/httpcache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 2fb4b6a15..648b32ec7 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -13,7 +13,7 @@ from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.project import data_path from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.python import to_bytes, to_unicode, garbage_collect logger = logging.getLogger(__name__) @@ -362,6 +362,7 @@ class LeveldbCacheStorage(object): # avoid them being removed in storages with timestamp-based autoremoval. self.db.CompactRange() del self.db + garbage_collect() def retrieve_response(self, spider, request): data = self._read_data(spider, request) From ea08b952801b9af9bc45ae9918fcf5ab9bcb5aaf Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 19 Jun 2017 16:45:29 +0300 Subject: [PATCH 209/362] Remove Jython gc branch: it's not supported --- scrapy/utils/python.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index d28d71bd3..72f8f4311 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -360,14 +360,7 @@ def global_object_name(obj): return "%s.%s" % (obj.__module__, obj.__name__) -if sys.platform.startswith('java'): - def garbage_collect(): - # Some JVM GCs will execute finalizers in a different thread, meaning - # we need to wait for that to complete before we go on looking for the - # effects of that. - gc.collect() - time.sleep(0.1) -elif hasattr(sys, "pypy_version_info"): +if hasattr(sys, "pypy_version_info"): def garbage_collect(): # Collecting weakreferences can take two collections on PyPy. gc.collect() From 271b3a485cbdce1b0a866d9fb9938d46d7ddb497 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 19 Jun 2017 16:46:16 +0300 Subject: [PATCH 210/362] Require pypy build to pass --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 906115096..449eee96b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,8 @@ matrix: env: TOXENV=py27 - python: 2.7 env: TOXENV=jessie + - python: 2.7 + env: TOXENV=pypy - python: 3.3 env: TOXENV=py33 - python: 3.5 @@ -21,9 +23,6 @@ matrix: env: TOXENV=pypy - python: 3.6 env: TOXENV=docs - allow_failures: - - python: 2.7 - env: TOXENV=pypy install: - | if [ "$TOXENV" = "pypy" ]; then From 5ba8e5adc0e1aa7d9430b63a0094c6a71c51564a Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 19 Jun 2017 17:45:28 +0300 Subject: [PATCH 211/362] Remove duplicate PyPy toxenv from Travis config Thanks for the catch @redapple --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 449eee96b..4f44d1e6d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,8 +19,6 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - - python: 2.7 - env: TOXENV=pypy - python: 3.6 env: TOXENV=docs install: From b0a9236357dd74381b5a014e3de15a3a52de4f7d Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 19 Jun 2017 19:16:50 +0300 Subject: [PATCH 212/362] Use environment markers for custom PyPy requirements --- setup.py | 23 ++++++++++++++++++++++- tox.ini | 1 - 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 086ab8142..c03f0b9f7 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,31 @@ from os.path import dirname, join -from setuptools import setup, find_packages +from pkg_resources import parse_version +from setuptools import setup, find_packages, __version__ as setuptools_version with open(join(dirname(__file__), 'scrapy/VERSION'), 'rb') as f: version = f.read().decode('ascii').strip() +def has_environment_marker_platform_impl_support(): + """Code extracted from 'pytest/setup.py' + https://github.com/pytest-dev/pytest/blob/7538680c/setup.py#L31 + + The first known release to support environment marker with range operators + it is 18.5, see: + https://setuptools.readthedocs.io/en/latest/history.html#id235 + """ + return parse_version(setuptools_version) >= parse_version('18.5') + + +extras_require = {} + +if has_environment_marker_platform_impl_support(): + extras_require[':platform_python_implementation == "PyPy"'] = [ + 'PyPyDispatcher>=2.1.0', + ] + + setup( name='Scrapy', version=version, @@ -53,4 +73,5 @@ setup( 'PyDispatcher>=2.0.5', 'service_identity', ], + extras_require=extras_require, ) diff --git a/tox.ini b/tox.ini index c559f1e47..6987847f8 100644 --- a/tox.ini +++ b/tox.ini @@ -57,7 +57,6 @@ commands = [testenv:pypy] basepython = pypy commands = - pip install PyPyDispatcher>=2.0.6 py.test {posargs:scrapy tests} [testenv:py33] From 793b2376f83caa01d3e883f7642147de8e174c01 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Jul 2017 11:28:04 -0300 Subject: [PATCH 213/362] Populate spider variable when using shell.inspect_response --- scrapy/shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/shell.py b/scrapy/shell.py index 6f94635a1..80b625633 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -164,7 +164,7 @@ class Shell(object): def inspect_response(response, spider): """Open a shell to inspect the given response""" - Shell(spider.crawler).start(response=response) + Shell(spider.crawler).start(response=response, spider=spider) def _request_deferred(request): From 1f08d9a64884a05a541fe9649e3349cce5aa47be Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 4 Jul 2017 23:10:19 +0200 Subject: [PATCH 214/362] Add test for DNS cache disabling --- tests/test_commands.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index 922098668..cb1301c95 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -226,6 +226,27 @@ class MySpider(scrapy.Spider): self.assertNotIn("DEBUG: It Works!", log) self.assertIn("INFO: Spider opened", log) + def test_runspider_dnscache_disabled(self): + # see https://github.com/scrapy/scrapy/issues/2811 + # The spider below should not be able to connect to localhost:12345, + # which is intended, + # but this should not be because of DNS lookup error + # assumption: localhost will resolve in all cases (true?) + log = self.get_log(""" +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + start_urls = ['http://localhost:12345'] + + def parse(self, response): + return {'test': 'value'} +""", + args=('-s', 'DNSCACHE_ENABLED=False')) + print(log) + self.assertNotIn("DNSLookupError", log) + self.assertIn("INFO: Spider opened", log) + def test_runspider_log_short_names(self): log1 = self.get_log(self.debug_log_spider, args=('-s', 'LOG_SHORT_NAMES=1')) From f0ded6b7759c9c2ec4b8946cdfc4abab97bb14d9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 4 Jul 2017 23:18:15 +0200 Subject: [PATCH 215/362] Do not cache DNS responses when cache size is 0 --- scrapy/resolver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 4f4f0b04f..0aaced7e4 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -22,7 +22,8 @@ class CachingThreadedResolver(ThreadedResolver): # to enforce Scrapy's DNS_TIMEOUT setting's value timeout = (self.timeout,) d = super(CachingThreadedResolver, self).getHostByName(name, timeout) - d.addCallback(self._cache_result, name) + if dnscache.limit: + d.addCallback(self._cache_result, name) return d def _cache_result(self, result, name): From dedc4a8b8f7cd3870f89bec2bf89a5ec11a95e4f Mon Sep 17 00:00:00 2001 From: Danny Guo Date: Thu, 13 Jul 2017 22:58:10 -0500 Subject: [PATCH 216/362] Tweak the CSVFeedSpider documentation --- docs/topics/spiders.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 6ac946003..bf1532d1b 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -578,8 +578,7 @@ CSVFeedSpider .. attribute:: headers - A list of the rows contained in the file CSV feed which will be used to - extract fields from it. + A list of the column names in the CSV file. .. method:: parse_row(response, row) From 18b96dd82af0ac58a2fc6e1ba2227b4468ab2bb7 Mon Sep 17 00:00:00 2001 From: Claus Conrad Date: Sat, 15 Jul 2017 11:31:09 +0200 Subject: [PATCH 217/362] Spelling mistake --- docs/topics/broad-crawls.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 28ed7c064..040c8cfde 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -85,8 +85,8 @@ When doing broad crawls you are often only interested in the crawl rates you get and any errors found. These stats are reported by Scrapy when using the ``INFO`` log level. In order to save CPU (and log storage requirements) you should not use ``DEBUG`` log level when preforming large broad crawls in -production. Using ``DEBUG`` level when developing your (broad) crawler may fine -though. +production. Using ``DEBUG`` level when developing your (broad) crawler may be +fine though. To set the log level use:: From 26c488970c51c06506cfc88c4a76440c009cdd48 Mon Sep 17 00:00:00 2001 From: Danny Guo Date: Tue, 18 Jul 2017 19:56:51 -0500 Subject: [PATCH 218/362] Fix a typo in the Items documentation --- docs/topics/items.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 4a8f47e93..4423bbda2 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -21,7 +21,7 @@ their available fields. Various Scrapy components use extra information provided by Items: exporters look at declared fields to figure out columns to export, serialization can be customized using Item fields metadata, :mod:`trackref` -tracks Item instances to help finding memory leaks +tracks Item instances to help find memory leaks (see :ref:`topics-leaks-trackrefs`), etc. .. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict From bb0bd691d9dd42725af6291a15e357f0146de17f Mon Sep 17 00:00:00 2001 From: Valdir Stumm Junior Date: Mon, 24 Jul 2017 11:12:09 -0300 Subject: [PATCH 219/362] Improve error message when callback is not callable --- scrapy/http/request/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index b9c5f8541..13a92ffa0 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -28,9 +28,9 @@ class Request(object_ref): self.priority = priority if callback is not None and not callable(callback): - raise TypeError('callback must be a function, got %s' % type(callback).__name__) + raise TypeError('callback must be a callable, got %s' % type(callback).__name__) if errback is not None and not callable(errback): - raise TypeError('errback must be a function, got %s' % type(errback).__name__) + raise TypeError('errback must be a callable, got %s' % type(errback).__name__) assert callback or not errback, "Cannot use errback without a callback" self.callback = callback self.errback = errback From 1a18587d41cc7dabc0a829fc0b69016b054d64f8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 24 Jul 2017 19:30:08 +0200 Subject: [PATCH 220/362] Jessi toxenv: Add cryptography as per https://packages.debian.org/jessie/python-cryptography --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 6987847f8..c7e1e43c9 100644 --- a/tox.ini +++ b/tox.ini @@ -38,6 +38,7 @@ deps = # https://packages.debian.org/en/jessie/zope/ basepython = python2.7 deps = + cryptography==0.6.1 pyOpenSSL==0.14 lxml==3.4.0 Twisted==14.0.2 From 33dfac50185b1940dda89bec3e3480a7e76e9ca7 Mon Sep 17 00:00:00 2001 From: cclauss Date: Mon, 24 Jul 2017 22:06:17 +0200 Subject: [PATCH 221/362] xrange() --> range() for Python 3 Either this PR or #2845. --- extras/qpsclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index bb83588dd..7554f7eec 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -41,7 +41,7 @@ class QPSSpider(Spider): slots = int(self.slots) if slots > 1: - urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in xrange(slots)] + urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)] else: urls = [url] From 11a1f970b7b7b68b3d968df4b29c4269ab220ac6 Mon Sep 17 00:00:00 2001 From: Pengyu Chen Date: Wed, 26 Jul 2017 16:11:13 +0800 Subject: [PATCH 222/362] Added: HTTP status code 522/524 to retry. --- scrapy/settings/default_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 87dbf6974..697314b7f 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -234,7 +234,7 @@ REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests -RETRY_HTTP_CODES = [500, 502, 503, 504, 408] +RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408] RETRY_PRIORITY_ADJUST = -1 ROBOTSTXT_OBEY = False From 5dc9a88c347db3497b03949938184ca339f4e9cb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 24 Jul 2017 18:10:58 +0200 Subject: [PATCH 223/362] Handle HTTP 308 Permanent Redirect --- scrapy/downloadermiddlewares/redirect.py | 4 ++-- tests/test_downloadermiddleware_redirect.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 26677e527..30cae3fee 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -64,7 +64,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): request.meta.get('handle_httpstatus_all', False)): return response - allowed_status = (301, 302, 303, 307) + allowed_status = (301, 302, 303, 307, 308) if 'Location' not in response.headers or response.status not in allowed_status: return response @@ -72,7 +72,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): redirected_url = urljoin(request.url, location) - if response.status in (301, 307) or request.method == 'HEAD': + if response.status in (301, 307, 308) or request.method == 'HEAD': redirected = request.replace(url=redirected_url) return self._redirect(redirected, request, spider, response.status) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index e8c92affa..a2da4aa8f 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -22,12 +22,12 @@ class RedirectMiddlewareTest(unittest.TestCase): req2 = self.mw.process_response(req, rsp, self.spider) assert req2.priority > req.priority - def test_redirect_301(self): - def _test(method): - url = 'http://www.example.com/301' + def test_redirect_3xx_permanent(self): + def _test(method, status=301): + url = 'http://www.example.com/{}'.format(status) url2 = 'http://www.example.com/redirected' req = Request(url, method=method) - rsp = Response(url, headers={'Location': url2}, status=301) + rsp = Response(url, headers={'Location': url2}, status=status) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) @@ -42,6 +42,10 @@ class RedirectMiddlewareTest(unittest.TestCase): _test('POST') _test('HEAD') + _test('GET', status=308) + _test('POST', status=308) + _test('HEAD', status=308) + def test_dont_redirect(self): url = 'http://www.example.com/301' url2 = 'http://www.example.com/redirected' From 1fdc10684fc427e5446350cada465ec934330a3f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 24 Jul 2017 18:25:11 +0200 Subject: [PATCH 224/362] HTTP Cache: treat 308 as 301 --- tests/test_downloadermiddleware_httpcache.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 12b69860a..22946b98c 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -322,6 +322,7 @@ class RFC2616PolicyTest(DefaultStorageTest): (True, 203, {'Last-Modified': self.yesterday}), (True, 300, {'Last-Modified': self.yesterday}), (True, 301, {'Last-Modified': self.yesterday}), + (True, 308, {'Last-Modified': self.yesterday}), (True, 401, {'Last-Modified': self.yesterday}), (True, 404, {'Cache-Control': 'public, max-age=600'}), (True, 302, {'Expires': self.tomorrow}), From 15a5c533fa6f448b7e5cd72ef099725c2295ef6f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 26 Jul 2017 19:07:57 +0200 Subject: [PATCH 225/362] Add tests for HTTP 307 permanent redirects --- tests/test_downloadermiddleware_redirect.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index a2da4aa8f..35e474418 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -42,6 +42,10 @@ class RedirectMiddlewareTest(unittest.TestCase): _test('POST') _test('HEAD') + _test('GET', status=307) + _test('POST', status=307) + _test('HEAD', status=307) + _test('GET', status=308) _test('POST', status=308) _test('HEAD', status=308) From 219c8aa0b622260b9814379be41132367ef33e39 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 27 Jul 2017 17:30:30 +0200 Subject: [PATCH 226/362] Log versions information at startup --- scrapy/commands/version.py | 40 +++---------------------------- scrapy/utils/log.py | 48 +++++++++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 38 deletions(-) diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index e22f98f5a..92b70d88b 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,12 +1,8 @@ from __future__ import print_function -import sys -import platform - -import twisted -import OpenSSL import scrapy from scrapy.commands import ScrapyCommand +from scrapy.utils.log import scrapy_components_versions class Command(ScrapyCommand): @@ -27,38 +23,8 @@ class Command(ScrapyCommand): def run(self, args, opts): if opts.verbose: - import cssselect - import parsel - import lxml.etree - import w3lib - - lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) - libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) - - try: - w3lib_version = w3lib.__version__ - except AttributeError: - w3lib_version = "<1.14.3" - - print("Scrapy : %s" % scrapy.__version__) - print("lxml : %s" % lxml_version) - print("libxml2 : %s" % libxml2_version) - print("cssselect : %s" % cssselect.__version__) - print("parsel : %s" % parsel.__version__) - print("w3lib : %s" % w3lib_version) - print("Twisted : %s" % twisted.version.short()) - print("Python : %s" % sys.version.replace("\n", "- ")) - print("pyOpenSSL : %s" % self._get_openssl_version()) - print("Platform : %s" % platform.platform()) + for name, version in scrapy_components_versions(): + print("%-9s : %s" % (name, version)) else: print("Scrapy %s" % scrapy.__version__) - def _get_openssl_version(self): - try: - openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\ - .decode('ascii', errors='replace') - # pyOpenSSL 0.12 does not expose openssl version - except AttributeError: - openssl = 'Unknown OpenSSL version' - - return '{} ({})'.format(OpenSSL.version.__version__, openssl) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 6ceb61a82..660b3c9f5 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -139,10 +139,56 @@ def _get_handler(settings): return handler +def scrapy_components_versions(): + import platform + + import cssselect + import parsel + import lxml.etree + import twisted + import w3lib + + lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) + libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) + + try: + w3lib_version = w3lib.__version__ + except AttributeError: + w3lib_version = "<1.14.3" + + return [ + ("Scrapy", scrapy.__version__), + ("lxml", lxml_version), + ("libxml2", libxml2_version), + ("cssselect", cssselect.__version__), + ("parsel", parsel.__version__), + ("w3lib", w3lib_version), + ("Twisted", twisted.version.short()), + ("Python", sys.version.replace("\n", "- ")), + ("pyOpenSSL", _get_openssl_version()), + ("Platform", platform.platform()), + ] + + +def _get_openssl_version(): + try: + import OpenSSL + openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\ + .decode('ascii', errors='replace') + # pyOpenSSL 0.12 does not expose openssl version + except AttributeError: + openssl = 'Unknown OpenSSL version' + + return '{} ({})'.format(OpenSSL.version.__version__, openssl) + + def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) - + logger.info("Versions: %(versions)s}", + {'versions': ", ".join("%s %s" % (name, version) + for name, version in scrapy_components_versions() + if name != "Scrapy")}) d = dict(overridden_settings(settings)) logger.info("Overridden settings: %(settings)r", {'settings': d}) From bf7ef3e4c3ea70a60f6e5f9e5c8a6d842e2b72b9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 27 Jul 2017 20:07:14 +0200 Subject: [PATCH 227/362] Move methods to a new scrapy.utils.versions --- scrapy/commands/version.py | 2 +- scrapy/utils/log.py | 45 ++------------------------------------ scrapy/utils/versions.py | 44 +++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 44 deletions(-) create mode 100644 scrapy/utils/versions.py diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 92b70d88b..71b1026fa 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -2,7 +2,7 @@ from __future__ import print_function import scrapy from scrapy.commands import ScrapyCommand -from scrapy.utils.log import scrapy_components_versions +from scrapy.utils.versions import scrapy_components_versions class Command(ScrapyCommand): diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 660b3c9f5..905c1bfc1 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -11,6 +11,8 @@ from twisted.python import log as twisted_log import scrapy from scrapy.settings import overridden_settings, Settings from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.versions import scrapy_components_versions + logger = logging.getLogger(__name__) @@ -139,49 +141,6 @@ def _get_handler(settings): return handler -def scrapy_components_versions(): - import platform - - import cssselect - import parsel - import lxml.etree - import twisted - import w3lib - - lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) - libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) - - try: - w3lib_version = w3lib.__version__ - except AttributeError: - w3lib_version = "<1.14.3" - - return [ - ("Scrapy", scrapy.__version__), - ("lxml", lxml_version), - ("libxml2", libxml2_version), - ("cssselect", cssselect.__version__), - ("parsel", parsel.__version__), - ("w3lib", w3lib_version), - ("Twisted", twisted.version.short()), - ("Python", sys.version.replace("\n", "- ")), - ("pyOpenSSL", _get_openssl_version()), - ("Platform", platform.platform()), - ] - - -def _get_openssl_version(): - try: - import OpenSSL - openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\ - .decode('ascii', errors='replace') - # pyOpenSSL 0.12 does not expose openssl version - except AttributeError: - openssl = 'Unknown OpenSSL version' - - return '{} ({})'.format(OpenSSL.version.__version__, openssl) - - def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py new file mode 100644 index 000000000..d2cff09fe --- /dev/null +++ b/scrapy/utils/versions.py @@ -0,0 +1,44 @@ +import platform +import sys + +import cssselect +import lxml.etree +import parsel +import twisted +import w3lib + +import scrapy + + +def scrapy_components_versions(): + lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION)) + libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION)) + try: + w3lib_version = w3lib.__version__ + except AttributeError: + w3lib_version = "<1.14.3" + + return [ + ("Scrapy", scrapy.__version__), + ("lxml", lxml_version), + ("libxml2", libxml2_version), + ("cssselect", cssselect.__version__), + ("parsel", parsel.__version__), + ("w3lib", w3lib_version), + ("Twisted", twisted.version.short()), + ("Python", sys.version.replace("\n", "- ")), + ("pyOpenSSL", _get_openssl_version()), + ("Platform", platform.platform()), + ] + + +def _get_openssl_version(): + try: + import OpenSSL + openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\ + .decode('ascii', errors='replace') + # pyOpenSSL 0.12 does not expose openssl version + except AttributeError: + openssl = 'Unknown OpenSSL version' + + return '{} ({})'.format(OpenSSL.version.__version__, openssl) From aaaa4da7a4e1fe75396028189de88fb9a6604200 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 24 May 2017 12:42:54 +0200 Subject: [PATCH 228/362] Add template for a downloader middleware --- .../project/module/middlewares.py.tmpl | 47 +++++++++++++++++++ .../templates/project/module/settings.py.tmpl | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 292bf572e..1a4b0caa5 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -54,3 +54,50 @@ class ${ProjectName}SpiderMiddleware(object): def spider_opened(self, spider): spider.logger.info('Spider opened: %s' % spider.name) + + +class ${ProjectName}DownloaderMiddleware(object): + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 486df6b71..35a0f9a45 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -53,7 +53,7 @@ ROBOTSTXT_OBEY = True # Enable or disable downloader middlewares # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { -# '$project_name.middlewares.MyCustomDownloaderMiddleware': 543, +# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543, #} # Enable or disable extensions From a65fec050ae4a07d233ae886457daf10f14929fe Mon Sep 17 00:00:00 2001 From: simik-ru Date: Sun, 30 Jul 2017 17:04:02 +0300 Subject: [PATCH 229/362] Small fix in description of startproject arguments --- docs/topics/commands.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8de858f8a..dc8067d7e 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -187,7 +187,7 @@ startproject Creates a new Scrapy project named ``project_name``, under the ``project_dir`` directory. -If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``myproject``. +If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``project_name``. Usage example:: From 6e6b5cc29f15dbf4f1941fca70dd9c126e4ba556 Mon Sep 17 00:00:00 2001 From: Andrei Petre Date: Tue, 1 Aug 2017 17:14:43 +0300 Subject: [PATCH 230/362] Use getfullargspec under the scenes for py3 to stop DeprecationWarning (#2864) Use getfullargspec under the scenes for py3 to stop DeprecationWarning. Closes #2862 --- scrapy/utils/python.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 72f8f4311..732ca13a0 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -10,7 +10,6 @@ import errno import six from functools import partial, wraps import sys -import time from scrapy.utils.decorators import deprecated @@ -198,10 +197,30 @@ def binary_is_text(data): return all(c not in _BINARYCHARS for c in data) +def _getargspec_py23(func): + """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords, + defaults) + + Identical to inspect.getargspec() in python2, but uses + inspect.getfullargspec() for python3 behind the scenes to avoid + DeprecationWarning. + + >>> def f(a, b=2, *ar, **kw): + ... pass + + >>> _getargspec_py23(f) + ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,)) + """ + if six.PY2: + return inspect.getargspec(func) + + return inspect.ArgSpec(*inspect.getfullargspec(func)[:4]) + + def get_func_args(func, stripself=False): """Return the argument name list of a callable""" if inspect.isfunction(func): - func_args, _, _, _ = inspect.getargspec(func) + func_args, _, _, _ = _getargspec_py23(func) elif inspect.isclass(func): return get_func_args(func.__init__, True) elif inspect.ismethod(func): @@ -248,9 +267,9 @@ def get_spec(func): """ if inspect.isfunction(func) or inspect.ismethod(func): - spec = inspect.getargspec(func) + spec = _getargspec_py23(func) elif hasattr(func, '__call__'): - spec = inspect.getargspec(func.__call__) + spec = _getargspec_py23(func.__call__) else: raise TypeError('%s is not callable' % type(func)) From 71d5b7d75a579360ab02b9f9594199993ff91f61 Mon Sep 17 00:00:00 2001 From: david watson Date: Tue, 1 Aug 2017 13:49:22 -0400 Subject: [PATCH 231/362] fix typo (#2867) --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 040c8cfde..eb02086dc 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -20,7 +20,7 @@ These are some common properties often found in broad crawls: * they crawl many domains (often, unbounded) instead of a specific set of sites -* they don't necessarily crawl domains to completion, because it would +* they don't necessarily crawl domains to completion, because it would be impractical (or impossible) to do so, and instead limit the crawl by time or number of pages crawled From 01ac8838934071b89c5c711688f3d833387e27d4 Mon Sep 17 00:00:00 2001 From: Eugene Vorobev Date: Wed, 26 Jul 2017 19:51:27 +0300 Subject: [PATCH 232/362] Follow alternate link for all types of sitemaps --- scrapy/spiders/sitemap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index e54001d88..0ee8ba5e7 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -48,7 +48,7 @@ class SitemapSpider(Spider): if any(x.search(loc) for x in self._follow): yield Request(loc, callback=self._parse_sitemap) elif s.type == 'urlset': - for loc in iterloc(s): + for loc in iterloc(s, self.sitemap_alternate_links): for r, c in self._cbs: if r.search(loc): yield Request(loc, callback=c) From 0cb3085f8453265a8f37de684e62eda8b5398c75 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 3 Aug 2017 16:35:47 +0200 Subject: [PATCH 233/362] Add test for alternate links --- tests/test_spider.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index e55f0fa6d..2507964b5 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -348,6 +348,33 @@ Sitemap: /sitemap-relative-url.xml 'http://example.com/sitemap-uppercase.xml', 'http://www.example.com/sitemap-relative-url.xml']) + def test_alternate_url_locs(self): + sitemap = b""" + + + http://www.example.com/english/ + + + + + + """ + r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap) + spider = self.spider_class("example.com") + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/']) + + spider.sitemap_alternate_links = True + self.assertEqual([req.url for req in spider._parse_sitemap(r)], + ['http://www.example.com/english/', + 'http://www.example.com/deutsch/', + 'http://www.example.com/schweiz-deutsch/', + 'http://www.example.com/italiano/']) + class DeprecationTest(unittest.TestCase): From c016a4309dbb045c17842e329043b7d9951e8f14 Mon Sep 17 00:00:00 2001 From: cclauss Date: Fri, 4 Aug 2017 01:44:23 +0200 Subject: [PATCH 234/362] # noqa to close #2836 Marks #2836 as will not fix. --- tests/test_item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_item.py b/tests/test_item.py index 85a554de0..3c645649c 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -270,7 +270,7 @@ class ItemMetaTest(unittest.TestCase): def f(self): # For rationale of this see: # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 - return __class__ + return __class__ # noqa https://github.com/scrapy/scrapy/issues/2836 MyItem() From 0a69a32b5a2cab7575fdbb5f2cd4b7c7b900aabc Mon Sep 17 00:00:00 2001 From: cclauss Date: Fri, 4 Aug 2017 14:35:43 +0200 Subject: [PATCH 235/362] Force Travis CI to test again --- tests/test_item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_item.py b/tests/test_item.py index 3c645649c..2c1eb0dd3 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -270,7 +270,7 @@ class ItemMetaTest(unittest.TestCase): def f(self): # For rationale of this see: # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 - return __class__ # noqa https://github.com/scrapy/scrapy/issues/2836 + return __class__ # noqa https://github.com/scrapy/scrapy/issues/2836 MyItem() From be71f98e92688c759d3af48101617229dcdfe05f Mon Sep 17 00:00:00 2001 From: kirankoduru Date: Sat, 29 Jul 2017 20:51:54 -0400 Subject: [PATCH 236/362] Explicit message for scrapy parse callback The scrapy parse method raises a NotImplementedError when not defined, but for new comers it can be hard to debug what might be going wrong. Adding an explicit message for NotImplementedError will help new users. --- scrapy/spiders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 30cb7590a..c6b92f8eb 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -87,7 +87,7 @@ class Spider(object_ref): return Request(url, dont_filter=True) def parse(self, response): - raise NotImplementedError + raise NotImplementedError('Spider.parse callback is not defined') @classmethod def update_settings(cls, settings): From 7adab61a7a5f88c78311cc44a468c7b8d0a4c954 Mon Sep 17 00:00:00 2001 From: kirankoduru Date: Tue, 1 Aug 2017 22:42:26 -0400 Subject: [PATCH 237/362] Added test for NotImplemented Spider.parse method --- tests/test_spider.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index e55f0fa6d..6c845d826 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -429,3 +429,17 @@ class DeprecationTest(unittest.TestCase): self.assertEqual(len(requests), 1) self.assertEqual(requests[0].url, 'http://example.com/foo') self.assertEqual(len(w), 1) + + +class NoParseMethodSpiderTest(unittest.TestCase): + + spider_class = Spider + + def test_undefined_parse_method(self): + spider = self.spider_class('example.com') + text = 'Random text response' + resp = TextResponse(url="http://www.example.com/random_url", body=text) + + exc_msg = 'Spider.parse callback is not defined' + with self.assertRaisesRegexp(NotImplementedError, exc_msg): + spider.parse(resp) From 12409a0cf6c37ff5c19588bb064690549798bb37 Mon Sep 17 00:00:00 2001 From: Kiran Koduru Date: Wed, 2 Aug 2017 08:32:38 -0400 Subject: [PATCH 238/362] Fix broken encoding on text for py 3 --- tests/test_spider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index 6c845d826..6a52b3ea7 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -437,7 +437,7 @@ class NoParseMethodSpiderTest(unittest.TestCase): def test_undefined_parse_method(self): spider = self.spider_class('example.com') - text = 'Random text response' + text = b'Random text' resp = TextResponse(url="http://www.example.com/random_url", body=text) exc_msg = 'Spider.parse callback is not defined' From 2960c9b5683dceb149823e7f927d8c86ee83deb8 Mon Sep 17 00:00:00 2001 From: Kiran Koduru Date: Sat, 5 Aug 2017 16:29:41 -0400 Subject: [PATCH 239/362] Use self.__class__.__name__ instead of showing generic Spider class name --- scrapy/spiders/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index c6b92f8eb..e9c131e3b 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -87,7 +87,7 @@ class Spider(object_ref): return Request(url, dont_filter=True) def parse(self, response): - raise NotImplementedError('Spider.parse callback is not defined') + raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__)) @classmethod def update_settings(cls, settings): From 4ca61a20512b7306e7266e6667a0c02ae5ebe557 Mon Sep 17 00:00:00 2001 From: Chomba Ng'ang'a Date: Mon, 7 Aug 2017 18:29:36 +0300 Subject: [PATCH 240/362] Update deprecated test aliases - change ``failIf`` to ``assertFalse`` - change ``asertEquals`` to ``assertEqual`` - change ``assert_`` to ``assertTrue`` https://docs.python.org/2/library/unittest.html#deprecated-aliases --- tests/test_cmdline/__init__.py | 2 +- tests/test_downloader_handlers.py | 88 +++++++++---------- tests/test_downloadermiddleware_cookies.py | 20 ++--- ...est_downloadermiddleware_defaultheaders.py | 6 +- ...st_downloadermiddleware_downloadtimeout.py | 8 +- tests/test_downloadermiddleware_httpauth.py | 4 +- ...st_downloadermiddleware_httpcompression.py | 2 +- tests/test_downloadermiddleware_httpproxy.py | 44 +++++----- tests/test_downloadermiddleware_redirect.py | 4 +- tests/test_downloadermiddleware_useragent.py | 6 +- tests/test_http_cookies.py | 2 +- tests/test_http_request.py | 4 +- tests/test_loader.py | 18 ++-- tests/test_pipeline_images.py | 16 ++-- tests/test_selector.py | 2 +- tests/test_spider.py | 6 +- tests/test_spidermiddleware_depth.py | 8 +- tests/test_spidermiddleware_httperror.py | 24 ++--- tests/test_spidermiddleware_offsite.py | 6 +- tests/test_spidermiddleware_referer.py | 10 +-- tests/test_spidermiddleware_urllength.py | 2 +- tests/test_urlparse_monkeypatches.py | 8 +- tests/test_utils_datatypes.py | 8 +- tests/test_utils_defer.py | 2 +- tests/test_utils_iterators.py | 4 +- tests/test_utils_misc/__init__.py | 8 +- tests/test_utils_project.py | 8 +- tests/test_utils_python.py | 26 +++--- tests/test_utils_signal.py | 2 +- tests/test_webclient.py | 28 +++--- 30 files changed, 188 insertions(+), 188 deletions(-) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 7733e7180..10076bbca 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -68,4 +68,4 @@ class CmdlineTest(unittest.TestCase): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys()) - self.assertEquals(200, settingsdict[EXT_PATH]) + self.assertEqual(200, settingsdict[EXT_PATH]) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 7d88dbcba..bd2c86292 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -98,9 +98,9 @@ class FileTestCase(unittest.TestCase): def test_download(self): def _test(response): - self.assertEquals(response.url, request.url) - self.assertEquals(response.status, 200) - self.assertEquals(response.body, b'0123456789') + self.assertEqual(response.url, request.url) + self.assertEqual(response.status, 200) + self.assertEqual(response.body, b'0123456789') request = Request(path_to_file_uri(self.tmpname + '^')) assert request.url.upper().endswith('%5E') @@ -241,28 +241,28 @@ class HttpTestCase(unittest.TestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") return d def test_download_head(self): request = Request(self.getURL('file'), method='HEAD') d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b'') + d.addCallback(self.assertEqual, b'') return d def test_redirect_status(self): request = Request(self.getURL('redirect')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.status) - d.addCallback(self.assertEquals, 302) + d.addCallback(self.assertEqual, 302) return d def test_redirect_status_head(self): request = Request(self.getURL('redirect'), method='HEAD') d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.status) - d.addCallback(self.assertEquals, 302) + d.addCallback(self.assertEqual, 302) return d @defer.inlineCallbacks @@ -285,24 +285,24 @@ class HttpTestCase(unittest.TestCase): def test_host_header_not_in_request_headers(self): def _test(response): - self.assertEquals( + self.assertEqual( response.body, to_bytes('%s:%d' % (self.host, self.portno))) - self.assertEquals(request.headers, {}) + self.assertEqual(request.headers, {}) request = Request(self.getURL('host')) return self.download_request(request, Spider('foo')).addCallback(_test) def test_host_header_seted_in_request_headers(self): def _test(response): - self.assertEquals(response.body, b'example.com') - self.assertEquals(request.headers.get('Host'), b'example.com') + self.assertEqual(response.body, b'example.com') + self.assertEqual(request.headers.get('Host'), b'example.com') request = Request(self.getURL('host'), headers={'Host': 'example.com'}) return self.download_request(request, Spider('foo')).addCallback(_test) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b'example.com') + d.addCallback(self.assertEqual, b'example.com') return d def test_content_length_zero_bodyless_post_request_headers(self): @@ -317,7 +317,7 @@ class HttpTestCase(unittest.TestCase): https://bugs.python.org/issue14721 """ def _test(response): - self.assertEquals(response.body, b'0') + self.assertEqual(response.body, b'0') request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'}) return self.download_request(request, Spider('foo')).addCallback(_test) @@ -327,8 +327,8 @@ class HttpTestCase(unittest.TestCase): import json headers = Headers(json.loads(response.text)['headers']) contentlengths = headers.getlist('Content-Length') - self.assertEquals(len(contentlengths), 1) - self.assertEquals(contentlengths, [b"0"]) + self.assertEqual(len(contentlengths), 1) + self.assertEqual(contentlengths, [b"0"]) request = Request(self.getURL('echo'), method='POST') return self.download_request(request, Spider('foo')).addCallback(_test) @@ -338,7 +338,7 @@ class HttpTestCase(unittest.TestCase): request = Request(self.getURL('payload'), method='POST', body=body) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, body) + d.addCallback(self.assertEqual, body) return d @@ -364,7 +364,7 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") return d def test_response_class_choosing_request(self): @@ -374,7 +374,7 @@ class Http11TestCase(HttpTestCase): body = b'Some plain text\ndata with tabs\t and null bytes\0' def _test_type(response): - self.assertEquals(type(response), TextResponse) + self.assertEqual(type(response), TextResponse) request = Request(self.getURL('nocontenttype'), body=body) d = self.download_request(request, Spider('foo')) @@ -389,7 +389,7 @@ class Http11TestCase(HttpTestCase): # response body. (regardless of headers) d = self.download_request(request, Spider('foo', download_maxsize=10)) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") yield d d = self.download_request(request, Spider('foo', download_maxsize=9)) @@ -431,14 +431,14 @@ class Http11TestCase(HttpTestCase): request = Request(self.getURL('file')) d = self.download_request(request, Spider('foo', download_maxsize=100)) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") return d def test_download_chunked_content(self): request = Request(self.getURL('chunked')) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) - d.addCallback(self.assertEquals, b"chunked content\n") + d.addCallback(self.assertEqual, b"chunked content\n") return d def test_download_broken_content_cause_data_loss(self, url='broken'): @@ -597,9 +597,9 @@ class HttpProxyTestCase(unittest.TestCase): def test_download_with_proxy(self): def _test(response): - self.assertEquals(response.status, 200) - self.assertEquals(response.url, request.url) - self.assertEquals(response.body, b'http://example.com') + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'http://example.com') http_proxy = self.getURL('') request = Request('http://example.com', meta={'proxy': http_proxy}) @@ -607,9 +607,9 @@ class HttpProxyTestCase(unittest.TestCase): def test_download_with_proxy_https_noconnect(self): def _test(response): - self.assertEquals(response.status, 200) - self.assertEquals(response.url, request.url) - self.assertEquals(response.body, b'https://example.com') + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'https://example.com') http_proxy = '%s?noconnect' % self.getURL('') request = Request('https://example.com', meta={'proxy': http_proxy}) @@ -617,9 +617,9 @@ class HttpProxyTestCase(unittest.TestCase): def test_download_without_proxy(self): def _test(response): - self.assertEquals(response.status, 200) - self.assertEquals(response.url, request.url) - self.assertEquals(response.body, b'/path/to/resource') + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'/path/to/resource') request = Request(self.getURL('path/to/resource')) return self.download_request(request, Spider('foo')).addCallback(_test) @@ -978,7 +978,7 @@ class DataURITestCase(unittest.TestCase): uri = "data:,A%20brief%20note" def _test(response): - self.assertEquals(response.url, uri) + self.assertEqual(response.url, uri) self.assertFalse(response.headers) request = Request(uri) @@ -986,39 +986,39 @@ class DataURITestCase(unittest.TestCase): def test_default_mediatype_encoding(self): def _test(response): - self.assertEquals(response.text, 'A brief note') - self.assertEquals(type(response), + self.assertEqual(response.text, 'A brief note') + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) - self.assertEquals(response.encoding, "US-ASCII") + self.assertEqual(response.encoding, "US-ASCII") request = Request("data:,A%20brief%20note") return self.download_request(request, self.spider).addCallback(_test) def test_default_mediatype(self): def _test(response): - self.assertEquals(response.text, u'\u038e\u03a3\u038e') - self.assertEquals(type(response), + self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) - self.assertEquals(response.encoding, "iso-8859-7") + self.assertEqual(response.encoding, "iso-8859-7") request = Request("data:;charset=iso-8859-7,%be%d3%be") return self.download_request(request, self.spider).addCallback(_test) def test_text_charset(self): def _test(response): - self.assertEquals(response.text, u'\u038e\u03a3\u038e') - self.assertEquals(response.body, b'\xbe\xd3\xbe') - self.assertEquals(response.encoding, "iso-8859-7") + self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(response.body, b'\xbe\xd3\xbe') + self.assertEqual(response.encoding, "iso-8859-7") request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be") return self.download_request(request, self.spider).addCallback(_test) def test_mediatype_parameters(self): def _test(response): - self.assertEquals(response.text, u'\u038e\u03a3\u038e') - self.assertEquals(type(response), + self.assertEqual(response.text, u'\u038e\u03a3\u038e') + self.assertEqual(type(response), responsetypes.from_mimetype("text/plain")) - self.assertEquals(response.encoding, "utf-8") + self.assertEqual(response.encoding, "utf-8") request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;' 'charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22' @@ -1027,7 +1027,7 @@ class DataURITestCase(unittest.TestCase): def test_base64(self): def _test(response): - self.assertEquals(response.text, 'Hello, world.') + self.assertEqual(response.text, 'Hello, world.') request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D') return self.download_request(request, self.spider).addCallback(_test) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 26d9794b6..17801e502 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -36,7 +36,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request('http://scrapytest.org/sub1/') assert self.mw.process_request(req2, self.spider) is None - self.assertEquals(req2.headers.get('Cookie'), b"C1=value1") + self.assertEqual(req2.headers.get('Cookie'), b"C1=value1") def test_setting_false_cookies_enabled(self): self.assertRaises( @@ -131,12 +131,12 @@ class CookiesMiddlewareTest(TestCase): # check that cookies are merged back req = Request('http://scrapytest.org/mergeme') assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req.headers.get('Cookie'), b'C1=value1') # check that cookies are merged when dont_merge_cookies is passed as 0 req = Request('http://scrapytest.org/mergeme', meta={'dont_merge_cookies': 0}) assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req.headers.get('Cookie'), b'C1=value1') def test_complex_cookies(self): # merge some cookies into jar @@ -157,7 +157,7 @@ class CookiesMiddlewareTest(TestCase): # embed C2 for scrapytest.org/bar req = Request('http://scrapytest.org/bar') self.mw.process_request(req, self.spider) - self.assertEquals(req.headers.get('Cookie'), b'C2=value2') + self.assertEqual(req.headers.get('Cookie'), b'C2=value2') # embed nothing for scrapytest.org/baz req = Request('http://scrapytest.org/baz') @@ -167,7 +167,7 @@ class CookiesMiddlewareTest(TestCase): def test_merge_request_cookies(self): req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}) assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers.get('Cookie'), b'galleta=salada') + self.assertEqual(req.headers.get('Cookie'), b'galleta=salada') headers = {'Set-Cookie': 'C1=value1; path=/'} res = Response('http://scrapytest.org/', headers=headers) @@ -181,7 +181,7 @@ class CookiesMiddlewareTest(TestCase): def test_cookiejar_key(self): req = Request('http://scrapytest.org/', cookies={'galleta': 'salada'}, meta={'cookiejar': "store1"}) assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers.get('Cookie'), b'galleta=salada') + self.assertEqual(req.headers.get('Cookie'), b'galleta=salada') headers = {'Set-Cookie': 'C1=value1; path=/'} res = Response('http://scrapytest.org/', headers=headers, request=req) @@ -193,7 +193,7 @@ class CookiesMiddlewareTest(TestCase): req3 = Request('http://scrapytest.org/', cookies={'galleta': 'dulce'}, meta={'cookiejar': "store2"}) assert self.mw.process_request(req3, self.spider) is None - self.assertEquals(req3.headers.get('Cookie'), b'galleta=dulce') + self.assertEqual(req3.headers.get('Cookie'), b'galleta=dulce') headers = {'Set-Cookie': 'C2=value2; path=/'} res2 = Response('http://scrapytest.org/', headers=headers, request=req3) @@ -213,16 +213,16 @@ class CookiesMiddlewareTest(TestCase): req5_2 = Request('http://scrapytest.org:1104/some-redirected-path') assert self.mw.process_request(req5_2, self.spider) is None - self.assertEquals(req5_2.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req5_2.headers.get('Cookie'), b'C1=value1') req5_3 = Request('http://scrapytest.org/some-redirected-path') assert self.mw.process_request(req5_3, self.spider) is None - self.assertEquals(req5_3.headers.get('Cookie'), b'C1=value1') + self.assertEqual(req5_3.headers.get('Cookie'), b'C1=value1') #skip cookie retrieval for not http request req6 = Request('file:///scrapy/sometempfile') assert self.mw.process_request(req6, self.spider) is None - self.assertEquals(req6.headers.get('Cookie'), None) + self.assertEqual(req6.headers.get('Cookie'), None) def test_local_domain(self): request = Request("http://example-host/", cookies={'currencyCookie': 'USD'}) diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 80efa83f9..6a31dfcf8 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -22,15 +22,15 @@ class TestDefaultHeadersMiddleware(TestCase): defaults, spider, mw = self.get_defaults_spider_mw() req = Request('http://www.scrapytest.org') mw.process_request(req, spider) - self.assertEquals(req.headers, defaults) + self.assertEqual(req.headers, defaults) def test_update_headers(self): defaults, spider, mw = self.get_defaults_spider_mw() headers = {'Accept-Language': ['es'], 'Test-Header': ['test']} bytes_headers = {b'Accept-Language': [b'es'], b'Test-Header': [b'test']} req = Request('http://www.scrapytest.org', headers=headers) - self.assertEquals(req.headers, bytes_headers) + self.assertEqual(req.headers, bytes_headers) mw.process_request(req, spider) defaults.update(bytes_headers) - self.assertEquals(req.headers, defaults) + self.assertEqual(req.headers, defaults) diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 446a99f36..586bdc0d1 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -18,20 +18,20 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): req, spider, mw = self.get_request_spider_mw() mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta.get('download_timeout'), 180) + self.assertEqual(req.meta.get('download_timeout'), 180) def test_string_download_timeout(self): req, spider, mw = self.get_request_spider_mw({'DOWNLOAD_TIMEOUT': '20.1'}) mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta.get('download_timeout'), 20.1) + self.assertEqual(req.meta.get('download_timeout'), 20.1) def test_spider_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() spider.download_timeout = 2 mw.spider_opened(spider) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta.get('download_timeout'), 2) + self.assertEqual(req.meta.get('download_timeout'), 2) def test_request_has_download_timeout(self): req, spider, mw = self.get_request_spider_mw() @@ -39,4 +39,4 @@ class DownloadTimeoutMiddlewareTest(unittest.TestCase): mw.spider_opened(spider) req.meta['download_timeout'] = 1 assert mw.process_request(req, spider) is None - self.assertEquals(req.meta.get('download_timeout'), 1) + self.assertEqual(req.meta.get('download_timeout'), 1) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 425a5cc79..3381632b0 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -23,10 +23,10 @@ class HttpAuthMiddlewareTest(unittest.TestCase): def test_auth(self): req = Request('http://scrapytest.org/') assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') + self.assertEqual(req.headers['Authorization'], b'Basic Zm9vOmJhcg==') def test_auth_already_set(self): req = Request('http://scrapytest.org/', headers=dict(Authorization='Digest 123')) assert self.mw.process_request(req, self.spider) is None - self.assertEquals(req.headers['Authorization'], b'Digest 123') + self.assertEqual(req.headers['Authorization'], b'Digest 123') diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 0678fcb14..0745c8dd3 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -248,4 +248,4 @@ class HttpCompressionTest(TestCase): response = response.replace(body = None) newresponse = self.mw.process_response(request, response, self.spider) self.assertIs(newresponse, response) - self.assertEquals(response.body, b'') + self.assertEqual(response.body, b'') diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index c77179ceb..0ea83aaf9 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -35,8 +35,8 @@ class TestDefaultHeadersMiddleware(TestCase): for url in ('http://e.com', 'https://e.com', 'file:///tmp/a'): req = Request(url) assert mw.process_request(req, spider) is None - self.assertEquals(req.url, url) - self.assertEquals(req.meta, {}) + self.assertEqual(req.url, url) + self.assertEqual(req.meta, {}) def test_enviroment_proxies(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' @@ -48,41 +48,41 @@ class TestDefaultHeadersMiddleware(TestCase): ('https://e.com', https_proxy), ('file://tmp/a', None)]: req = Request(url) assert mw.process_request(req, spider) is None - self.assertEquals(req.url, url) - self.assertEquals(req.meta.get('proxy'), proxy) + self.assertEqual(req.url, url) + self.assertEqual(req.meta.get('proxy'), proxy) def test_proxy_precedence_meta(self): os.environ['http_proxy'] = 'https://proxy.com' mw = HttpProxyMiddleware() req = Request('http://scrapytest.org', meta={'proxy': 'https://new.proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://new.proxy:3128'}) + self.assertEqual(req.meta, {'proxy': 'https://new.proxy:3128'}) def test_proxy_auth(self): os.environ['http_proxy'] = 'https://user:pass@proxy:3128' mw = HttpProxyMiddleware() req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') def test_proxy_auth_empty_passwd(self): os.environ['http_proxy'] = 'https://user:@proxy:3128' mw = HttpProxyMiddleware() req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6') def test_proxy_auth_encoding(self): # utf-8 encoding @@ -90,27 +90,27 @@ class TestDefaultHeadersMiddleware(TestCase): mw = HttpProxyMiddleware(auth_encoding='utf-8') req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') # proxy from request.meta req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') # default latin-1 encoding mw = HttpProxyMiddleware(auth_encoding='latin-1') req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') # proxy from request.meta, latin-1 encoding req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') + self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') def test_proxy_already_seted(self): os.environ['http_proxy'] = 'https://proxy.for.http:3128' @@ -142,4 +142,4 @@ class TestDefaultHeadersMiddleware(TestCase): os.environ['no_proxy'] = '*' req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.meta, {'proxy': 'http://proxy.com'}) + self.assertEqual(req.meta, {'proxy': 'http://proxy.com'}) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 35e474418..74137b4cd 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -166,7 +166,7 @@ class RedirectMiddlewareTest(unittest.TestCase): resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o' - self.assertEquals(perc_encoded_utf8_url, req_result.url) + self.assertEqual(perc_encoded_utf8_url, req_result.url) def test_utf8_location(self): req = Request('http://scrapytest.org/first') @@ -174,7 +174,7 @@ class RedirectMiddlewareTest(unittest.TestCase): resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302) req_result = self.mw.process_response(req, resp, self.spider) perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o' - self.assertEquals(perc_encoded_utf8_url, req_result.url) + self.assertEqual(perc_encoded_utf8_url, req_result.url) class MetaRefreshMiddlewareTest(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py index 1e41fdace..a286764fd 100644 --- a/tests/test_downloadermiddleware_useragent.py +++ b/tests/test_downloadermiddleware_useragent.py @@ -17,7 +17,7 @@ class UserAgentMiddlewareTest(TestCase): spider, mw = self.get_spider_and_mw('default_useragent') req = Request('http://scrapytest.org/') assert mw.process_request(req, spider) is None - self.assertEquals(req.headers['User-Agent'], b'default_useragent') + self.assertEqual(req.headers['User-Agent'], b'default_useragent') def test_remove_agent(self): # settings UESR_AGENT to None should remove the user agent @@ -34,7 +34,7 @@ class UserAgentMiddlewareTest(TestCase): mw.spider_opened(spider) req = Request('http://scrapytest.org/') assert mw.process_request(req, spider) is None - self.assertEquals(req.headers['User-Agent'], b'spider_useragent') + self.assertEqual(req.headers['User-Agent'], b'spider_useragent') def test_header_agent(self): spider, mw = self.get_spider_and_mw('default_useragent') @@ -43,7 +43,7 @@ class UserAgentMiddlewareTest(TestCase): req = Request('http://scrapytest.org/', headers={'User-Agent': 'header_useragent'}) assert mw.process_request(req, spider) is None - self.assertEquals(req.headers['User-Agent'], b'header_useragent') + self.assertEqual(req.headers['User-Agent'], b'header_useragent') def test_no_agent(self): spider, mw = self.get_spider_and_mw(None) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 549f779d8..caa6fe83e 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -62,7 +62,7 @@ class WrappedResponseTest(TestCase): self.wrapped = WrappedResponse(self.response) def test_info(self): - self.assert_(self.wrapped.info() is self.wrapped) + self.assertTrue(self.wrapped.info() is self.wrapped) def test_getheaders(self): self.assertEqual(self.wrapped.getheaders('content-type'), ['text/html']) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 9b0ee63dc..21c0dd746 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -64,9 +64,9 @@ class RequestTest(unittest.TestCase): h = Headers({'key1': u'val1', u'key2': 'val2'}) h[u'newkey'] = u'newval' for k, v in h.iteritems(): - self.assert_(isinstance(k, bytes)) + self.assertTrue(isinstance(k, bytes)) for s in v: - self.assert_(isinstance(s, bytes)) + self.assertTrue(isinstance(s, bytes)) def test_eq(self): url = 'http://www.scrapy.org' diff --git a/tests/test_loader.py b/tests/test_loader.py index 9d07eb95b..2569ccf5e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -437,7 +437,7 @@ class ProcessorsTest(unittest.TestCase): self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) self.assertEqual(proc(['', 'hello', 'world']), u' hello world') self.assertEqual(proc(['hello', 'world']), u'hello world') - self.assert_(isinstance(proc(['hello', 'world']), six.text_type)) + self.assertTrue(isinstance(proc(['hello', 'world']), six.text_type)) def test_compose(self): proc = Compose(lambda v: v[0], str.upper) @@ -482,7 +482,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_constructor_with_selector(self): sel = Selector(text=u"
marta
") l = TestItemLoader(selector=sel) - self.assert_(l.selector is sel) + self.assertTrue(l.selector is sel) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) @@ -490,21 +490,21 @@ class SelectortemLoaderTest(unittest.TestCase): def test_constructor_with_selector_css(self): sel = Selector(text=u"
marta
") l = TestItemLoader(selector=sel) - self.assert_(l.selector is sel) + self.assertTrue(l.selector is sel) l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), [u'Marta']) def test_constructor_with_response(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) def test_constructor_with_response_css(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), [u'Marta']) @@ -526,7 +526,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_replace_xpath(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) l.replace_xpath('name', '//p/text()') @@ -552,7 +552,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_replace_xpath_re(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) l.replace_xpath('name', '//div/text()', re='ma') @@ -568,7 +568,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_replace_css(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), [u'Marta']) l.replace_css('name', 'p::text') @@ -606,7 +606,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_replace_css_re(self): l = TestItemLoader(response=self.response) - self.assert_(l.selector) + self.assertTrue(l.selector) l.add_css('url', 'a::attr(href)') self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) l.replace_css('url', 'a::attr(href)', re='http://www\.(.+)') diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 0f3047602..03c6d8059 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -81,28 +81,28 @@ class ImagesPipelineTestCase(unittest.TestCase): COLOUR = (0, 127, 255) im = _create_image('JPEG', 'RGB', SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im) - self.assertEquals(converted.mode, 'RGB') - self.assertEquals(converted.getcolors(), [(10000, COLOUR)]) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, COLOUR)]) # check that thumbnail keep image ratio thumbnail, _ = self.pipeline.convert_image(converted, size=(10, 25)) - self.assertEquals(thumbnail.mode, 'RGB') - self.assertEquals(thumbnail.size, (10, 10)) + self.assertEqual(thumbnail.mode, 'RGB') + self.assertEqual(thumbnail.size, (10, 10)) # transparency case: RGBA and PNG COLOUR = (0, 127, 255, 50) im = _create_image('PNG', 'RGBA', SIZE, COLOUR) converted, _ = self.pipeline.convert_image(im) - self.assertEquals(converted.mode, 'RGB') - self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))]) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) # transparency case with palette: P and PNG COLOUR = (0, 127, 255, 50) im = _create_image('PNG', 'RGBA', SIZE, COLOUR) im = im.convert('P') converted, _ = self.pipeline.convert_image(im) - self.assertEquals(converted.mode, 'RGB') - self.assertEquals(converted.getcolors(), [(10000, (205, 230, 255))]) + self.assertEqual(converted.mode, 'RGB') + self.assertEqual(converted.getcolors(), [(10000, (205, 230, 255))]) class DeprecatedImagesPipeline(ImagesPipeline): diff --git a/tests/test_selector.py b/tests/test_selector.py index af0cc4de2..526660cc8 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -84,7 +84,7 @@ class SelectorTestCase(unittest.TestCase): headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = Selector(response) - self.assertEquals(x.xpath("//span[@id='blank']/text()").extract(), + self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(), [u'\xa3']) def test_badly_encoded_body(self): diff --git a/tests/test_spider.py b/tests/test_spider.py index e55f0fa6d..0a343549e 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -207,7 +207,7 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEquals([r.url for r in output], + self.assertEqual([r.url for r in output], ['http://example.org/somepage/item/12.html', 'http://example.org/about.html', 'http://example.org/nofollow.html']) @@ -234,7 +234,7 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 2) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEquals([r.url for r in output], + self.assertEqual([r.url for r in output], ['http://example.org/somepage/item/12.html', 'http://example.org/about.html']) @@ -258,7 +258,7 @@ class CrawlSpiderTest(SpiderTest): output = list(spider._requests_to_follow(response)) self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEquals([r.url for r in output], + self.assertEqual([r.url for r in output], ['http://example.org/somepage/item/12.html', 'http://example.org/about.html', 'http://example.org/nofollow.html']) diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index a3cdc0114..3685d5a6f 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -25,18 +25,18 @@ class TestDepthMiddleware(TestCase): result = [Request('http://scrapytest.org')] out = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEquals(out, result) + self.assertEqual(out, result) rdc = self.stats.get_value('request_depth_count/1', spider=self.spider) - self.assertEquals(rdc, 1) + self.assertEqual(rdc, 1) req.meta['depth'] = 1 out2 = list(self.mw.process_spider_output(resp, result, self.spider)) - self.assertEquals(out2, []) + self.assertEqual(out2, []) rdm = self.stats.get_value('request_depth_max', spider=self.spider) - self.assertEquals(rdm, 1) + self.assertEqual(rdm, 1) def tearDown(self): self.stats.close_spider(self.spider, '') diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e1407e6b3..19e6bbdcd 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -67,16 +67,16 @@ class TestHttpErrorMiddleware(TestCase): self.res200, self.res404 = _responses(self.req, [200, 404]) def test_process_spider_input(self): - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res200, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) def test_process_spider_exception(self): - self.assertEquals([], + self.assertEqual([], self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider)) - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_exception(self.res404, Exception(), self.spider)) @@ -84,11 +84,11 @@ class TestHttpErrorMiddleware(TestCase): res = self.res404.copy() res.request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(res, self.spider)) self.spider.handle_httpstatus_list = [404] - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res404, self.spider)) @@ -102,11 +102,11 @@ class TestHttpErrorMiddlewareSettings(TestCase): self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res200, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): @@ -117,14 +117,14 @@ class TestHttpErrorMiddlewareSettings(TestCase): res402 = self.res402.copy() res402.request = request - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider) @@ -139,9 +139,9 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res200, self.spider)) - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(self.res404, self.spider)) def test_meta_overrides_settings(self): @@ -152,7 +152,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402 = self.res402.copy() res402.request = request - self.assertEquals(None, + self.assertEqual(None, self.mw.process_spider_input(res404, self.spider)) self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 37c3a450b..9ad86313c 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -37,7 +37,7 @@ class TestOffsiteMiddleware(TestCase): reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out, onsite_reqs) + self.assertEqual(out, onsite_reqs) class TestOffsiteMiddleware2(TestOffsiteMiddleware): @@ -49,7 +49,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): res = Response('http://scrapytest.org') reqs = [Request('http://a.com/b.html'), Request('http://b.com/1')] out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out, reqs) + self.assertEqual(out, reqs) class TestOffsiteMiddleware3(TestOffsiteMiddleware2): @@ -67,4 +67,4 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3): res = Response('http://scrapytest.org') reqs = [Request('http://scrapytest.org/1')] out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEquals(out, reqs) + self.assertEqual(out, reqs) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index f27f31b74..21439c20e 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -45,7 +45,7 @@ class TestRefererMiddleware(TestCase): response = self.get_response(origin) request = self.get_request(target) out = list(self.mw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), referrer) + self.assertEqual(out[0].headers.get('Referer'), referrer) class MixinDefault(object): @@ -490,7 +490,7 @@ class TestSettingsPolicyByName(TestCase): ]: settings = Settings({'REFERRER_POLICY': s}) mw = RefererMiddleware(settings) - self.assertEquals(mw.default_policy, p) + self.assertEqual(mw.default_policy, p) def test_valid_name_casevariants(self): for s, p in [ @@ -506,7 +506,7 @@ class TestSettingsPolicyByName(TestCase): ]: settings = Settings({'REFERRER_POLICY': s.upper()}) mw = RefererMiddleware(settings) - self.assertEquals(mw.default_policy, p) + self.assertEqual(mw.default_policy, p) def test_invalid_name(self): settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'}) @@ -581,7 +581,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): request = self.get_request(target) out = list(self.referrermw.process_spider_output(response, [request], self.spider)) - self.assertEquals(out[0].headers.get('Referer'), init_referrer) + self.assertEqual(out[0].headers.get('Referer'), init_referrer) for status, url in redirections: response = Response(request.url, headers={'Location': url}, status=status) @@ -589,7 +589,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): self.referrermw.request_scheduled(request, self.spider) assert isinstance(request, Request) - self.assertEquals(request.headers.get('Referer'), final_referrer) + self.assertEqual(request.headers.get('Referer'), final_referrer) class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index dca868ecf..a0aae0fdd 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -17,5 +17,5 @@ class TestUrlLengthMiddleware(TestCase): mw = UrlLengthMiddleware(maxlength=25) spider = Spider('foo') out = list(mw.process_spider_output(res, reqs, spider)) - self.assertEquals(out, [short_url_req]) + self.assertEqual(out, [short_url_req]) diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py index 052dde37f..22e39821c 100644 --- a/tests/test_urlparse_monkeypatches.py +++ b/tests/test_urlparse_monkeypatches.py @@ -6,7 +6,7 @@ class UrlparseTestCase(unittest.TestCase): def test_s3_url(self): p = urlparse('s3://bucket/key/name?param=value') - self.assertEquals(p.scheme, 's3') - self.assertEquals(p.hostname, 'bucket') - self.assertEquals(p.path, '/key/name') - self.assertEquals(p.query, 'param=value') + self.assertEqual(p.scheme, 's3') + self.assertEqual(p.hostname, 'bucket') + self.assertEqual(p.path, '/key/name') + self.assertEqual(p.query, 'param=value') diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 49323f0ff..5b83869b8 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -202,22 +202,22 @@ class SequenceExcludeTest(unittest.TestCase): seq = range(10, 20, 3) d = SequenceExclude(seq) are_not_in = [v for v in range(10, 20, 3) if v in d] - self.assertEquals([], are_not_in) + self.assertEqual([], are_not_in) are_not_in = [v for v in range(10, 20) if v in d] - self.assertEquals([11, 12, 14, 15, 17, 18], are_not_in) + self.assertEqual([11, 12, 14, 15, 17, 18], are_not_in) def test_string_seq(self): seq = "cde" d = SequenceExclude(seq) chars = "".join(v for v in "abcdefg" if v in d) - self.assertEquals("abfg", chars) + self.assertEqual("abfg", chars) def test_stringset_seq(self): seq = set("cde") d = SequenceExclude(seq) chars = "".join(v for v in "abcdefg" if v in d) - self.assertEquals("abfg", chars) + self.assertEqual("abfg", chars) def test_set(self): """Anything that is not in the supplied sequence will evaluate as 'in' the container.""" diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index f49bbfafe..003bb9b02 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -89,7 +89,7 @@ class IterErrbackTest(unittest.TestCase): errors = [] out = list(iter_errback(itergood(), errors.append)) self.assertEqual(out, list(range(10))) - self.failIf(errors) + self.assertFalse(errors) def test_iter_errback_bad(self): def iterbad(): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index b2e3889a4..b2e8610f8 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -252,8 +252,8 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assert_(all((isinstance(k, six.text_type) for k in result_row.keys()))) - self.assert_(all((isinstance(v, six.text_type) for v in result_row.values()))) + self.assertTrue(all((isinstance(k, six.text_type) for k in result_row.keys()))) + self.assertTrue(all((isinstance(v, six.text_type) for v in result_row.values()))) def test_csviter_delimiter(self): body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 01460a10b..832253aa4 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -23,20 +23,20 @@ class UtilsMiscTestCase(unittest.TestCase): 'tests.test_utils_misc.test_walk_modules.mod.mod0', 'tests.test_utils_misc.test_walk_modules.mod1', ] - self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual(set([m.__name__ for m in mods]), set(expected)) mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod') expected = [ 'tests.test_utils_misc.test_walk_modules.mod', 'tests.test_utils_misc.test_walk_modules.mod.mod0', ] - self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual(set([m.__name__ for m in mods]), set(expected)) mods = walk_modules('tests.test_utils_misc.test_walk_modules.mod1') expected = [ 'tests.test_utils_misc.test_walk_modules.mod1', ] - self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual(set([m.__name__ for m in mods]), set(expected)) self.assertRaises(ImportError, walk_modules, 'nomodule999') @@ -51,7 +51,7 @@ class UtilsMiscTestCase(unittest.TestCase): 'testegg.spiders.b', 'testegg' ] - self.assertEquals(set([m.__name__ for m in mods]), set(expected)) + self.assertEqual(set([m.__name__ for m in mods]), set(expected)) finally: sys.path.remove(egg) diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 6b7fcd4c2..7e2caace8 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -25,14 +25,14 @@ def inside_a_project(): class ProjectUtilsTest(unittest.TestCase): def test_data_path_outside_project(self): - self.assertEquals('.scrapy/somepath', data_path('somepath')) - self.assertEquals('/absolute/path', data_path('/absolute/path')) + self.assertEqual('.scrapy/somepath', data_path('somepath')) + self.assertEqual('/absolute/path', data_path('/absolute/path')) def test_data_path_inside_project(self): with inside_a_project() as proj_path: expected = os.path.join(proj_path, '.scrapy', 'somepath') - self.assertEquals( + self.assertEqual( os.path.realpath(expected), os.path.realpath(data_path('somepath')) ) - self.assertEquals('/absolute/path', data_path('/absolute/path')) + self.assertEqual('/absolute/path', data_path('/absolute/path')) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 8becca0f1..c2e4037e8 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -97,9 +97,9 @@ class UtilsPythonTestCase(unittest.TestCase): a = Obj() b = Obj() # no attributes given return False - self.failIf(equal_attributes(a, b, [])) + self.assertFalse(equal_attributes(a, b, [])) # not existent attributes - self.failIf(equal_attributes(a, b, ['x', 'y'])) + self.assertFalse(equal_attributes(a, b, ['x', 'y'])) a.x = 1 b.x = 1 @@ -108,7 +108,7 @@ class UtilsPythonTestCase(unittest.TestCase): b.y = 2 # obj1 has no attribute y - self.failIf(equal_attributes(a, b, ['x', 'y'])) + self.assertFalse(equal_attributes(a, b, ['x', 'y'])) a.y = 2 # equal attributes @@ -116,7 +116,7 @@ class UtilsPythonTestCase(unittest.TestCase): a.y = 1 # differente attributes - self.failIf(equal_attributes(a, b, ['x', 'y'])) + self.assertFalse(equal_attributes(a, b, ['x', 'y'])) # test callable a.meta = {} @@ -134,7 +134,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertTrue(equal_attributes(a, b, [compare_z, 'x'])) # fail z equality a.meta['z'] = 2 - self.failIf(equal_attributes(a, b, [compare_z, 'x'])) + self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) def test_weakkeycache(self): class _Weakme(object): pass @@ -156,9 +156,9 @@ class UtilsPythonTestCase(unittest.TestCase): d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'} d2 = stringify_dict(d, keys_only=False) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, six.text_type) for x in d2.keys())) - self.failIf(any(isinstance(x, six.text_type) for x in d2.values())) + self.assertFalse(d is d2) # shouldn't modify in place + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) @unittest.skipUnless(six.PY2, "deprecated function") def test_stringify_dict_tuples(self): @@ -166,17 +166,17 @@ class UtilsPythonTestCase(unittest.TestCase): d = dict(tuples) d2 = stringify_dict(tuples, keys_only=False) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys()) - self.failIf(any(isinstance(x, six.text_type) for x in d2.values())) + self.assertFalse(d is d2) # shouldn't modify in place + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys()) + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) @unittest.skipUnless(six.PY2, "deprecated function") def test_stringify_dict_keys_only(self): d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'} d2 = stringify_dict(d) self.assertEqual(d, d2) - self.failIf(d is d2) # shouldn't modify in place - self.failIf(any(isinstance(x, six.text_type) for x in d2.keys())) + self.assertFalse(d is d2) # shouldn't modify in place + self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) def test_get_func_args(self): def f1(a, b, c): diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index b7de85049..dea81adf6 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -29,7 +29,7 @@ class SendCatchLogTest(unittest.TestCase): self.assertIn('error_handler', record.getMessage()) self.assertEqual(record.levelname, 'ERROR') self.assertEqual(result[0][0], self.error_handler) - self.assert_(isinstance(result[0][1], Failure)) + self.assertTrue(isinstance(result[0][1], Failure)) self.assertEqual(result[1], (self.ok_handler, "OK")) dispatcher.disconnect(self.error_handler, signal=test_signal) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 3ad1aa70e..fedac2634 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -71,7 +71,7 @@ class ParseUrlTestCase(unittest.TestCase): for url, test in tests: test = tuple( to_bytes(x) if not isinstance(x, int) else x for x in test) - self.assertEquals(client._parse(url), test, url) + self.assertEqual(client._parse(url), test, url) def test_externalUnicodeInterference(self): """ @@ -258,16 +258,16 @@ class WebClientTestCase(unittest.TestCase): def testPayload(self): s = "0123456789" * 10 return getPage(self.getURL("payload"), body=s).addCallback( - self.assertEquals, to_bytes(s)) + self.assertEqual, to_bytes(s)) def testHostHeader(self): # if we pass Host header explicitly, it should be used, otherwise # it should extract from url return defer.gatherResults([ getPage(self.getURL("host")).addCallback( - self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno)), + self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)), getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback( - self.assertEquals, to_bytes("www.example.com"))]) + self.assertEqual, to_bytes("www.example.com"))]) def test_getPage(self): """ @@ -275,7 +275,7 @@ class WebClientTestCase(unittest.TestCase): the body of the response if the default method B{GET} is used. """ d = getPage(self.getURL("file")) - d.addCallback(self.assertEquals, b"0123456789") + d.addCallback(self.assertEqual, b"0123456789") return d def test_getPageHead(self): @@ -298,7 +298,7 @@ class WebClientTestCase(unittest.TestCase): """ d = getPage(self.getURL("host"), timeout=100) d.addCallback( - self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno)) + self.assertEqual, to_bytes("127.0.0.1:%d" % self.portno)) return d def test_timeoutTriggering(self): @@ -326,7 +326,7 @@ class WebClientTestCase(unittest.TestCase): return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile) def _cbNoSuchFile(self, pageData): - self.assert_(b'404 - No Such Resource' in pageData) + self.assertTrue(b'404 - No Such Resource' in pageData) def testFactoryInfo(self): url = self.getURL('file') @@ -336,16 +336,16 @@ class WebClientTestCase(unittest.TestCase): return factory.deferred.addCallback(self._cbFactoryInfo, factory) def _cbFactoryInfo(self, ignoredResult, factory): - self.assertEquals(factory.status, b'200') - self.assert_(factory.version.startswith(b'HTTP/')) - self.assertEquals(factory.message, b'OK') - self.assertEquals(factory.response_headers[b'content-length'], b'10') + self.assertEqual(factory.status, b'200') + self.assertTrue(factory.version.startswith(b'HTTP/')) + self.assertEqual(factory.message, b'OK') + self.assertEqual(factory.response_headers[b'content-length'], b'10') def testRedirect(self): return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) def _cbRedirect(self, pageData): - self.assertEquals(pageData, + self.assertEqual(pageData, b'\n\n \n \n' b' \n \n ' b'click here\n \n\n') @@ -360,6 +360,6 @@ class WebClientTestCase(unittest.TestCase): def _check_Encoding(self, response, original_body): content_encoding = to_unicode(response.headers[b'Content-Encoding']) - self.assertEquals(content_encoding, EncodingResource.out_encoding) - self.assertEquals( + self.assertEqual(content_encoding, EncodingResource.out_encoding) + self.assertEqual( response.body.decode(content_encoding), to_unicode(original_body)) From fd27cde24d273e30f72f53f2711515403270838a Mon Sep 17 00:00:00 2001 From: Chomba Ng'ang'a Date: Tue, 8 Aug 2017 19:08:53 +0300 Subject: [PATCH 241/362] Update asserts to use more generic ones --- tests/test_http_cookies.py | 2 +- tests/test_http_request.py | 4 ++-- tests/test_loader.py | 6 +++--- tests/test_utils_python.py | 6 +++--- tests/test_utils_signal.py | 2 +- tests/test_webclient.py | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index caa6fe83e..0a9ed500a 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -62,7 +62,7 @@ class WrappedResponseTest(TestCase): self.wrapped = WrappedResponse(self.response) def test_info(self): - self.assertTrue(self.wrapped.info() is self.wrapped) + self.assertIs(self.wrapped.info(), self.wrapped) def test_getheaders(self): self.assertEqual(self.wrapped.getheaders('content-type'), ['text/html']) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 21c0dd746..fca8ff411 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -64,9 +64,9 @@ class RequestTest(unittest.TestCase): h = Headers({'key1': u'val1', u'key2': 'val2'}) h[u'newkey'] = u'newval' for k, v in h.iteritems(): - self.assertTrue(isinstance(k, bytes)) + self.assertIsInstance(k, bytes) for s in v: - self.assertTrue(isinstance(s, bytes)) + self.assertIsInstance(s, bytes) def test_eq(self): url = 'http://www.scrapy.org' diff --git a/tests/test_loader.py b/tests/test_loader.py index 2569ccf5e..3b5714058 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -437,7 +437,7 @@ class ProcessorsTest(unittest.TestCase): self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) self.assertEqual(proc(['', 'hello', 'world']), u' hello world') self.assertEqual(proc(['hello', 'world']), u'hello world') - self.assertTrue(isinstance(proc(['hello', 'world']), six.text_type)) + self.assertIsInstance(proc(['hello', 'world']), six.text_type) def test_compose(self): proc = Compose(lambda v: v[0], str.upper) @@ -482,7 +482,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_constructor_with_selector(self): sel = Selector(text=u"
marta
") l = TestItemLoader(selector=sel) - self.assertTrue(l.selector is sel) + self.assertIs(l.selector, sel) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) @@ -490,7 +490,7 @@ class SelectortemLoaderTest(unittest.TestCase): def test_constructor_with_selector_css(self): sel = Selector(text=u"
marta
") l = TestItemLoader(selector=sel) - self.assertTrue(l.selector is sel) + self.assertIs(l.selector, sel) l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), [u'Marta']) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index c2e4037e8..115f523e9 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -156,7 +156,7 @@ class UtilsPythonTestCase(unittest.TestCase): d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'} d2 = stringify_dict(d, keys_only=False) self.assertEqual(d, d2) - self.assertFalse(d is d2) # shouldn't modify in place + self.assertIsNot(d, d2) # shouldn't modify in place self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) @@ -166,7 +166,7 @@ class UtilsPythonTestCase(unittest.TestCase): d = dict(tuples) d2 = stringify_dict(tuples, keys_only=False) self.assertEqual(d, d2) - self.assertFalse(d is d2) # shouldn't modify in place + self.assertIsNot(d, d2) # shouldn't modify in place self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys()) self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) @@ -175,7 +175,7 @@ class UtilsPythonTestCase(unittest.TestCase): d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'} d2 = stringify_dict(d) self.assertEqual(d, d2) - self.assertFalse(d is d2) # shouldn't modify in place + self.assertIsNot(d, d2) # shouldn't modify in place self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) def test_get_func_args(self): diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index dea81adf6..62edd420d 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -29,7 +29,7 @@ class SendCatchLogTest(unittest.TestCase): self.assertIn('error_handler', record.getMessage()) self.assertEqual(record.levelname, 'ERROR') self.assertEqual(result[0][0], self.error_handler) - self.assertTrue(isinstance(result[0][1], Failure)) + self.assertIsInstance(result[0][1], Failure) self.assertEqual(result[1], (self.ok_handler, "OK")) dispatcher.disconnect(self.error_handler, signal=test_signal) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index fedac2634..766329b57 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -326,7 +326,7 @@ class WebClientTestCase(unittest.TestCase): return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile) def _cbNoSuchFile(self, pageData): - self.assertTrue(b'404 - No Such Resource' in pageData) + self.assertIn(b'404 - No Such Resource', pageData) def testFactoryInfo(self): url = self.getURL('file') From 1dcea6a9d4615afc463fa9839d002d861aed5274 Mon Sep 17 00:00:00 2001 From: kim minji Date: Wed, 16 Aug 2017 18:07:52 +0900 Subject: [PATCH 242/362] fix typo --- scrapy/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 7b8518832..1367536ab 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -40,7 +40,7 @@ class Crawler(object): handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL')) logging.root.addHandler(handler) if get_scrapy_root_handler() is not None: - # scrapy root handler alread installed: update it with new settings + # scrapy root handler already installed: update it with new settings install_scrapy_root_handler(self.settings) # lambda is assigned to Crawler attribute because this way it is not # garbage collected after leaving __init__ scope From 1968a8ec02913273a99a3137cb419e2649d69a5f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 23 Aug 2017 15:08:10 +0200 Subject: [PATCH 243/362] Move logging of overriden settings to Crawler init --- scrapy/crawler.py | 5 ++++- scrapy/utils/log.py | 4 +--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1367536ab..0a56ef57a 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -11,7 +11,7 @@ from scrapy.core.engine import ExecutionEngine from scrapy.resolver import CachingThreadedResolver from scrapy.interfaces import ISpiderLoader from scrapy.extension import ExtensionManager -from scrapy.settings import Settings +from scrapy.settings import overridden_settings, Settings from scrapy.signalmanager import SignalManager from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.ossignal import install_shutdown_handlers, signal_names @@ -34,6 +34,9 @@ class Crawler(object): self.settings = settings.copy() self.spidercls.update_settings(self.settings) + d = dict(overridden_settings(self.settings)) + logger.info("Overridden settings: %(settings)r", {'settings': d}) + self.signals = SignalManager(self) self.stats = load_object(self.settings['STATS_CLASS'])(self) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 905c1bfc1..c6d1cdf46 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -9,7 +9,7 @@ from twisted.python.failure import Failure from twisted.python import log as twisted_log import scrapy -from scrapy.settings import overridden_settings, Settings +from scrapy.settings import Settings from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.versions import scrapy_components_versions @@ -148,8 +148,6 @@ def log_scrapy_info(settings): {'versions': ", ".join("%s %s" % (name, version) for name, version in scrapy_components_versions() if name != "Scrapy")}) - d = dict(overridden_settings(settings)) - logger.info("Overridden settings: %(settings)r", {'settings': d}) class StreamLogger(object): From 7a35a1ad4ad38b2f413528b0184d8709dea2a495 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 23 Aug 2017 17:08:21 +0200 Subject: [PATCH 244/362] Remove trailing bracket from components versions log --- scrapy/utils/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 905c1bfc1..61b978c55 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -144,7 +144,7 @@ def _get_handler(settings): def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) - logger.info("Versions: %(versions)s}", + logger.info("Versions: %(versions)s", {'versions': ", ".join("%s %s" % (name, version) for name, version in scrapy_components_versions() if name != "Scrapy")}) From a429d78019a379fff29c7aa3fff0a0f0427b6995 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 24 Aug 2017 16:03:36 -0300 Subject: [PATCH 245/362] update scrapinghub.com urls to use https --- docs/intro/install.rst | 2 +- docs/topics/deploy.rst | 6 +++--- docs/topics/logging.rst | 4 ++-- docs/topics/practices.rst | 2 +- docs/topics/ubuntu.rst | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 47af8292e..deb0118d4 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -235,7 +235,7 @@ After any of these workarounds you should be able to install Scrapy:: .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ .. _homebrew: http://brew.sh/ .. _zsh: http://www.zsh.org/ -.. _Scrapinghub: http://scrapinghub.com +.. _Scrapinghub: https://scrapinghub.com .. _Anaconda: http://docs.continuum.io/anaconda/index .. _Miniconda: http://conda.pydata.org/docs/install/quick.html .. _conda-forge: https://conda-forge.github.io/ diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index f4186ea7a..f2e11fe8f 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -51,9 +51,9 @@ just like ``scrapyd-deploy``. .. _Scrapyd: https://github.com/scrapy/scrapyd .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud: http://scrapinghub.com/scrapy-cloud/ +.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud .. _scrapyd-client: https://github.com/scrapy/scrapyd-client -.. _shub: http://doc.scrapinghub.com/shub.html +.. _shub: https://doc.scrapinghub.com/shub.html .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html .. _Scrapy Cloud documentation: http://doc.scrapinghub.com/scrapy-cloud.html -.. _Scrapinghub: http://scrapinghub.com/ +.. _Scrapinghub: https://scrapinghub.com/ diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index a3281dd6b..0986929ad 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -102,7 +102,7 @@ instance, which can be accessed and used like this:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['http://scrapinghub.com'] + start_urls = ['https://scrapinghub.com'] def parse(self, response): self.logger.info('Parse function called on %s', response.url) @@ -118,7 +118,7 @@ Python logger you want. For example:: class MySpider(scrapy.Spider): name = 'myspider' - start_urls = ['http://scrapinghub.com'] + start_urls = ['https://scrapinghub.com'] def parse(self, response): logger.info('Parse function called on %s', response.url) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 63913d3c4..21aa4a0a7 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -253,5 +253,5 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders .. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html -.. _Crawlera: http://scrapinghub.com/crawlera +.. _Crawlera: https://scrapinghub.com/crawlera .. _scrapoxy: http://scrapoxy.io/ diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst index 679bb56ff..81ce800aa 100644 --- a/docs/topics/ubuntu.rst +++ b/docs/topics/ubuntu.rst @@ -37,5 +37,5 @@ To use the packages: .. warning:: `python-scrapy` is a different package provided by official debian repositories, it's very outdated and it isn't supported by Scrapy team. -.. _Scrapinghub: http://scrapinghub.com/ +.. _Scrapinghub: https://scrapinghub.com/ .. _GitHub repo: https://github.com/scrapy/scrapy From 9f16f040b661f18f2e61a35427199c099bfd2f90 Mon Sep 17 00:00:00 2001 From: cclauss Date: Fri, 1 Sep 2017 11:53:59 +0200 Subject: [PATCH 246/362] ur'string' not needed in Py 2, syntax error in Py3 Convert `ur'Scrapy Documentation'`--> `u'Scrapy Documentation'`to be compatible with both Python 2 and Python 3. See #2891 --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 640dcd7cb..5780db65d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -191,7 +191,7 @@ htmlhelp_basename = 'Scrapydoc' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). latex_documents = [ - ('index', 'Scrapy.tex', ur'Scrapy Documentation', + ('index', 'Scrapy.tex', u'Scrapy Documentation', ur'Scrapy developers', 'manual'), ] From b7022360824cd1a8aa19fb0d65ea01700f06a208 Mon Sep 17 00:00:00 2001 From: cclauss Date: Fri, 1 Sep 2017 11:56:09 +0200 Subject: [PATCH 247/362] ur'string' not needed in Py 2, syntax error in Py3 Convert `u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))'`--> `u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))'`to be compatible with both Python 2 and Python 3. See #2891 --- docs/utils/linkfix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 40316968f..6290adbe2 100755 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -20,7 +20,7 @@ _filename = None _contents = None # A regex that matches standard linkcheck output lines -line_re = re.compile(ur'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') +line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') # Read lines from the linkcheck output file try: From b8fabeed8652d22725959345700b9e7d00073de4 Mon Sep 17 00:00:00 2001 From: cclauss Date: Fri, 1 Sep 2017 13:55:05 +0200 Subject: [PATCH 248/362] ur'string' not needed in Py 2, syntax error in Py3 This instance was missed in #2909 --> ur'Scrapy developers' --> u'Scrapy developers' --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 5780db65d..007dc2788 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -192,7 +192,7 @@ htmlhelp_basename = 'Scrapydoc' # (source start file, target name, title, author, document class [howto/manual]). latex_documents = [ ('index', 'Scrapy.tex', u'Scrapy Documentation', - ur'Scrapy developers', 'manual'), + u'Scrapy developers', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of From abaf466bb311f6416a58763ad7974825d88f4855 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 7 Sep 2017 11:37:40 +0200 Subject: [PATCH 249/362] Print cryptography package version --- scrapy/commands/version.py | 7 +++++-- scrapy/utils/versions.py | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 71b1026fa..577365c3b 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -23,8 +23,11 @@ class Command(ScrapyCommand): def run(self, args, opts): if opts.verbose: - for name, version in scrapy_components_versions(): - print("%-9s : %s" % (name, version)) + versions = scrapy_components_versions() + width = max(len(n) for (n, _) in versions) + patt = "%-{}s : %s".format(width) + for name, version in versions: + print(patt % (name, version)) else: print("Scrapy %s" % scrapy.__version__) diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index d2cff09fe..58c7aef85 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -17,6 +17,11 @@ def scrapy_components_versions(): w3lib_version = w3lib.__version__ except AttributeError: w3lib_version = "<1.14.3" + try: + import cryptography + cryptography_version = cryptography.__version__ + except ImportError: + cryptography_version = "unknown" return [ ("Scrapy", scrapy.__version__), @@ -28,6 +33,7 @@ def scrapy_components_versions(): ("Twisted", twisted.version.short()), ("Python", sys.version.replace("\n", "- ")), ("pyOpenSSL", _get_openssl_version()), + ("cryptography", cryptography_version), ("Platform", platform.platform()), ] From aab98080a06281ca3a88646990b81b92f492c517 Mon Sep 17 00:00:00 2001 From: Iulian Onofrei Date: Mon, 11 Sep 2017 00:40:55 +0300 Subject: [PATCH 250/362] Add option to disable automatic log handler install --- scrapy/crawler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 0a56ef57a..a33ce9805 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -237,15 +237,18 @@ class CrawlerProcess(CrawlerRunner): The CrawlerProcess object must be instantiated with a :class:`~scrapy.settings.Settings` object. + :param install_root_handler: whether to install root logging handler + (default: True) + This class shouldn't be needed (since Scrapy is responsible of using it accordingly) unless writing scripts that manually handle the crawling process. See :ref:`run-from-script` for an example. """ - def __init__(self, settings=None): + def __init__(self, settings=None, install_root_handler=True): super(CrawlerProcess, self).__init__(settings) install_shutdown_handlers(self._signal_shutdown) - configure_logging(self.settings) + configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) def _signal_shutdown(self, signum, _): From 3637b75a6702cb3fb4962477c0f2ec38e366f3e2 Mon Sep 17 00:00:00 2001 From: Steven Almeroth Date: Tue, 12 Sep 2017 15:54:09 -0400 Subject: [PATCH 251/362] [Doc] Update Response.body type --- docs/topics/request-response.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 6ca37b7c9..92aae1ad0 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -525,11 +525,11 @@ Response objects (for single valued headers) or lists (for multi-valued headers). :type headers: dict - :param body: the response body. It must be str, not unicode, unless you're - using a encoding-aware :ref:`Response subclass - `, such as - :class:`TextResponse`. - :type body: str + :param body: the response body. To access the decoded text as str (unicode + in Python 2) you can use ``response.text`` from an encoding-aware + :ref:`Response subclass `, + such as :class:`TextResponse`. + :type body: bytes :param flags: is a list containing the initial values for the :attr:`Response.flags` attribute. If given, the list will be shallow From d71a0634039d637dc10509ebb63fa2f4ef595ebb Mon Sep 17 00:00:00 2001 From: rhoboro Date: Tue, 12 Sep 2017 18:30:15 +0900 Subject: [PATCH 252/362] Support for Google Cloud Storage --- scrapy/pipelines/files.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index eae03752a..304d89bcf 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -194,6 +194,41 @@ class S3FilesStore(object): return extra +class GCSFilesStore(object): + + GCS_PROJECT_ID = None + + CACHE_CONTROL = 'max-age=172800' + + def __init__(self, uri): + from google.cloud import storage + client = storage.Client(project=self.GCS_PROJECT_ID) + bucket, prefix = uri[5:].split('/', 1) + self.bucket = client.bucket(bucket) + self.prefix = prefix + + def stat_file(self, path, info): + def _onsuccess(blob): + if blob: + checksum = blob.md5_hash + last_modified = time.mktime(blob.updated.timetuple()) + return {'checksum': checksum, 'last_modified': last_modified} + else: + return {} + + return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess) + + def persist_file(self, path, buf, info, meta=None, headers=None): + blob = self.bucket.blob(self.prefix + path) + blob.cache_control = self.CACHE_CONTROL + blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})} + return threads.deferToThread( + blob.upload_from_string, + data=buf.getvalue(), + content_type='application/octet-stream' + ) + + class FilesPipeline(MediaPipeline): """Abstract pipeline that implement the file downloading @@ -219,6 +254,7 @@ class FilesPipeline(MediaPipeline): '': FSFilesStore, 'file': FSFilesStore, 's3': S3FilesStore, + 'gs': GCSFilesStore, } DEFAULT_FILES_URLS_FIELD = 'file_urls' DEFAULT_FILES_RESULT_FIELD = 'files' @@ -258,6 +294,9 @@ class FilesPipeline(MediaPipeline): s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] s3store.POLICY = settings['FILES_STORE_S3_ACL'] + gcs_store = cls.STORE_SCHEMES['gs'] + gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + store_uri = settings['FILES_STORE'] return cls(store_uri, settings=settings) From e5d4364b2a0e7ae205605905ac0c5ac6fd8d15db Mon Sep 17 00:00:00 2001 From: rhoboro Date: Wed, 13 Sep 2017 16:24:04 +0900 Subject: [PATCH 253/362] Add tests for GCS Storage --- scrapy/utils/test.py | 15 +++++++++++++++ tests/test_pipeline_files.py | 28 +++++++++++++++++++++++++++- tox.ini | 3 +++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index d2ef68912..60b931f48 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -20,6 +20,12 @@ def assert_aws_environ(): if 'AWS_ACCESS_KEY_ID' not in os.environ: raise SkipTest("AWS keys not found") + +def assert_gcs_environ(): + if 'GCS_PROJECT_ID' not in os.environ: + raise SkipTest("GCS_PROJECT_ID not found") + + def skip_if_no_boto(): try: is_botocore() @@ -45,6 +51,15 @@ def get_s3_content_and_delete(bucket, path, with_key=False): bucket.delete_key(path) return (content, key) if with_key else content +def get_gcs_content_and_delete(bucket, path): + from google.cloud import storage + client = storage.Client(project=os.environ.get('GCS_PROJECT_ID')) + bucket = client.get_bucket(bucket) + blob = bucket.get_blob(path) + content = blob.download_as_string() + bucket.delete_blob(path) + return content, blob + def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e3ec04b8d..c761bd606 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -11,12 +11,13 @@ from six import BytesIO from twisted.trial import unittest from twisted.internet import defer -from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore +from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GCSFilesStore from scrapy.item import Item, Field from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.utils.python import to_bytes from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete +from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete from scrapy.utils.boto import is_botocore from tests import mock @@ -375,6 +376,31 @@ class TestS3FilesStore(unittest.TestCase): self.assertEqual(key.content_type, 'image/png') +class TestGCSFilesStore(unittest.TestCase): + @defer.inlineCallbacks + def test_persist(self): + assert_gcs_environ() + uri = os.environ.get('GCS_TEST_FILE_URI') + if not uri: + raise unittest.SkipTest("No GCS URI available for testing") + data = b"TestGCSFilesStore: \xe2\x98\x83" + buf = BytesIO(data) + meta = {'foo': 'bar'} + path = 'full/filename' + store = GCSFilesStore(uri) + yield store.persist_file(path, buf, info=None, meta=meta, headers=None) + s = yield store.stat_file(path, info=None) + self.assertIn('last_modified', s) + self.assertIn('checksum', s) + self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==') + u = urlparse(uri) + content, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) + self.assertEqual(content, data) + self.assertEqual(blob.metadata, {'foo': 'bar'}) + self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) + self.assertEqual(blob.content_type, 'application/octet-stream') + + class ItemWithFiles(Item): file_urls = Field() files = Field() diff --git a/tox.ini b/tox.ini index c7e1e43c9..0608693ba 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,7 @@ deps = -rrequirements.txt # Extras botocore + google-cloud-storage Pillow != 3.0.0 leveldb -rtests/requirements.txt @@ -18,6 +19,8 @@ passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + GCS_TEST_FILE_URI + GCS_PROJECT_ID commands = py.test --cov=scrapy --cov-report= {posargs:scrapy tests} From ee166ec44f38da7f5b99c6c164a7c6ff02b37c16 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Wed, 13 Sep 2017 17:35:46 +0900 Subject: [PATCH 254/362] Support for ImagesPipeline --- scrapy/pipelines/files.py | 8 +++++++- scrapy/pipelines/images.py | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 304d89bcf..7fdb8a086 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -218,6 +218,12 @@ class GCSFilesStore(object): return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess) + def _get_content_type(self, headers): + if headers and 'Content-Type' in headers: + return headers['Content-Type'] + else: + return 'application/octet-stream' + def persist_file(self, path, buf, info, meta=None, headers=None): blob = self.bucket.blob(self.prefix + path) blob.cache_control = self.CACHE_CONTROL @@ -225,7 +231,7 @@ class GCSFilesStore(object): return threads.deferToThread( blob.upload_from_string, data=buf.getvalue(), - content_type='application/octet-stream' + content_type=self._get_content_type(headers) ) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index bc449431f..c5fc12afe 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -91,6 +91,9 @@ class ImagesPipeline(FilesPipeline): s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] s3store.POLICY = settings['IMAGES_STORE_S3_ACL'] + gcs_store = cls.STORE_SCHEMES['gs'] + gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) From 088b80d41a12a7b79e440cc4d4a3aae678d5c4af Mon Sep 17 00:00:00 2001 From: Renze Yu Date: Wed, 13 Sep 2017 23:29:22 +0800 Subject: [PATCH 255/362] minor fix typo --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 3b3bd8d21..29f54bc86 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -452,7 +452,7 @@ For historic reasons, Scrapy appends to a given file instead of overwriting its contents. If you run this command twice without removing the file before the second time, you'll end up with a broken JSON file. -You can also used other formats, like `JSON Lines`_:: +You can also use other formats, like `JSON Lines`_:: scrapy crawl quotes -o quotes.jl From dcb279bd6cc85cf1743b548e44b050edba6a2ed8 Mon Sep 17 00:00:00 2001 From: djunzu Date: Sun, 17 Sep 2017 16:09:22 -0300 Subject: [PATCH 256/362] Add m4v extension to IGNORED_EXTENSIONS in LinkExtractor. modified: scrapy/linkextractors/__init__.py --- scrapy/linkextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 8676c3b92..2d7115cc5 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -28,7 +28,7 @@ IGNORED_EXTENSIONS = [ # video '3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv', - 'm4a', + 'm4a', 'm4v', # office suites 'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg', From 84111969c4250b486b15729b42f88300fc983511 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Wed, 20 Sep 2017 13:35:48 +0200 Subject: [PATCH 257/362] Update pypy version regexp to get last release PyPy changed naming conention since 5.8 release, not it's called pypy2.7-x.x.x --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4f44d1e6d..9c51fafb2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ install: rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT" fi # get latest portable PyPy from pyenv directly (thanks to natural version sort option -V) - export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-portable-[0-9][\.0-9]*$' |sort -V |tail -1` + export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy2.7-portable-[0-9][\.0-9]*$' |sort -V |tail -1` "$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION" virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" From 00c81a32ee58d0a59c14373471bf152c46131aec Mon Sep 17 00:00:00 2001 From: Craig Rodrigues Date: Sat, 23 Sep 2017 11:01:34 -0700 Subject: [PATCH 258/362] Bump Twisted requirement to 17.9.0 to catch many Python 3 fixes. --- requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index cc0a7f644..2aae3ae65 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted >= 15.5.0 +Twisted >= 17.9.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 From e733f51d4b04f209bfec32d1bd7559a258f45d0c Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 25 Sep 2017 12:49:27 +0200 Subject: [PATCH 259/362] Fix test --- tests/test_command_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 2789d207c..4ac7fb786 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -28,4 +28,4 @@ class VersionTest(ProcessTest, unittest.TestCase): self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', 'cssselect', 'parsel', 'w3lib', 'Twisted', 'Python', 'pyOpenSSL', - 'Platform']) + 'cryptography', 'Platform']) From d4555b2bcc387292e5fd5bd8321c946e2e374fb7 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Fri, 29 Sep 2017 12:07:29 +0900 Subject: [PATCH 260/362] update docs for supporting google cloud storage --- docs/topics/media-pipeline.rst | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index e948913a4..9580a15d9 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -15,7 +15,8 @@ typically you'll either use the Files Pipeline or the Images Pipeline. Both pipelines implement these features: * Avoid re-downloading media that was downloaded recently -* Specifying where to store the media (filesystem directory, Amazon S3 bucket) +* Specifying where to store the media (filesystem directory, Amazon S3 bucket, + Google Cloud Storage bucket) The Images Pipeline has a few extra functions for processing images: @@ -116,10 +117,11 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting:: Supported Storage ================= -File system is currently the only officially supported storage, but there is -also support for storing files in `Amazon S3`_. +File system is currently the only officially supported storage, but there are +also support for storing files in `Amazon S3`_ and `Google Cloud Storage`_. .. _Amazon S3: https://aws.amazon.com/s3/ +.. _Google Cloud Storage: https://cloud.google.com/storage/ File system storage ------------------- @@ -171,6 +173,25 @@ For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. .. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +Google Cloud Storage +--------------------- + +.. setting:: GCS_PROJECT_ID + +:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage +bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ ) + +.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python + +For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings:: + + IMAGES_STORE = 'gs://bucket/images/' + GCS_PROJECT_ID = 'project_id' + +For information about authentication, see this `documentation`_. + +.. _documentation: https://cloud.google.com/docs/authentication/production + Usage example ============= From 59c3f6f095d7605a825a90f376d41493a78e5da7 Mon Sep 17 00:00:00 2001 From: Lucas Moauro Date: Sun, 1 Oct 2017 12:24:56 -0300 Subject: [PATCH 261/362] Fix typos in tests --- tests/test_downloadermiddleware_httpproxy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 0ea83aaf9..17be875c1 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -28,7 +28,7 @@ class TestDefaultHeadersMiddleware(TestCase): crawler = Crawler(spider, settings) self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler)) - def test_no_enviroment_proxies(self): + def test_no_environment_proxies(self): os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} mw = HttpProxyMiddleware() @@ -38,7 +38,7 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEqual(req.url, url) self.assertEqual(req.meta, {}) - def test_enviroment_proxies(self): + def test_environment_proxies(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' os.environ['https_proxy'] = https_proxy = 'http://proxy.for.https:8080' os.environ.pop('file_proxy', None) From fc406801f1783392935fcd7faf603d8339c74675 Mon Sep 17 00:00:00 2001 From: Craig Rodrigues Date: Tue, 21 Mar 2017 00:21:41 -0700 Subject: [PATCH 262/362] ESMTPSenderFactory takes a message of bytes --- scrapy/mail.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 0bb395521..7365f25b7 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -5,10 +5,10 @@ See documentation in docs/topics/email.rst """ import logging -from six.moves import cStringIO as StringIO import six from email.utils import COMMASPACE, formatdate +from io import BytesIO from six.moves.email_mime_multipart import MIMEMultipart from six.moves.email_mime_text import MIMEText from six.moves.email_mime_base import MIMEBase @@ -21,6 +21,14 @@ else: from twisted.internet import defer, reactor, ssl +try: + from twisted.mail.smtp import ESMTPSenderFactory +except ImportError: + """ + twisted.mail.smtp was not available in + older versions of Twisted on Python 3 + """ + from .utils.misc import arg_to_iter logger = logging.getLogger(__name__) @@ -110,9 +118,7 @@ class MailSender(object): 'mailattachs': nattachs, 'mailerr': errstr}) def _sendmail(self, to_addrs, msg): - # Import twisted.mail here because it is not available in python3 - from twisted.mail.smtp import ESMTPSenderFactory - msg = StringIO(msg) + msg = BytesIO(msg.encode('utf-8')) d = defer.Deferred() factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \ to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \ From 12c7628fcbcfb3927f595e1fd6806ea6aefdf6fd Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 8 Aug 2017 17:13:45 +0200 Subject: [PATCH 263/362] Encode message using supplied charset --- scrapy/mail.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 7365f25b7..7f237820f 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -5,10 +5,13 @@ See documentation in docs/topics/email.rst """ import logging +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO import six from email.utils import COMMASPACE, formatdate -from io import BytesIO from six.moves.email_mime_multipart import MIMEMultipart from six.moves.email_mime_text import MIMEText from six.moves.email_mime_base import MIMEBase @@ -21,14 +24,6 @@ else: from twisted.internet import defer, reactor, ssl -try: - from twisted.mail.smtp import ESMTPSenderFactory -except ImportError: - """ - twisted.mail.smtp was not available in - older versions of Twisted on Python 3 - """ - from .utils.misc import arg_to_iter logger = logging.getLogger(__name__) @@ -96,7 +91,7 @@ class MailSender(object): 'mailattachs': len(attachs)}) return - dfd = self._sendmail(rcpts, msg.as_string()) + dfd = self._sendmail(rcpts, msg.as_string().encode(charset or 'utf-8')) dfd.addCallbacks(self._sent_ok, self._sent_failed, callbackArgs=[to, cc, subject, len(attachs)], errbackArgs=[to, cc, subject, len(attachs)]) @@ -118,7 +113,9 @@ class MailSender(object): 'mailattachs': nattachs, 'mailerr': errstr}) def _sendmail(self, to_addrs, msg): - msg = BytesIO(msg.encode('utf-8')) + # Import twisted.mail here because it is not available in python3 + from twisted.mail.smtp import ESMTPSenderFactory + msg = BytesIO(msg) d = defer.Deferred() factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \ to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \ From 80bb4fcf9710598138d7604190636d723d6392df Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 8 Aug 2017 17:40:36 +0200 Subject: [PATCH 264/362] Convert SMTP credentials to bytes if needed --- scrapy/mail.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 7f237820f..6d809dc99 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -24,7 +24,8 @@ else: from twisted.internet import defer, reactor, ssl -from .utils.misc import arg_to_iter +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.python import to_bytes logger = logging.getLogger(__name__) @@ -35,8 +36,8 @@ class MailSender(object): smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): self.smtphost = smtphost self.smtpport = smtpport - self.smtpuser = smtpuser - self.smtppass = smtppass + self.smtpuser = to_bytes(smtpuser) + self.smtppass = to_bytes(smtppass) self.smtptls = smtptls self.smtpssl = smtpssl self.mailfrom = mailfrom From 0d8a33fddccd7083c0f72fe75f89e9e7fd52cd82 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 8 Aug 2017 17:42:56 +0200 Subject: [PATCH 265/362] Update docs --- docs/topics/email.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/email.rst b/docs/topics/email.rst index aac93a91a..949cdc638 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -54,10 +54,10 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. :param smtpuser: the SMTP user. If omitted, the :setting:`MAIL_USER` setting will be used. If not given, no SMTP authentication will be performed. - :type smtphost: str + :type smtphost: str or bytes :param smtppass: the SMTP pass for authentication. - :type smtppass: str + :type smtppass: str or bytes :param smtpport: the SMTP port to connect to :type smtpport: int From 9cd348d94af782008f1a561a0b36da9231878833 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 22 Aug 2017 12:29:47 +0200 Subject: [PATCH 266/362] Handle None values for smtp user and password --- scrapy/mail.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 6d809dc99..0cfb4ec79 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -36,8 +36,8 @@ class MailSender(object): smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): self.smtphost = smtphost self.smtpport = smtpport - self.smtpuser = to_bytes(smtpuser) - self.smtppass = to_bytes(smtppass) + self.smtpuser = to_bytes(smtpuser) if smtpuser is not None else None + self.smtppass = to_bytes(smtppass) if smtppass is not None else None self.smtptls = smtptls self.smtpssl = smtpssl self.mailfrom = mailfrom From f729d74886be2290fcbbcaa21d366b770ff21008 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 28 Aug 2017 11:21:11 +0200 Subject: [PATCH 267/362] Use a helper for to_bytes() and None input --- scrapy/mail.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 0cfb4ec79..5b944e1c4 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -30,14 +30,20 @@ from scrapy.utils.python import to_bytes logger = logging.getLogger(__name__) +def _to_bytes_or_none(text): + if text is None: + return None + return to_bytes(text) + + class MailSender(object): def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): self.smtphost = smtphost self.smtpport = smtpport - self.smtpuser = to_bytes(smtpuser) if smtpuser is not None else None - self.smtppass = to_bytes(smtppass) if smtppass is not None else None + self.smtpuser = _to_bytes_or_none(smtpuser) + self.smtppass = _to_bytes_or_none(smtppass) self.smtptls = smtptls self.smtpssl = smtpssl self.mailfrom = mailfrom From e914556adf8e556d4184db415c49266cc4c91bf5 Mon Sep 17 00:00:00 2001 From: NoExitTV Date: Thu, 5 Oct 2017 15:12:01 +0200 Subject: [PATCH 268/362] Changed the log message to make it more clear. As requested in issue #2927 --- scrapy/core/downloader/handlers/http11.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9bfdd803c..23343d92a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -412,9 +412,10 @@ class _ResponseReader(protocol.Protocol): if self._maxsize and self._bytes_received > self._maxsize: logger.error("Received (%(bytes)s) bytes larger than download " - "max size (%(maxsize)s).", + "max size (%(maxsize)s) in request %(request)s.", {'bytes': self._bytes_received, - 'maxsize': self._maxsize}) + 'maxsize': self._maxsize, + 'request': self._request}) # Clear buffer earlier to avoid keeping data in memory for a long # time. self._bodybuf.truncate(0) From 938bc18405ca2cf60836bb8490c391c0fe445af1 Mon Sep 17 00:00:00 2001 From: NoExitTV Date: Thu, 5 Oct 2017 15:31:00 +0200 Subject: [PATCH 269/362] Changed the log message to make it more clear. As requested in issue #2927 --- scrapy/core/downloader/handlers/http11.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 23343d92a..0a5538947 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -344,8 +344,8 @@ class ScrapyAgent(object): if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " - "download warn size (%(warnsize)s).", - {'size': expected_size, 'warnsize': warnsize}) + "download warn size (%(warnsize)s) in request (%(request)s).", + {'size': expected_size, 'warnsize': warnsize, 'request': request}) def _cancel(_): # Abort connection inmediately. @@ -412,10 +412,9 @@ class _ResponseReader(protocol.Protocol): if self._maxsize and self._bytes_received > self._maxsize: logger.error("Received (%(bytes)s) bytes larger than download " - "max size (%(maxsize)s) in request %(request)s.", + "max size (%(maxsize)s).", {'bytes': self._bytes_received, - 'maxsize': self._maxsize, - 'request': self._request}) + 'maxsize': self._maxsize}) # Clear buffer earlier to avoid keeping data in memory for a long # time. self._bodybuf.truncate(0) From 345d948f2f55ad81ec7de7cb1f5619d80971a6c0 Mon Sep 17 00:00:00 2001 From: NoExitTV Date: Thu, 5 Oct 2017 15:37:05 +0200 Subject: [PATCH 270/362] Changed the log message to make it more clear. As requested in issue #2927 --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 0a5538947..4e1bb0cd5 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -344,7 +344,7 @@ class ScrapyAgent(object): if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " - "download warn size (%(warnsize)s) in request (%(request)s).", + "download warn size (%(warnsize)s) in request %(request)s.", {'size': expected_size, 'warnsize': warnsize, 'request': request}) def _cancel(_): From 9b8503011e1da0507e82e15631194ed99b7e699a Mon Sep 17 00:00:00 2001 From: NoExitTV Date: Fri, 6 Oct 2017 13:45:35 +0200 Subject: [PATCH 271/362] Changed log message to include information about request as user djunzu commented --- scrapy/core/downloader/handlers/http11.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 4e1bb0cd5..48d2481b4 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -412,9 +412,10 @@ class _ResponseReader(protocol.Protocol): if self._maxsize and self._bytes_received > self._maxsize: logger.error("Received (%(bytes)s) bytes larger than download " - "max size (%(maxsize)s).", + "max size (%(maxsize)s) in request %(request)s.", {'bytes': self._bytes_received, - 'maxsize': self._maxsize}) + 'maxsize': self._maxsize, + 'request': self._request}) # Clear buffer earlier to avoid keeping data in memory for a long # time. self._bodybuf.truncate(0) From 9cdf34b7c791359b1f86678f758cf64368723c53 Mon Sep 17 00:00:00 2001 From: Aditya Date: Tue, 10 Oct 2017 22:49:22 +0530 Subject: [PATCH 272/362] Link "Debugging in Python" article to its new location Reference: https://web.archive.org/web/20170203104051/http://www.ferg.org/papers/debugging_in_python.html --- docs/topics/extensions.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 7a67cf295..c421a5e05 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -280,7 +280,7 @@ An integer which specifies a number of items. If the spider scrapes more than that amount and those items are passed by the item pipeline, the spider will be closed with the reason ``closespider_itemcount``. Requests which are currently in the downloader queue (up to -:setting:`CONCURRENT_REQUESTS` requests) are still processed. +:setting:`CONCURRENT_REQUESTS` requests) are still processed. If zero (or non set), spiders won't be closed by number of passed items. .. setting:: CLOSESPIDER_PAGECOUNT @@ -373,4 +373,4 @@ For more info see `Debugging in Python`. This extension only works on POSIX-compliant platforms (ie. not Windows). .. _Python debugger: https://docs.python.org/2/library/pdb.html -.. _Debugging in Python: http://www.ferg.org/papers/debugging_in_python.html +.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/ From 8288f78a39939dbffea467bf110c64e005847117 Mon Sep 17 00:00:00 2001 From: djunzu Date: Mon, 16 Oct 2017 21:34:37 -0200 Subject: [PATCH 273/362] Add note about request.meta['depth'] in DepthMiddleware --- docs/topics/spider-middleware.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 9a0ccd0c1..a2d2556c5 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -188,9 +188,13 @@ DepthMiddleware .. class:: DepthMiddleware - DepthMiddleware is a scrape middleware used for tracking the depth of each - Request inside the site being scraped. It can be used to limit the maximum - depth to scrape or things like that. + DepthMiddleware is used for tracking the depth of each Request inside the + site being scraped. It works by setting `request.meta['depth'] = 0` whenever + there is no value previously set (usually just the first Request) and + incrementing it by 1 otherwise. + + It can be used to limit the maximum depth to scrape, control Request + priority based on their depth, and things like that. The :class:`DepthMiddleware` can be configured through the following settings (see the settings documentation for more info): From 169dc2860e9f7054c50c84f5adcd8a0d5afe161e Mon Sep 17 00:00:00 2001 From: Weldon Malbrough Date: Mon, 16 Oct 2017 22:46:32 -0400 Subject: [PATCH 274/362] Update tutorial.rst startproject files Added middlewares.py to accurately reflect the file structure created by "scrapy startproject tutorial" --- docs/intro/tutorial.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 29f54bc86..a02c759bb 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -54,6 +54,8 @@ This will create a ``tutorial`` directory with the following contents:: __init__.py items.py # project items definition file + + middlewares.py # project middlewares file pipelines.py # project pipelines file From 95815d27e89a6eea4676358697e2846959e4725b Mon Sep 17 00:00:00 2001 From: Weldon Malbrough Date: Wed, 25 Oct 2017 23:16:30 -0400 Subject: [PATCH 275/362] updated file structure to include middlewares.py --- docs/topics/commands.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index dc8067d7e..b15349598 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -55,6 +55,7 @@ structure by default, similar to this:: myproject/ __init__.py items.py + middlewares.py pipelines.py settings.py spiders/ From 9dd680d5c94340ac308f1450d9d3dc226a015326 Mon Sep 17 00:00:00 2001 From: Aditya Date: Thu, 26 Oct 2017 23:32:20 +0530 Subject: [PATCH 276/362] Use https for external links wherever possible in docs --- README.rst | 4 ++-- artwork/README.rst | 4 ++-- docs/contributing.rst | 2 +- docs/faq.rst | 2 +- docs/intro/install.rst | 12 ++++++------ docs/intro/overview.rst | 2 +- docs/intro/tutorial.rst | 2 +- docs/topics/debug.rst | 2 +- docs/topics/deploy.rst | 2 +- docs/topics/firebug.rst | 2 +- docs/topics/firefox.rst | 8 ++++---- docs/topics/jobs.rst | 2 +- docs/topics/loaders.rst | 4 ++-- docs/topics/media-pipeline.rst | 2 +- docs/topics/practices.rst | 4 ++-- docs/topics/request-response.rst | 2 +- docs/topics/selectors.rst | 2 +- docs/topics/settings.rst | 4 ++-- docs/topics/shell.rst | 6 +++--- docs/topics/spiders.rst | 4 ++-- extras/coverage-report.sh | 2 +- 21 files changed, 37 insertions(+), 37 deletions(-) diff --git a/README.rst b/README.rst index 27fab8e29..da63f2b93 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,7 @@ Scrapy :alt: PyPI Version .. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg - :target: http://travis-ci.org/scrapy/scrapy + :target: https://travis-ci.org/scrapy/scrapy :alt: Build Status .. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg @@ -15,7 +15,7 @@ Scrapy :alt: Wheel Status .. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg - :target: http://codecov.io/github/scrapy/scrapy?branch=master + :target: https://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report .. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg diff --git a/artwork/README.rst b/artwork/README.rst index 016462f2c..92f6ecb7e 100644 --- a/artwork/README.rst +++ b/artwork/README.rst @@ -10,10 +10,10 @@ scrapy-logo.jpg Main Scrapy logo, in JPEG format. -qlassik.zip +qlassik.zip ----------- -Font used for Scrapy logo. Homepage: http://www.dafont.com/qlassik.font +Font used for Scrapy logo. Homepage: https://www.dafont.com/qlassik.font scrapy-blog.logo.xcf -------------------- diff --git a/docs/contributing.rst b/docs/contributing.rst index c969bd842..291a1054e 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -188,7 +188,7 @@ And their unit-tests are in:: .. _issue tracker: https://github.com/scrapy/scrapy/issues .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users -.. _Scrapy subreddit: http://reddit.com/r/scrapy +.. _Scrapy subreddit: https://reddit.com/r/scrapy .. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests diff --git a/docs/faq.rst b/docs/faq.rst index f0ee20b5e..42c3abbfa 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -21,7 +21,7 @@ Python code. In other words, comparing `BeautifulSoup`_ (or `lxml`_) to Scrapy is like comparing `jinja2`_ to `Django`_. -.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ +.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ .. _lxml: http://lxml.de/ .. _jinja2: http://jinja.pocoo.org/ .. _Django: https://www.djangoproject.com/ diff --git a/docs/intro/install.rst b/docs/intro/install.rst index deb0118d4..12d489612 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -188,7 +188,7 @@ solutions: that doesn't conflict with the rest of your system. Here's how to do it using the `homebrew`_ package manager: - * Install `homebrew`_ following the instructions in http://brew.sh/ + * Install `homebrew`_ following the instructions in https://brew.sh/ * Update your ``PATH`` variable to state that homebrew packages should be used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly @@ -233,9 +233,9 @@ After any of these workarounds you should be able to install Scrapy:: .. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL .. _setuptools: https://pypi.python.org/pypi/setuptools .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ -.. _homebrew: http://brew.sh/ -.. _zsh: http://www.zsh.org/ +.. _homebrew: https://brew.sh/ +.. _zsh: https://www.zsh.org/ .. _Scrapinghub: https://scrapinghub.com -.. _Anaconda: http://docs.continuum.io/anaconda/index -.. _Miniconda: http://conda.pydata.org/docs/install/quick.html -.. _conda-forge: https://conda-forge.github.io/ +.. _Anaconda: https://docs.anaconda.com/anaconda/ +.. _Miniconda: https://conda.io/docs/user-guide/install/index.html +.. _conda-forge: https://conda-forge.org/ diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 1da1a4059..d0ce07a8e 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -164,4 +164,4 @@ interest! .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping .. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html .. _Amazon S3: https://aws.amazon.com/s3/ -.. _Sitemaps: http://www.sitemaps.org +.. _Sitemaps: https://www.sitemaps.org/index.html diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index a02c759bb..20538e90f 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -34,7 +34,7 @@ list of Python resources for non-programmers`_. .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers .. _Dive Into Python 3: http://www.diveintopython3.net .. _Python Tutorial: https://docs.python.org/3/tutorial -.. _Learn Python The Hard Way: http://learnpythonthehardway.org/book/ +.. _Learn Python The Hard Way: https://learnpythonthehardway.org/book/ Creating a project diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index a3e72097c..d1991c02f 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -142,4 +142,4 @@ available in all future runs should they be necessary again:: For more information, check the :ref:`topics-logging` section. -.. _base tag: http://www.w3schools.com/tags/tag_base.asp +.. _base tag: https://www.w3schools.com/tags/tag_base.asp diff --git a/docs/topics/deploy.rst b/docs/topics/deploy.rst index f2e11fe8f..361914a29 100644 --- a/docs/topics/deploy.rst +++ b/docs/topics/deploy.rst @@ -55,5 +55,5 @@ just like ``scrapyd-deploy``. .. _scrapyd-client: https://github.com/scrapy/scrapyd-client .. _shub: https://doc.scrapinghub.com/shub.html .. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html -.. _Scrapy Cloud documentation: http://doc.scrapinghub.com/scrapy-cloud.html +.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html .. _Scrapinghub: https://scrapinghub.com/ diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 8f0a5767b..4ea8d3bd0 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -23,7 +23,7 @@ In this example, we'll show how to use `Firebug`_ to scrape data from the Project`_ used in the :ref:`tutorial ` but with a different face. -.. _Firebug: http://getfirebug.com +.. _Firebug: https://getfirebug.com/ .. _Google Directory: http://directory.google.com/ .. _Open Directory Project: http://www.dmoz.org diff --git a/docs/topics/firefox.rst b/docs/topics/firefox.rst index 0cf45861a..2c85848be 100644 --- a/docs/topics/firefox.rst +++ b/docs/topics/firefox.rst @@ -17,7 +17,7 @@ when inspecting the page source is not the original HTML, but a modified one after applying some browser clean up and executing Javascript code. Firefox, in particular, is known for adding ```` elements to tables. Scrapy, on the other hand, does not modify the original page HTML, so you won't be able to -extract any data if you use ```` in your XPath expressions. +extract any data if you use ```` in your XPath expressions. Therefore, you should keep in mind the following things when working with Firefox and XPath: @@ -71,11 +71,11 @@ Firecookie `Firecookie`_ makes it easier to view and manage cookies. You can use this extension to create a new cookie, delete existing cookies, see a list of cookies -for the current site, manage cookies permissions and a lot more. +for the current site, manage cookies permissions and a lot more. -.. _Firebug: http://getfirebug.com +.. _Firebug: https://getfirebug.com/ .. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA -.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/ +.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/ .. _XPath Checker: https://addons.mozilla.org/en-US/firefox/addon/xpath-checker/ .. _Tamper Data: https://addons.mozilla.org/en-US/firefox/addon/tamper-data/ .. _Firecookie: https://addons.mozilla.org/en-US/firefox/addon/firecookie/ diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 4f9e38086..06c7fff3d 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -100,4 +100,4 @@ If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. It is ``False`` by default. -.. _pickle: http://docs.python.org/library/pickle.html +.. _pickle: https://docs.python.org/library/pickle.html diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index ad86dba63..0849090b4 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -518,8 +518,8 @@ a footer of a page that looks something like: Example:: diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 9580a15d9..4c634ace5 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -171,7 +171,7 @@ policy:: For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide. -.. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl +.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl Google Cloud Storage --------------------- diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 21aa4a0a7..e0dd4000f 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -249,9 +249,9 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _Tor project: https://www.torproject.org/ .. _commercial support: http://scrapy.org/support/ -.. _ProxyMesh: http://proxymesh.com/ +.. _ProxyMesh: https://proxymesh.com/ .. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders .. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html .. _Crawlera: https://scrapinghub.com/crawlera -.. _scrapoxy: http://scrapoxy.io/ +.. _scrapoxy: https://scrapoxy.io/ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 92aae1ad0..121abe6b5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -734,7 +734,7 @@ HtmlResponse objects which adds encoding auto-discovering support by looking into the HTML `meta http-equiv`_ attribute. See :attr:`TextResponse.encoding`. -.. _meta http-equiv: http://www.w3schools.com/TAGS/att_meta_http_equiv.asp +.. _meta http-equiv: https://www.w3schools.com/TAGS/att_meta_http_equiv.asp XmlResponse objects ------------------- diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 9bae53f45..cb4c25391 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -36,7 +36,7 @@ documents. For a complete reference of the selectors API see :ref:`Selector reference ` -.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/ +.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ .. _lxml: http://lxml.de/ .. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html .. _cssselect: https://pypi.python.org/pypi/cssselect/ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 37e3828a4..4b15cb607 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1002,7 +1002,7 @@ The randomization policy is the same used by `wget`_ ``--random-wait`` option. If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect. -.. _wget: http://www.gnu.org/software/wget/manual/wget.html +.. _wget: https://www.gnu.org/software/wget/manual/wget.html .. setting:: REACTOR_THREADPOOL_MAXSIZE @@ -1317,7 +1317,7 @@ Default: ``2083`` Scope: ``spidermiddlewares.urllength`` The maximum URL length to allow for crawled URLs. For more information about -the default value for this setting see: http://www.boutell.com/newfaq/misc/urllength.html +the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html .. setting:: USER_AGENT diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index ef6aeeed3..527116418 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -39,9 +39,9 @@ variable; or by defining it in your :ref:`scrapy.cfg `:: [settings] shell = bpython -.. _IPython: http://ipython.org/ -.. _IPython installation guide: http://ipython.org/install.html -.. _bpython: http://www.bpython-interpreter.org/ +.. _IPython: https://ipython.org/ +.. _IPython installation guide: https://ipython.org/install.html +.. _bpython: https://www.bpython-interpreter.org/ Launch the shell ================ diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index bf1532d1b..c2c271245 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -751,8 +751,8 @@ Combine SitemapSpider with other sources of urls:: def parse_other(self, response): pass # ... scrape other here ... -.. _Sitemaps: http://www.sitemaps.org -.. _Sitemap index files: http://www.sitemaps.org/protocol.html#index +.. _Sitemaps: https://www.sitemaps.org/index.html +.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index .. _robots.txt: http://www.robotstxt.org/ .. _TLD: https://en.wikipedia.org/wiki/Top-level_domain .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/ diff --git a/extras/coverage-report.sh b/extras/coverage-report.sh index dc20e16e4..842d0e46e 100755 --- a/extras/coverage-report.sh +++ b/extras/coverage-report.sh @@ -1,6 +1,6 @@ # Run tests, generate coverage report and open it on a browser # -# Requires: coverage 3.3 or above from http://pypi.python.org/pypi/coverage +# Requires: coverage 3.3 or above from https://pypi.python.org/pypi/coverage coverage run --branch $(which trial) --reporter=text tests coverage html -i From 9d9d83a8c31b6a18d7aaac35a30ffb69db4bb81d Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 28 Oct 2017 16:24:40 +0530 Subject: [PATCH 277/362] Use https links wherever possible --- CONTRIBUTING.md | 4 ++-- INSTALL | 2 +- README.rst | 14 ++++++------- debian/control | 6 +++--- debian/copyright | 8 ++++---- docs/contributing.rst | 2 +- docs/intro/overview.rst | 2 +- docs/topics/practices.rst | 2 +- docs/topics/selectors.rst | 4 ++-- docs/topics/shell.rst | 8 ++++---- scrapy/_monkeypatches.py | 4 ++-- scrapy/core/downloader/contextfactory.py | 4 ++-- scrapy/crawler.py | 2 +- scrapy/downloadermiddlewares/chunked.py | 2 +- scrapy/downloadermiddlewares/httpcache.py | 2 +- scrapy/exporters.py | 2 +- scrapy/extensions/httpcache.py | 10 +++++----- scrapy/extensions/telnet.py | 2 +- scrapy/pipelines/files.py | 4 ++-- scrapy/signalmanager.py | 2 +- scrapy/templates/project/module/items.py.tmpl | 2 +- .../project/module/middlewares.py.tmpl | 2 +- .../project/module/pipelines.py.tmpl | 2 +- .../templates/project/module/settings.py.tmpl | 20 +++++++++---------- scrapy/utils/defer.py | 2 +- scrapy/utils/deprecate.py | 12 +++++------ scrapy/utils/http.py | 2 +- scrapy/utils/log.py | 2 +- scrapy/utils/url.py | 2 +- sep/sep-001.rst | 2 +- sep/sep-006.rst | 4 ++-- sep/sep-013.rst | 2 +- sep/sep-017.rst | 2 +- sep/sep-020.rst | 2 +- setup.py | 2 +- tests/__init__.py | 2 +- tests/keys/example-com.conf | 4 ++-- 37 files changed, 76 insertions(+), 76 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 88c472f6f..0a11b05d2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ The guidelines for contributing are available here: -http://doc.scrapy.org/en/master/contributing.html +https://doc.scrapy.org/en/master/contributing.html Please do not abuse the issue tracker for support questions. If your issue topic can be rephrased to "How to ...?", please use the -support channels to get it answered: http://scrapy.org/community/ +support channels to get it answered: https://scrapy.org/community/ diff --git a/INSTALL b/INSTALL index 84803a933..a3c7899c6 100644 --- a/INSTALL +++ b/INSTALL @@ -1,4 +1,4 @@ For information about installing Scrapy see: * docs/intro/install.rst (local file) -* http://doc.scrapy.org/en/latest/intro/install.html (online version) +* https://doc.scrapy.org/en/latest/intro/install.html (online version) diff --git a/README.rst b/README.rst index da63f2b93..45135c7a2 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,7 @@ crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. For more information including a list of features check the Scrapy homepage at: -http://scrapy.org +https://scrapy.org Requirements ============ @@ -47,12 +47,12 @@ The quick way:: pip install scrapy For more details see the install section in the documentation: -http://doc.scrapy.org/en/latest/intro/install.html +https://doc.scrapy.org/en/latest/intro/install.html Documentation ============= -Documentation is available online at http://doc.scrapy.org/ and in the ``docs`` +Documentation is available online at https://doc.scrapy.org/ and in the ``docs`` directory. Releases @@ -63,12 +63,12 @@ You can find release notes at https://doc.scrapy.org/en/latest/news.html Community (blog, twitter, mail list, IRC) ========================================= -See http://scrapy.org/community/ +See https://scrapy.org/community/ Contributing ============ -See http://doc.scrapy.org/en/master/contributing.html +See https://doc.scrapy.org/en/master/contributing.html Code of Conduct --------------- @@ -82,9 +82,9 @@ Please report unacceptable behavior to opensource@scrapinghub.com. Companies using Scrapy ====================== -See http://scrapy.org/companies/ +See https://scrapy.org/companies/ Commercial Support ================== -See http://scrapy.org/support/ +See https://scrapy.org/support/ diff --git a/debian/control b/debian/control index f3a31753b..2cc8eedf4 100644 --- a/debian/control +++ b/debian/control @@ -4,7 +4,7 @@ Priority: optional Maintainer: Scrapinghub Team Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2) Standards-Version: 3.8.4 -Homepage: http://scrapy.org/ +Homepage: https://scrapy.org/ Package: scrapy Architecture: all @@ -15,6 +15,6 @@ Conflicts: python-scrapy, scrapy-0.25 Provides: python-scrapy, scrapy-0.25 Description: Python web crawling and web scraping framework Scrapy is a fast high-level web crawling and web scraping framework, - used to crawl websites and extract structured data from their pages. - It can be used for a wide range of purposes, from data mining to + used to crawl websites and extract structured data from their pages. + It can be used for a wide range of purposes, from data mining to monitoring and automated testing. diff --git a/debian/copyright b/debian/copyright index 4cc239002..c1bf47565 100644 --- a/debian/copyright +++ b/debian/copyright @@ -1,6 +1,6 @@ This package was debianized by the Scrapinghub team . -It was downloaded from http://scrapy.org +It was downloaded from https://scrapy.org Upstream Author: Scrapy Developers @@ -14,10 +14,10 @@ All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - 1. Redistributions of source code must retain the above copyright notice, + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright + + 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. diff --git a/docs/contributing.rst b/docs/contributing.rst index 291a1054e..f3732ab06 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -7,7 +7,7 @@ Contributing to Scrapy .. important:: Double check you are reading the most recent version of this document at - http://doc.scrapy.org/en/master/contributing.html + https://doc.scrapy.org/en/master/contributing.html There are many ways to contribute to Scrapy. Here are some of them: diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index d0ce07a8e..6f1c2c43f 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -160,7 +160,7 @@ The next steps for you are to :ref:`install Scrapy `, a full-blown Scrapy project and `join the community`_. Thanks for your interest! -.. _join the community: http://scrapy.org/community/ +.. _join the community: https://scrapy.org/community/ .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping .. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html .. _Amazon S3: https://aws.amazon.com/s3/ diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index e0dd4000f..02cfa9b05 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -248,7 +248,7 @@ If you are still unable to prevent your bot getting banned, consider contacting `commercial support`_. .. _Tor project: https://www.torproject.org/ -.. _commercial support: http://scrapy.org/support/ +.. _commercial support: https://scrapy.org/support/ .. _ProxyMesh: https://proxymesh.com/ .. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index cb4c25391..8ac40c3cc 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -86,7 +86,7 @@ To explain how to use the selectors we'll use the `Scrapy shell` (which provides interactive testing) and an example page located in the Scrapy documentation server: - http://doc.scrapy.org/en/latest/_static/selectors-sample1.html + https://doc.scrapy.org/en/latest/_static/selectors-sample1.html .. _topics-selectors-htmlcode: @@ -99,7 +99,7 @@ Here's its HTML code: First, let's open the shell:: - scrapy shell http://doc.scrapy.org/en/latest/_static/selectors-sample1.html + scrapy shell https://doc.scrapy.org/en/latest/_static/selectors-sample1.html Then, after the shell loads, you'll have the response available as ``response`` shell variable, and its attached selector in ``response.selector`` attribute. diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 527116418..11ab199f2 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -142,7 +142,7 @@ Example of shell session ======================== Here's an example of a typical shell session where we start by scraping the -http://scrapy.org page, and then proceed to scrape the https://reddit.com +https://scrapy.org page, and then proceed to scrape the https://reddit.com page. Finally, we modify the (Reddit) request method to POST and re-fetch it getting an error. We end the session by typing Ctrl-D (in Unix systems) or Ctrl-Z in Windows. @@ -154,7 +154,7 @@ shell works. First, we launch the shell:: - scrapy shell 'http://scrapy.org' --nolog + scrapy shell 'https://scrapy.org' --nolog Then, the shell fetches the URL (using the Scrapy downloader) and prints the list of available objects and useful shortcuts (you'll notice that these lines @@ -164,7 +164,7 @@ all start with the ``[s]`` prefix):: [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) [s] crawler [s] item {} - [s] request + [s] request [s] response <200 https://scrapy.org/> [s] settings [s] spider @@ -182,7 +182,7 @@ After that, we can start playing with the objects:: >>> response.xpath('//title/text()').extract_first() 'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' - >>> fetch("http://reddit.com") + >>> fetch("https://reddit.com") >>> response.xpath('//title/text()').extract() ['reddit: the front page of the internet'] diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index 60e0de1f2..f55ecc213 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -4,12 +4,12 @@ from six.moves import copyreg if sys.version_info[0] == 2: from urlparse import urlparse - # workaround for http://bugs.python.org/issue7904 - Python < 2.7 + # workaround for https://bugs.python.org/issue7904 - Python < 2.7 if urlparse('s3://bucket/key').netloc != 'bucket': from urlparse import uses_netloc uses_netloc.append('s3') - # workaround for http://bugs.python.org/issue9374 - Python < 2.7.4 + # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4 if urlparse('s3://bucket/key?key=value').query != 'key=value': from urlparse import uses_query uses_query.append('s3') diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index a94a89205..783d4c383 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -64,7 +64,7 @@ if twisted_version >= (14, 0, 0): """ Twisted-recommended context factory for web clients. - Quoting http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html: + Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html: "The default is to use a BrowserLikePolicyForHTTPS, so unless you have special requirements you can leave this as-is." @@ -100,6 +100,6 @@ else: def getContext(self, hostname=None, port=None): ctx = ClientContextFactory.getContext(self) # Enable all workarounds to SSL bugs as documented by - # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html + # https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_options.html ctx.set_options(SSL.OP_ALL) return ctx diff --git a/scrapy/crawler.py b/scrapy/crawler.py index a33ce9805..5cbc2d7c5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -83,7 +83,7 @@ class Crawler(object): yield defer.maybeDeferred(self.engine.start) except Exception: # In Python 2 reraising an exception after yield discards - # the original traceback (see http://bugs.python.org/issue7563), + # the original traceback (see https://bugs.python.org/issue7563), # so sys.exc_info() workaround is used. # This workaround also works in Python 3, but it is not needed, # and it is slower, so in Python 3 we use native `raise`. diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py index 64d94c489..6748d0265 100644 --- a/scrapy/downloadermiddlewares/chunked.py +++ b/scrapy/downloadermiddlewares/chunked.py @@ -11,7 +11,7 @@ warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, " class ChunkedTransferMiddleware(object): """This middleware adds support for chunked transfer encoding, as - documented in: http://en.wikipedia.org/wiki/Chunked_transfer_encoding + documented in: https://en.wikipedia.org/wiki/Chunked_transfer_encoding """ def process_response(self, request, response, spider): diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 30e49b886..495b103d1 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -75,7 +75,7 @@ class HttpCacheMiddleware(object): return response # RFC2616 requires origin server to set Date header, - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18 + # https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18 if 'Date' not in response.headers: response.headers['Date'] = formatdate(usegmt=1) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index e2d42b6ab..07f43b494 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -188,7 +188,7 @@ class XmlItemExporter(BaseItemExporter): self.xg.endElement(name) self._beautify_newline() - # Workaround for http://bugs.python.org/issue17606 + # Workaround for https://bugs.python.org/issue17606 # Before Python 2.7.4 xml.sax.saxutils required bytes; # since 2.7.4 it requires unicode. The bug is likely to be # fixed in 2.7.6, but 2.7.6 will still support unicode, diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 648b32ec7..1b5e05b1b 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -70,8 +70,8 @@ class RFC2616Policy(object): return True def should_cache_response(self, response, request): - # What is cacheable - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1 - # Response cacheability - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 + # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1 + # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 # Status code 206 is not included because cache can not deal with partial contents cc = self._parse_cachecontrol(response) # obey directive "Cache-Control: no-store" @@ -163,7 +163,7 @@ class RFC2616Policy(object): def _compute_freshness_lifetime(self, response, request, now): # Reference nsHttpResponseHead::ComputeFreshnessLifetime - # http://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#410 + # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#706 cc = self._parse_cachecontrol(response) maxage = self._get_max_age(cc) if maxage is not None: @@ -194,7 +194,7 @@ class RFC2616Policy(object): def _compute_current_age(self, response, request, now): # Reference nsHttpResponseHead::ComputeCurrentAge - # http://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#366 + # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#658 currentage = 0 # If Date header is not set we assume it is a fast connection, and # clock is in sync with the server @@ -414,7 +414,7 @@ class LeveldbCacheStorage(object): def parse_cachecontrol(header): """Parse Cache-Control header - http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 + https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 >>> parse_cachecontrol(b'public, max-age=3600') == {b'public': None, ... b'max-age': b'3600'} diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index d9add1d97..5ca0d19a0 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -82,7 +82,7 @@ class TelnetConsole(protocol.ServerFactory): 'prefs': print_live_refs, 'hpy': hpy, 'help': "This is Scrapy telnet console. For more info see: " \ - "http://doc.scrapy.org/en/latest/topics/telnetconsole.html", + "https://doc.scrapy.org/en/latest/topics/telnetconsole.html", } self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) return telnet_vars diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 7fdb8a086..9f1faa313 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -120,7 +120,7 @@ class S3FilesStore(object): def _get_boto_bucket(self): # disable ssl (is_secure=False) because of this python bug: - # http://bugs.python.org/issue5103 + # https://bugs.python.org/issue5103 c = self.S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False) return c.get_bucket(self.bucket, validate=False) @@ -268,7 +268,7 @@ class FilesPipeline(MediaPipeline): def __init__(self, store_uri, download_func=None, settings=None): if not store_uri: raise NotConfigured - + if isinstance(settings, dict) or settings is None: settings = Settings(settings) diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index fd79905e9..296d27ed8 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -55,7 +55,7 @@ class SignalManager(object): The keyword arguments are passed to the signal handlers (connected through the :meth:`connect` method). - .. _deferreds: http://twistedmatrix.com/documents/current/core/howto/defer.html + .. _deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html """ kwargs.setdefault('sender', self.sender) return _signal.send_catch_log_deferred(signal, **kwargs) diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl index 2c746138f..7d766f4fc 100644 --- a/scrapy/templates/project/module/items.py.tmpl +++ b/scrapy/templates/project/module/items.py.tmpl @@ -3,7 +3,7 @@ # Define here the models for your scraped items # # See documentation in: -# http://doc.scrapy.org/en/latest/topics/items.html +# https://doc.scrapy.org/en/latest/topics/items.html import scrapy diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 1a4b0caa5..c5b542bd6 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -3,7 +3,7 @@ # Define here the models for your spider middleware # # See documentation in: -# http://doc.scrapy.org/en/latest/topics/spider-middleware.html +# https://doc.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index 4e9b32e9e..e58dab089 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -3,7 +3,7 @@ # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting -# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html +# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html class ${ProjectName}Pipeline(object): diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 35a0f9a45..a0557473e 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -5,9 +5,9 @@ # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # -# http://doc.scrapy.org/en/latest/topics/settings.html -# http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html -# http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html +# https://doc.scrapy.org/en/latest/topics/settings.html +# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html +# https://doc.scrapy.org/en/latest/topics/spider-middleware.html BOT_NAME = '$project_name' @@ -25,7 +25,7 @@ ROBOTSTXT_OBEY = True #CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) -# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay +# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs #DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: @@ -45,31 +45,31 @@ ROBOTSTXT_OBEY = True #} # Enable or disable spider middlewares -# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html +# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543, #} # Enable or disable downloader middlewares -# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html +# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html #DOWNLOADER_MIDDLEWARES = { # '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543, #} # Enable or disable extensions -# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html +# See https://doc.scrapy.org/en/latest/topics/extensions.html #EXTENSIONS = { # 'scrapy.extensions.telnet.TelnetConsole': None, #} # Configure item pipelines -# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html +# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html #ITEM_PIPELINES = { # '$project_name.pipelines.${ProjectName}Pipeline': 300, #} # Enable and configure the AutoThrottle extension (disabled by default) -# See http://doc.scrapy.org/en/latest/topics/autothrottle.html +# See https://doc.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 @@ -82,7 +82,7 @@ ROBOTSTXT_OBEY = True #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) -# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = 'httpcache' diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index bb4c74a6e..aa6dcffda 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -57,7 +57,7 @@ def parallel(iterable, count, callable, *args, **named): """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. - Taken from: http://jcalderone.livejournal.com/24285.html + Taken from: https://jcalderone.livejournal.com/24285.html """ coop = task.Cooperator() work = (callable(elem, *args, **named) for elem in iterable) diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 7ab39c97e..f76161a68 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -71,8 +71,8 @@ def create_deprecated_class(name, new_class, clsdict=None, warnings.warn(msg, warn_category, stacklevel=2) super(DeprecatedClass, cls).__init__(name, bases, clsdict_) - # see http://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass - # and http://docs.python.org/2/reference/datamodel.html#customizing-instance-and-subclass-checks + # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass + # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks # for implementation details def __instancecheck__(cls, inst): return any(cls.__subclasscheck__(c) @@ -159,10 +159,10 @@ def update_classpath(path): def method_is_overridden(subclass, base_class, method_name): - """ - Return True if a method named ``method_name`` of a ``base_class`` - is overridden in a ``subclass``. - + """ + Return True if a method named ``method_name`` of a ``base_class`` + is overridden in a ``subclass``. + >>> class Base(object): ... def foo(self): ... pass diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py index 8b659a22a..7cc8d1884 100644 --- a/scrapy/utils/http.py +++ b/scrapy/utils/http.py @@ -11,7 +11,7 @@ def decode_chunked_transfer(chunked_body): decoded body. For more info see: - http://en.wikipedia.org/wiki/Chunked_transfer_encoding + https://en.wikipedia.org/wiki/Chunked_transfer_encoding """ body, h, t = '', '', chunked_body diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 7c95e1e50..828880709 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -154,7 +154,7 @@ class StreamLogger(object): """Fake file-like stream object that redirects writes to a logger instance Taken from: - http://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/ + https://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/ """ def __init__(self, logger, log_level=logging.INFO): self.logger = logger diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 8eed31060..657c53815 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -47,7 +47,7 @@ def parse_url(url, encoding=None): def escape_ajax(url): """ Return the crawleable url according to: - http://code.google.com/web/ajaxcrawling/docs/getting-started.html + https://developers.google.com/webmasters/ajax-crawling/docs/getting-started >>> escape_ajax("www.example.com/ajax.html#!key=value") 'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue' diff --git a/sep/sep-001.rst b/sep/sep-001.rst index 2f0fe3500..3766f38fc 100644 --- a/sep/sep-001.rst +++ b/sep/sep-001.rst @@ -61,7 +61,7 @@ ItemForm -------- Pros: -- same API used for Items (see http://doc.scrapy.org/en/latest/topics/items.html) +- same API used for Items (see https://doc.scrapy.org/en/latest/topics/items.html) - some people consider setitem API more elegant than methods API Cons: diff --git a/sep/sep-006.rst b/sep/sep-006.rst index c0f945b66..522bba134 100644 --- a/sep/sep-006.rst +++ b/sep/sep-006.rst @@ -16,7 +16,7 @@ Motivation ========== When you use Selectors in Scrapy, your final goal is to "extract" the data that -you've selected, as the [http://doc.scrapy.org/en/latest/topics/selectors.html +you've selected, as the [https://doc.scrapy.org/en/latest/topics/selectors.html XPath Selectors documentation] says (bolding by me): When you’re scraping web pages, the most common task you need to perform is @@ -71,5 +71,5 @@ webpage or set of pages. References ========== - 1. XPath Selectors (http://doc.scrapy.org/topics/selectors.html) + 1. XPath Selectors (https://doc.scrapy.org/topics/selectors.html) 2. XPath and XSLT with lxml (http://codespeak.net/lxml/xpathxslt.html) diff --git a/sep/sep-013.rst b/sep/sep-013.rst index 4c11a0762..5b18b7501 100644 --- a/sep/sep-013.rst +++ b/sep/sep-013.rst @@ -44,7 +44,7 @@ Overview of changes proposed Most of the inconsistencies come from the fact that middlewares don't follow the typical -[http://twistedmatrix.com/projects/core/documentation/howto/defer.html +[https://twistedmatrix.com/projects/core/documentation/howto/defer.html deferred] callback/errback chaining logic. Twisted logic is fine and quite intuitive, and also fits middlewares very well. Due to some bad design choices the integration between middleware calls and deferred is far from optional. So diff --git a/sep/sep-017.rst b/sep/sep-017.rst index 7707a1622..86005e3c9 100644 --- a/sep/sep-017.rst +++ b/sep/sep-017.rst @@ -13,7 +13,7 @@ SEP-017: Spider Contracts The motivation for Spider Contracts is to build a lightweight mechanism for testing your spiders, and be able to run the tests quickly without having to wait for all the spider to run. It's partially based on the -[http://en.wikipedia.org/wiki/Design_by_contract Design by contract] approach +[https://en.wikipedia.org/wiki/Design_by_contract Design by contract] approach (hence its name) where you define certain conditions that spider callbacks must met, and you give example testing pages. diff --git a/sep/sep-020.rst b/sep/sep-020.rst index 49d068479..52d78097b 100644 --- a/sep/sep-020.rst +++ b/sep/sep-020.rst @@ -29,7 +29,7 @@ the rows and the further embedded ```` elements denoting the individual fields. One pattern that is particularly well suited for auto-populating an Item Loader -is the `definition list `_:: +is the `definition list `_::
diff --git a/setup.py b/setup.py index c03f0b9f7..327286f5a 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ if has_environment_marker_platform_impl_support(): setup( name='Scrapy', version=version, - url='http://scrapy.org', + url='https://scrapy.org', description='A high-level Web Crawling and Web Scraping framework', long_description=open('README.rst').read(), author='Scrapy developers', diff --git a/tests/__init__.py b/tests/__init__.py index c2e4fd2bf..55b1ecde8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,7 @@ """ tests: this package contains all Scrapy unittests -see http://doc.scrapy.org/en/latest/contributing.html#running-tests +see https://doc.scrapy.org/en/latest/contributing.html#running-tests """ import os diff --git a/tests/keys/example-com.conf b/tests/keys/example-com.conf index 8aa338cd5..1f9c25e43 100644 --- a/tests/keys/example-com.conf +++ b/tests/keys/example-com.conf @@ -1,4 +1,4 @@ -# this is copied from http://stackoverflow.com/a/27931596 +# this is copied from https://stackoverflow.com/a/27931596 [ req ] default_bits = 2048 default_keyfile = server-key.pem @@ -24,7 +24,7 @@ organizationName_default = Example, LLC # Use a friendly name here because its presented to the user. The server's DNS # names are placed in Subject Alternate Names. Plus, DNS names here is deprecated -# by both IETF and CA/Browser Forums. If you place a DNS name here, then you +# by both IETF and CA/Browser Forums. If you place a DNS name here, then you # must include the DNS name in the SAN too (otherwise, Chrome and others that # strictly follow the CA/Browser Baseline Requirements will fail). commonName = Common Name (e.g. server FQDN or YOUR name) From 23c7437e4629199e8ee1ae6bcdf75b7062466010 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 28 Oct 2017 16:34:49 +0530 Subject: [PATCH 278/362] Fix link for 'XPath and XSLT with lxml' --- sep/sep-006.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sep/sep-006.rst b/sep/sep-006.rst index 522bba134..7425c0930 100644 --- a/sep/sep-006.rst +++ b/sep/sep-006.rst @@ -58,7 +58,7 @@ As the name of the method for performing selection (the ``x`` method) is not descriptive nor mnemotechnic enough and clearly clashes with ``extract`` method (x sounds like a short for extract in english), we propose to rename it to `select`, `sel` (is shortness if required), or `xpath` after `lxml's -`_ ``xpath`` method. +`_ ``xpath`` method. Bonus (ItemBuilder) =================== @@ -72,4 +72,4 @@ References ========== 1. XPath Selectors (https://doc.scrapy.org/topics/selectors.html) - 2. XPath and XSLT with lxml (http://codespeak.net/lxml/xpathxslt.html) + 2. XPath and XSLT with lxml (http://lxml.de/xpathxslt.html) From 97d047a055b3af080047768b196ce677fbfaa12e Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 28 Oct 2017 16:48:41 +0530 Subject: [PATCH 279/362] Fix link for Tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 0608693ba..f35b894f3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,4 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests +# Tox (https://tox.readthedocs.io/) is a tool for running tests # in multiple virtualenvs. This configuration file will run the # test suite on all supported python versions. To use it, "pip install tox" # and then run "tox" from this directory. From dae7b1cdd06649db3c692eccd70195a499733448 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 28 Oct 2017 16:53:32 +0530 Subject: [PATCH 280/362] Migrate all subdomains on readthedocs.org to readthedocs.io --- scrapy/templates/project/scrapy.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/templates/project/scrapy.cfg b/scrapy/templates/project/scrapy.cfg index d7f02e0a2..1daeaa541 100644 --- a/scrapy/templates/project/scrapy.cfg +++ b/scrapy/templates/project/scrapy.cfg @@ -1,7 +1,7 @@ # Automatically created by: scrapy startproject # # For more information about the [deploy] section see: -# https://scrapyd.readthedocs.org/en/latest/deploy.html +# https://scrapyd.readthedocs.io/en/latest/deploy.html [settings] default = ${project_name}.settings From df7e0a4315f9db2c74fa9e9a0654f44277da2e55 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sat, 28 Oct 2017 23:37:44 +0530 Subject: [PATCH 281/362] Use https link in default user agent --- docs/topics/settings.rst | 2 +- scrapy/settings/default_settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4b15cb607..afa666659 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1324,7 +1324,7 @@ the default value for this setting see: https://boutell.com/newfaq/misc/urllengt USER_AGENT ---------- -Default: ``"Scrapy/VERSION (+http://scrapy.org)"`` +Default: ``"Scrapy/VERSION (+https://scrapy.org)"`` The default User-Agent to use when crawling, unless overridden. diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 697314b7f..ead511473 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -270,7 +270,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates')) URLLENGTH_LIMIT = 2083 -USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % import_module('scrapy').__version__ +USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__version__ TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_PORT = [6023, 6073] From 8a7552370de2d31fbd3564e47fa92049c8efee6f Mon Sep 17 00:00:00 2001 From: colinmorris Date: Tue, 31 Oct 2017 17:14:53 -0400 Subject: [PATCH 282/362] revise/modernize item exporter example in docs --- docs/topics/exporters.rst | 50 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index b6139af92..28f5ad9c2 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -36,38 +36,36 @@ to export 3. and finally call the :meth:`~BaseItemExporter.finish_exporting` to signal the end of the exporting process -Here you can see an :doc:`Item Pipeline ` which uses an Item -Exporter to export scraped items to different files, one per spider:: +Here you can see an :doc:`Item Pipeline ` which uses multiple +Item Exporters to group scraped items to different files according to the +value of one of their fields:: - from scrapy import signals - from scrapy.exporters import XmlItemExporter + from scrapy.exporters import XmlItemExporter - class XmlExportPipeline(object): + class PerYearXmlExportPipeline(object): + """Distribute items across multiple XML files according to their 'year' field""" - def __init__(self): - self.files = {} + def open_spider(self, spider): + self.year_to_exporter = {} - @classmethod - def from_crawler(cls, crawler): - pipeline = cls() - crawler.signals.connect(pipeline.spider_opened, signals.spider_opened) - crawler.signals.connect(pipeline.spider_closed, signals.spider_closed) - return pipeline + def close_spider(self, spider): + for exporter in self.year_to_exporter.itervalues(): + exporter.finish_exporting() + exporter.file.close() - def spider_opened(self, spider): - file = open('%s_products.xml' % spider.name, 'w+b') - self.files[spider] = file - self.exporter = XmlItemExporter(file) - self.exporter.start_exporting() + def _exporter_for_item(self, item): + year = item['year'] + if year not in self.year_to_exporter: + f = open('{}.xml'.format(year), 'w+b') + exporter = XmlItemExporter(f) + exporter.start_exporting() + self.year_to_exporter[year] = exporter + return self.year_to_exporter[year] - def spider_closed(self, spider): - self.exporter.finish_exporting() - file = self.files.pop(spider) - file.close() - - def process_item(self, item, spider): - self.exporter.export_item(item) - return item + def process_item(self, item, spider): + exporter = self._exporter_for_item(item) + exporter.export_item(item) + return item .. _topics-exporters-field-serialization: From 23e571e860729fad1f4351cde69b77d88837e628 Mon Sep 17 00:00:00 2001 From: colinmorris Date: Tue, 31 Oct 2017 18:08:47 -0400 Subject: [PATCH 283/362] fix issues identified in review --- docs/topics/exporters.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 28f5ad9c2..95f7920f8 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -49,14 +49,14 @@ value of one of their fields:: self.year_to_exporter = {} def close_spider(self, spider): - for exporter in self.year_to_exporter.itervalues(): + for exporter in self.year_to_exporter.values(): exporter.finish_exporting() exporter.file.close() def _exporter_for_item(self, item): year = item['year'] if year not in self.year_to_exporter: - f = open('{}.xml'.format(year), 'w+b') + f = open('{}.xml'.format(year), 'wb') exporter = XmlItemExporter(f) exporter.start_exporting() self.year_to_exporter[year] = exporter From abb6d0a1c1dd4d1ab6f734f2b366b9d851a52a48 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Wed, 1 Nov 2017 17:26:59 +0300 Subject: [PATCH 284/362] Use portable pypy directly They are provided by https://github.com/squeaky-pl/portable-pypy --- .travis.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9c51fafb2..d4f30814d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,16 +24,10 @@ matrix: install: - | if [ "$TOXENV" = "pypy" ]; then - export PYENV_ROOT="$HOME/.pyenv" - if [ -f "$PYENV_ROOT/bin/pyenv" ]; then - pushd "$PYENV_ROOT" && git pull && popd - else - rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT" - fi - # get latest portable PyPy from pyenv directly (thanks to natural version sort option -V) - export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy2.7-portable-[0-9][\.0-9]*$' |sort -V |tail -1` - "$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION" - virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION" + export PYPY_VERSION="pypy-5.9-linux_x86_64-portable" + wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi - pip install -U tox twine wheel codecov From 846fd83512bb45335195b060d76b46060b4b6e3d Mon Sep 17 00:00:00 2001 From: IAlwaysBeCoding Date: Sat, 11 Nov 2017 18:30:01 -0500 Subject: [PATCH 285/362] removed commented out code, wrapped line to pep-8 and removed backlashes --- docs/topics/commands.rst | 3 +++ scrapy/commands/parse.py | 23 ++++++++++++++++++++++ tests/test_command_parse.py | 39 +++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index dc8067d7e..07c69ddda 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -430,6 +430,9 @@ Supported options: * ``--callback`` or ``-c``: spider method to use as callback for parsing the response +* ``--meta`` or ``-m``: additional request meta that will be pass to the callback + request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' + * ``--pipelines``: process items through pipelines * ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider` diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index a90095146..69418a478 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,4 +1,5 @@ from __future__ import print_function +import json import logging from w3lib.url import is_url @@ -48,6 +49,8 @@ class Command(ScrapyCommand): help="use CrawlSpider rules to discover the callback") parser.add_option("-c", "--callback", dest="callback", help="use this callback for parsing, instead looking for a callback") + parser.add_option("-m", "--meta", dest="meta", + help="inject extra meta into the Request, it must be a valid raw json string") parser.add_option("-d", "--depth", dest="depth", type="int", default=1, help="maximum depth for parsing requests [default: %default]") parser.add_option("-v", "--verbose", dest="verbose", action="store_true", @@ -204,6 +207,10 @@ class Command(ScrapyCommand): req.callback = callback return requests + #update request meta if any extra meta was passed through the --meta/-m opts. + if opts.meta: + request.meta.update(opts.meta) + request.meta['_depth'] = 1 request.meta['_callback'] = request.callback request.callback = callback @@ -211,11 +218,27 @@ class Command(ScrapyCommand): def process_options(self, args, opts): ScrapyCommand.process_options(self, args, opts) + + self.process_spider_arguments(opts) + self.process_request_meta(opts) + + def process_spider_arguments(self, opts): + try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) + def process_request_meta(self, opts): + + if opts.meta: + try: + opts.meta = json.loads(opts.meta) + except ValueError: + raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \ + "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False) + + def run(self, args, opts): # parse arguments if not len(args) == 1 or not is_url(args[0]): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index b6d6db9ee..66dd17110 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -29,6 +29,21 @@ class MySpider(scrapy.Spider): self.logger.debug('It Works!') return [scrapy.Item(), dict(foo='bar')] + def parse_request_with_meta(self, response): + foo = response.meta.get('foo', 'bar') + + if foo == 'bar': + self.logger.debug('It Does Not Work :(') + else: + self.logger.debug('It Works!') + + def parse_request_without_meta(self, response): + foo = response.meta.get('foo', 'bar') + + if foo == 'bar': + self.logger.debug('It Works!') + else: + self.logger.debug('It Does Not Work :(') class MyGoodCrawlSpider(CrawlSpider): name = 'goodcrawl{0}' @@ -84,6 +99,30 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} self.url('/html')]) self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + @defer.inlineCallbacks + def test_request_with_meta(self): + raw_json_string = '{"foo" : "baz"}' + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '--meta', raw_json_string, + '-c', 'parse_request_with_meta', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '-m', raw_json_string, + '-c', 'parse_request_with_meta', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + + + @defer.inlineCallbacks + def test_request_without_meta(self): + _, _, stderr = yield self.execute(['--spider', self.spider_name, + '-c', 'parse_request_without_meta', + self.url('/html')]) + self.assertIn("DEBUG: It Works!", to_native_str(stderr)) + + @defer.inlineCallbacks def test_pipelines(self): _, _, stderr = yield self.execute(['--spider', self.spider_name, From 62a626102877de4998538717f34e61d2f7d2622c Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Sat, 18 Nov 2017 20:03:59 +0100 Subject: [PATCH 286/362] Issues a warning when user puts a URL into allowed_domains (#2250) --- scrapy/spidermiddlewares/offsite.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index ea1c9270f..f51b0a2b0 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -52,6 +52,10 @@ class OffsiteMiddleware(object): allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: return re.compile('') # allow all by default + for domainIndex in range(0, len(allowed_domains)): + url_pattern = re.compile("^https?://.*$") + if url_pattern.match(allowed_domains[domainIndex]): + logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % allowed_domains[domainIndex]) regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) return re.compile(regex) From 5441cc18e43cba6b8196ece49d6b454badb246f0 Mon Sep 17 00:00:00 2001 From: KosayJabre Date: Sun, 19 Nov 2017 18:09:38 -0400 Subject: [PATCH 287/362] Separated import statements Just separated the import statements. Tiny change - testing GitHub! --- scrapy/commands/edit.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index a7f8983b4..25d843a53 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,4 +1,5 @@ -import sys, os +import sys +import os from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError From 91ff194d1e9477d2196817ea1dc8beb220c3e058 Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Mon, 20 Nov 2017 21:23:31 +0100 Subject: [PATCH 288/362] looping over allowed_domains directly instead of via index --- scrapy/spidermiddlewares/offsite.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index f51b0a2b0..8ff35e29f 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -52,10 +52,10 @@ class OffsiteMiddleware(object): allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: return re.compile('') # allow all by default - for domainIndex in range(0, len(allowed_domains)): + for domain in allowed_domains: url_pattern = re.compile("^https?://.*$") - if url_pattern.match(allowed_domains[domainIndex]): - logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % allowed_domains[domainIndex]) + if url_pattern.match(domain): + logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain) regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) return re.compile(regex) From 0b14cb44aabcc8322e09a46c4e14913c94886e12 Mon Sep 17 00:00:00 2001 From: Jesse Bakker Date: Thu, 23 Nov 2017 15:25:43 +0100 Subject: [PATCH 289/362] Added from_crawler to middleware docs --- docs/topics/downloader-middleware.rst | 11 +++++++++++ docs/topics/spider-middleware.rst | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0d168017f..983a93290 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -157,6 +157,17 @@ more of the following methods: :param spider: the spider for which this request is intended :type spider: :class:`~scrapy.spiders.Spider` object + .. method:: from_crawler(cls, crawler) + + If present, this classmethod is called to create a middleware instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the middleware. Crawler object provides access to all Scrapy core + components like settings and signals; it is a way for middleware to + access them and hook its functionality into Scrapy. + + :param crawler: crawler that uses this middleware + :type crawler: :class:`~scrapy.crawler.Crawler` object + .. _topics-downloader-middleware-ref: Built-in downloader middleware reference diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index a2d2556c5..c297ed556 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -164,6 +164,17 @@ following methods: :param spider: the spider to whom the start requests belong :type spider: :class:`~scrapy.spiders.Spider` object + .. method:: from_crawler(cls, crawler) + + If present, this classmethod is called to create a middleware instance + from a :class:`~scrapy.crawler.Crawler`. It must return a new instance + of the middleware. Crawler object provides access to all Scrapy core + components like settings and signals; it is a way for middleware to + access them and hook its functionality into Scrapy. + + :param crawler: crawler that uses this middleware + :type crawler: :class:`~scrapy.crawler.Crawler` object + .. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception From 6af323d7c85eeee40d90a20133504df26a593304 Mon Sep 17 00:00:00 2001 From: IAlwaysBeCoding Date: Sun, 26 Nov 2017 00:24:52 +0100 Subject: [PATCH 290/362] Fix spelling mistake on scrapy parse command docs Fixed spelling mistake from "will be pass" to "will be passed" --- docs/topics/commands.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 07c69ddda..06f9a485b 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -430,7 +430,7 @@ Supported options: * ``--callback`` or ``-c``: spider method to use as callback for parsing the response -* ``--meta`` or ``-m``: additional request meta that will be pass to the callback +* ``--meta`` or ``-m``: additional request meta that will be passed to the callback request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' * ``--pipelines``: process items through pipelines From 8ec3b476b03d6b8424f6dfc556758392e7a5a61f Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Sun, 26 Nov 2017 16:36:15 +0100 Subject: [PATCH 291/362] triggering a warning when user puts URL in allowed_domains now covered by test --- scrapy/spidermiddlewares/offsite.py | 3 ++- tests/test_spidermiddleware_offsite.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 8ff35e29f..647792e5d 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -6,6 +6,7 @@ See documentation in docs/topics/spider-middleware.rst import re import logging +import warnings from scrapy import signals from scrapy.http import Request @@ -55,7 +56,7 @@ class OffsiteMiddleware(object): for domain in allowed_domains: url_pattern = re.compile("^https?://.*$") if url_pattern.match(domain): - logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain) + warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, Warning) regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) return re.compile(regex) diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 9ad86313c..b532cc2ec 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -6,6 +6,7 @@ from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.offsite import OffsiteMiddleware from scrapy.utils.test import get_crawler +import warnings class TestOffsiteMiddleware(TestCase): @@ -68,3 +69,13 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3): reqs = [Request('http://scrapytest.org/1')] out = list(self.mw.process_spider_output(res, reqs, self.spider)) self.assertEqual(out, reqs) + + +class TestOffsiteMiddleware5(TestOffsiteMiddleware4): + + def test_get_host_regex(self): + self.spider.allowed_domains = ['http://scrapytest.org', 'scrapy.org', 'scrapy.test.org'] + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + self.mw.get_host_regex(self.spider) + assert "allowed_domains accepts only domains, not URLs." in str(w[-1].message) From 454d5e57333e9f33c8d684e4e21f8f7e9493f310 Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Sun, 26 Nov 2017 20:07:04 +0100 Subject: [PATCH 292/362] checking for subclass of URLWarning instead of checking error message text when URL in allowed_domains --- scrapy/spidermiddlewares/offsite.py | 7 ++++++- tests/test_spidermiddleware_offsite.py | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 647792e5d..f595eef42 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -56,10 +56,15 @@ class OffsiteMiddleware(object): for domain in allowed_domains: url_pattern = re.compile("^https?://.*$") if url_pattern.match(domain): - warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, Warning) + warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, URLWarning) + regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) return re.compile(regex) def spider_opened(self, spider): self.host_regex = self.get_host_regex(spider) self.domains_seen = set() + + +class URLWarning(Warning): + pass \ No newline at end of file diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index b532cc2ec..7e4af0d4c 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -5,6 +5,7 @@ from six.moves.urllib.parse import urlparse from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.offsite import OffsiteMiddleware +from scrapy.spidermiddlewares.offsite import URLWarning from scrapy.utils.test import get_crawler import warnings @@ -78,4 +79,4 @@ class TestOffsiteMiddleware5(TestOffsiteMiddleware4): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") self.mw.get_host_regex(self.spider) - assert "allowed_domains accepts only domains, not URLs." in str(w[-1].message) + assert issubclass(w[-1].category, URLWarning) From 22c68baf990f15d249f38c481f24a984977be3e5 Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Thu, 7 Dec 2017 18:38:29 +0100 Subject: [PATCH 293/362] url_pattern is now being compiled before entering the loop --- scrapy/spidermiddlewares/offsite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index f595eef42..310166cad 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -53,8 +53,8 @@ class OffsiteMiddleware(object): allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: return re.compile('') # allow all by default + url_pattern = re.compile("^https?://.*$") for domain in allowed_domains: - url_pattern = re.compile("^https?://.*$") if url_pattern.match(domain): warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, URLWarning) @@ -67,4 +67,4 @@ class OffsiteMiddleware(object): class URLWarning(Warning): - pass \ No newline at end of file + pass From f716843a66829350063e55f8df768eb538c6b05c Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 12 Dec 2017 16:19:43 +0500 Subject: [PATCH 294/362] DOC update "Contributing" docs: * suggest Stack Overflow for Scrapy usage questions; * encourage users to submit test-only pull requests with reproducable examples; * encourage users to pick up stalled pull requests; * we don't use AUTHORS file as a main acknowledgement source; * suggest using Sphinx autodocs extension --- docs/contributing.rst | 58 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index f3732ab06..eb736bf30 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -19,12 +19,16 @@ There are many ways to contribute to Scrapy. Here are some of them: the guidelines detailed in `Reporting bugs`_ below. * Submit patches for new functionality and/or bug fixes. Please read - `Writing patches`_ and `Submitting patches`_ below for details on how to + :ref:`writing-patches` and `Submitting patches`_ below for details on how to write and submit a patch. * Join the `Scrapy subreddit`_ and share your ideas on how to improve Scrapy. We're always open to suggestions. +* Answer Scrapy questions at + `Stack Overflow `__. + + Reporting bugs ============== @@ -40,9 +44,14 @@ guidelines when reporting a new bug. * check the :ref:`FAQ ` first to see if your issue is addressed in a well-known question +* if you have a general question about scrapy usage, please ask it at + `Stack Overflow `__ + (use "scrapy" tag). + * check the `open issues`_ to see if it has already been reported. If it has, - don't dismiss the report but check the ticket history and comments, you may - find additional useful information to contribute. + don't dismiss the report, but check the ticket history and comments. If you + have additional useful information, please leave a comment, or consider + :ref:`sending a pull request ` with a fix. * search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has been discussed there, or if you're not sure if what you're seeing is a bug. @@ -54,12 +63,20 @@ guidelines when reporting a new bug. it. See for example StackOverflow's guide on creating a `Minimal, Complete, and Verifiable example`_ exhibiting the issue. +* the most awesome way to provide a complete reproducible example is to + send a pull request which adds a failing test case to the + Scrapy testing suite (see :ref:`submitting-patches`). + This is helpful even if you don't have an intention to + fix the issue yourselves. + * include the output of ``scrapy version -v`` so developers working on your bug know exactly which version and platform it occurred on, which is often very helpful for reproducing it, or knowing if it was already fixed. .. _Minimal, Complete, and Verifiable example: https://stackoverflow.com/help/mcve +.. _writing-patches: + Writing patches =============== @@ -83,6 +100,8 @@ Well-written patches should: the documentation changes in the same patch. See `Documentation policies`_ below. +.. _submitting-patches: + Submitting patches ================== @@ -100,11 +119,22 @@ starting point is to send a pull request on GitHub. It can be simple enough to illustrate your idea, and leave documentation/tests for later, after the idea has been validated and proven useful. Alternatively, you can start a conversation in the `Scrapy subreddit`_ to discuss your idea first. + +Sometimes there is an existing pull request for the problem you'd like to +solve, which is stalled for some reason. Often the pull request is in a +right direction, but changes are requested by Scrapy maintainers, and the +original pull request author haven't had time to address them. +In this case consider picking up this pull request: open +a new pull request with all commits from the original pull request, as well as +additional changes to address the raised issues. Doing so helps a lot; it is +not considered rude as soon as original the author is acknowledged by keeping +his/her commits. + When writing GitHub pull requests, try to keep titles short but descriptive. E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests" prefer "Fix hanging when exception occurs in start_requests (#411)" -instead of "Fix for #411". -Complete titles make it easy to skim through the issue tracker. +instead of "Fix for #411". Complete titles make it easy to skim through +the issue tracker. Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports removal, etc) in separate commits than functional changes. This will make pull @@ -121,21 +151,29 @@ Scrapy: * It's OK to use lines longer than 80 chars if it improves the code readability. -* Don't put your name in the code you contribute. Our policy is to keep - the contributor's name in the `AUTHORS`_ file distributed with Scrapy. +* Don't put your name in the code you contribute; git provides enough + metadata to identify author of the code. + See https://help.github.com/articles/setting-your-username-in-git/ for + setup instructions. Documentation policies ====================== * **Don't** use docstrings for documenting classes, or methods which are - already documented in the official (sphinx) documentation. For example, the - :meth:`ItemLoader.add_value` method should be documented in the sphinx - documentation, not its docstring. + already documented in the official (sphinx) documentation. Alternatively, + **do** provide a docstring, but make sure sphinx documentation uses + autodoc_ extension to pull the docstring. For example, the + :meth:`ItemLoader.add_value` method should be either + documented only in the sphinx documentation (not it a docstring), or + it should have a docstring which is pulled to sphinx documentation using + autodoc_ extension. * **Do** use docstrings for documenting functions not present in the official (sphinx) documentation, such as functions from ``scrapy.utils`` package and its sub-modules. +.. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html + Tests ===== From 9aa9dd8d45a2ce0c8e6ae0732e610f020735df7e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 12 Dec 2017 19:17:00 +0500 Subject: [PATCH 295/362] DOC mention an easier way to track pull requests locally. Thanks @eliasdorneles! --- docs/contributing.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index eb736bf30..9a02634cb 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -127,9 +127,16 @@ original pull request author haven't had time to address them. In this case consider picking up this pull request: open a new pull request with all commits from the original pull request, as well as additional changes to address the raised issues. Doing so helps a lot; it is -not considered rude as soon as original the author is acknowledged by keeping +not considered rude as soon as the original author is acknowledged by keeping his/her commits. +You can pull an existing pull request to a local branch +by running ``git fetch upstream pull/$PR_NUMBER/head:$BRANCH_NAME_TO_CREATE`` +(replace 'upstream' with a remote name for scrapy repository, +``$PR_NUMBER`` with an ID of the pull request, and ``$BRANCH_NAME_TO_CREATE`` +with a name of the branch you want to create locally). +See also: https://help.github.com/articles/checking-out-pull-requests-locally/#modifying-an-inactive-pull-request-locally. + When writing GitHub pull requests, try to keep titles short but descriptive. E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests" prefer "Fix hanging when exception occurs in start_requests (#411)" From 44623687ab8936c5696f68f74e438a2891880c82 Mon Sep 17 00:00:00 2001 From: Hugo Date: Tue, 19 Dec 2017 17:59:05 +0200 Subject: [PATCH 296/362] Drop support for EOL Python 3.3 --- .travis.yml | 2 -- README.rst | 6 +++++- docs/faq.rst | 2 +- docs/intro/install.rst | 10 +++++----- setup.py | 2 +- tox.ini | 12 ++++-------- 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index d4f30814d..66de9ed03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,8 +13,6 @@ matrix: env: TOXENV=jessie - python: 2.7 env: TOXENV=pypy - - python: 3.3 - env: TOXENV=py33 - python: 3.5 env: TOXENV=py35 - python: 3.6 diff --git a/README.rst b/README.rst index 45135c7a2..1361eac26 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,10 @@ Scrapy :target: https://pypi.python.org/pypi/Scrapy :alt: PyPI Version +.. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg + :target: https://pypi.python.org/pypi/Scrapy + :alt: Supported Python Versions + .. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg :target: https://travis-ci.org/scrapy/scrapy :alt: Build Status @@ -36,7 +40,7 @@ https://scrapy.org Requirements ============ -* Python 2.7 or Python 3.3+ +* Python 2.7 or Python 3.4+ * Works on Linux, Windows, Mac OSX, BSD Install diff --git a/docs/faq.rst b/docs/faq.rst index 42c3abbfa..484226979 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars What Python versions does Scrapy support? ----------------------------------------- -Scrapy is supported under Python 2.7 and Python 3.3+. +Scrapy is supported under Python 2.7 and Python 3.4+. Python 2.6 support was dropped starting at Scrapy 0.20. Python 3 support was added in Scrapy 1.1. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 12d489612..a2e3f506e 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,7 +7,7 @@ Installation guide Installing Scrapy ================= -Scrapy runs on Python 2.7 and Python 3.3 or above. +Scrapy runs on Python 2.7 and Python 3.4 or above. If you're using `Anaconda`_ or `Miniconda`_, you can install the package from the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows @@ -132,12 +132,12 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: .. _intro-install-ubuntu: -Ubuntu 12.04 or above +Ubuntu 14.04 or above --------------------- Scrapy is currently tested with recent-enough versions of lxml, twisted and pyOpenSSL, and is compatible with recent Ubuntu distributions. -But it should support older versions of Ubuntu too, like Ubuntu 12.04, +But it should support older versions of Ubuntu too, like Ubuntu 14.04, albeit with potential issues with TLS connections. **Don't** use the ``python-scrapy`` package provided by Ubuntu, they are @@ -163,8 +163,8 @@ you can install Scrapy with ``pip`` after that:: pip install scrapy .. note:: - The same non-python dependencies can be used to install Scrapy in Debian - Wheezy (7.0) and above. + The same non-Python dependencies can be used to install Scrapy in Debian + Jessue (8.0) and above. .. _intro-install-macos: diff --git a/setup.py b/setup.py index 327286f5a..2619bd544 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', @@ -61,6 +60,7 @@ setup( 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', install_requires=[ 'Twisted>=13.1.0', 'w3lib>=1.17.0', diff --git a/tox.ini b/tox.ini index f35b894f3..5c543475c 100644 --- a/tox.ini +++ b/tox.ini @@ -63,25 +63,21 @@ basepython = pypy commands = py.test {posargs:scrapy tests} -[testenv:py33] -basepython = python3.3 +[testenv:py34] +basepython = python3.4 deps = -rrequirements-py3.txt # Extras Pillow -rtests/requirements-py3.txt -[testenv:py34] -basepython = python3.4 -deps = {[testenv:py33]deps} - [testenv:py35] basepython = python3.5 -deps = {[testenv:py33]deps} +deps = {[testenv:py34]deps} [testenv:py36] basepython = python3.6 -deps = {[testenv:py33]deps} +deps = {[testenv:py34]deps} [docs] changedir = docs From f11c21c6fc62b64a2bbee0e19e2098ed6257cf19 Mon Sep 17 00:00:00 2001 From: Hugo Date: Wed, 20 Dec 2017 17:05:56 +0200 Subject: [PATCH 297/362] Test on Python 3.4 --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 66de9ed03..e2e9e0cc1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,8 @@ matrix: env: TOXENV=jessie - python: 2.7 env: TOXENV=pypy + - python: 3.4 + env: TOXENV=py34 - python: 3.5 env: TOXENV=py35 - python: 3.6 From cbcf80b98ff66db1ccf625fa52c4de8935331972 Mon Sep 17 00:00:00 2001 From: Hugo Date: Wed, 20 Dec 2017 17:34:13 +0200 Subject: [PATCH 298/362] Fix typo [CI skip] --- docs/intro/install.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index a2e3f506e..22bc84a40 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -164,7 +164,7 @@ you can install Scrapy with ``pip`` after that:: .. note:: The same non-Python dependencies can be used to install Scrapy in Debian - Jessue (8.0) and above. + Jessie (8.0) and above. .. _intro-install-macos: From ea41114cf0ab2782650792ad204cf43fc148c749 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 25 Dec 2017 12:29:02 +0300 Subject: [PATCH 299/362] Mention PyPy support, add PyPy to install docs --- docs/faq.rst | 4 +++- docs/intro/install.rst | 25 ++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 484226979..7eecc999f 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -69,9 +69,11 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars What Python versions does Scrapy support? ----------------------------------------- -Scrapy is supported under Python 2.7 and Python 3.4+. +Scrapy is supported under Python 2.7 and Python 3.4+ +under CPython (default Python implementation) and PyPy (only for Python 2.7). Python 2.6 support was dropped starting at Scrapy 0.20. Python 3 support was added in Scrapy 1.1. +PyPy support was added in Scrapy 1.4, PyPy version tested is PyPy2-v5.9.0. .. note:: For Python 3 support on Windows, it is recommended to use diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 22bc84a40..b00dc2cd6 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,7 +7,8 @@ Installation guide Installing Scrapy ================= -Scrapy runs on Python 2.7 and Python 3.4 or above. +Scrapy runs on Python 2.7 and Python 3.4 or above +under CPython (default Python implementation) and PyPy (only for Python 2.7). If you're using `Anaconda`_ or `Miniconda`_, you can install the package from the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows @@ -223,6 +224,28 @@ After any of these workarounds you should be able to install Scrapy:: pip install Scrapy +PyPy +---- + +We recommend using the latest PyPy version. The version tested is PyPy2-v5.9.0. + +Most scrapy dependencides now have binary wheels for CPython, but not for PyPy. +This means that these dependecies will be built during installation. +On OS X, you are likely to face an issue with building Cryptography dependency, +solution to this problem is described +`here `_, +that is to ``brew install openssl`` and then export the flags that this command +recommends (only needed when installing scrapy). Installing on Linux has no special +issues besides installing build dependencies. +Installing scrapy with PyPy on Windows is not tested. + +You can check that scrapy is installed correctly by running ``scrapy bench``. +If this command gives errors such as +``TypeError: ... got 2 unexpected keyword arguments``, this means +that setuptools was unable to pick up one PyPy-specific dependency. +To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``. + + .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing/ .. _lxml: http://lxml.de/ From 1058169f0e3a8646dbd20f9b4c0b599ed9f6d08e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 25 Dec 2017 15:31:07 +0500 Subject: [PATCH 300/362] setup.py: mention that we support PyPy. See GH-2213. --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 2619bd544..06a36e2ba 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,8 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', From f71df6f9addca10b562bb22890b5ea1c37efde5c Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 25 Dec 2017 13:46:22 +0300 Subject: [PATCH 301/362] Run tests for PyPy3 --- .travis.yml | 9 +++++++++ tox.ini | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/.travis.yml b/.travis.yml index e2e9e0cc1..6635f5d3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,8 @@ matrix: env: TOXENV=jessie - python: 2.7 env: TOXENV=pypy + - python: 2.7 + env: TOXENV=pypy3 - python: 3.4 env: TOXENV=py34 - python: 3.5 @@ -30,6 +32,13 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi + if [ "$TOXENV" = "pypy3" ]; then + export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable" + wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2" + tar -jxf ${PYPY_VERSION}.tar.bz2 + virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index 5c543475c..60ff8c15e 100644 --- a/tox.ini +++ b/tox.ini @@ -79,6 +79,12 @@ deps = {[testenv:py34]deps} basepython = python3.6 deps = {[testenv:py34]deps} +[testenv:pypy3] +basepython = pypy3 +deps = {[testenv:py34]deps} +commands = + py.test {posargs:scrapy tests} + [docs] changedir = docs deps = From 041308afe7c40de7088f75b0e0c312ecd5de428a Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 25 Dec 2017 14:27:20 +0300 Subject: [PATCH 302/362] Fix get_func_args test for pypy3 These built-in functions are exposed as methods in PyPy3. For scrapy this does not matter as: 1) they do not work for CPython at all 2) get_func_args is checked for presense of an argument in scrapy, extra "self" does not matter. But it still makes sense to leave these tests so that we know we shouldn't use get_func_args for built-in functions/methods. --- tests/test_utils_python.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 115f523e9..f6133657b 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -219,9 +219,12 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), []) else: - self.assertEqual(get_func_args(six.text_type.split), ['sep', 'maxsplit']) - self.assertEqual(get_func_args(" ".join), ['list']) - self.assertEqual(get_func_args(operator.itemgetter(2)), ['obj']) + stripself = not six.PY2 # PyPy3 exposes them as methods + self.assertEqual( + get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit']) + self.assertEqual(get_func_args(" ".join, stripself), ['list']) + self.assertEqual( + get_func_args(operator.itemgetter(2), stripself), ['obj']) def test_without_none_values(self): From bb1f31189128cb2272c1302350387075fbbb730a Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 25 Dec 2017 15:46:05 +0300 Subject: [PATCH 303/362] Add PyPy3 support to faq and install doc --- docs/faq.rst | 4 ++-- docs/intro/install.rst | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 7eecc999f..7a0628f88 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -70,10 +70,10 @@ What Python versions does Scrapy support? ----------------------------------------- Scrapy is supported under Python 2.7 and Python 3.4+ -under CPython (default Python implementation) and PyPy (only for Python 2.7). +under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). Python 2.6 support was dropped starting at Scrapy 0.20. Python 3 support was added in Scrapy 1.1. -PyPy support was added in Scrapy 1.4, PyPy version tested is PyPy2-v5.9.0. +PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5. .. note:: For Python 3 support on Windows, it is recommended to use diff --git a/docs/intro/install.rst b/docs/intro/install.rst index b00dc2cd6..4a9aa3cfb 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -8,7 +8,7 @@ Installing Scrapy ================= Scrapy runs on Python 2.7 and Python 3.4 or above -under CPython (default Python implementation) and PyPy (only for Python 2.7). +under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). If you're using `Anaconda`_ or `Miniconda`_, you can install the package from the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows @@ -227,7 +227,8 @@ After any of these workarounds you should be able to install Scrapy:: PyPy ---- -We recommend using the latest PyPy version. The version tested is PyPy2-v5.9.0. +We recommend using the latest PyPy version. The version tested is 5.9.0. +For PyPy3, only Linux installation was tested. Most scrapy dependencides now have binary wheels for CPython, but not for PyPy. This means that these dependecies will be built during installation. From a1cc5a63d3e253c325159fdc6ebf4cd3faa37c49 Mon Sep 17 00:00:00 2001 From: Raphael Date: Wed, 27 Dec 2017 18:54:17 -0200 Subject: [PATCH 304/362] Add mention to dont_merge_cookies in CookiesMiddlewares docs (#2999) (#3030) Add mention to dont_merge_cookies in CookiesMiddlewares docs (#2999) --- docs/topics/downloader-middleware.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 983a93290..863620900 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -237,6 +237,17 @@ Default: ``True`` Whether to enable the cookies middleware. If disabled, no cookies will be sent to web servers. +Notice that if the :class:`~scrapy.http.Request` +has ``meta['dont_merge_cookies']`` evaluated to ``True``. +despite the value of :setting:`COOKIES_ENABLED` the cookies will **not** be +sent to web servers and received cookies in +:class:`~scrapy.http.Response` will **not** be merged with the existing +cookies. + +For more detailed information see the ``cookies`` parameter in +:class:`~scrapy.http.Request` + + .. setting:: COOKIES_DEBUG COOKIES_DEBUG From 461f9daff5747728e26cd60e9dfe531092f58132 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 27 Jul 2017 19:56:17 +0200 Subject: [PATCH 305/362] Update release notes for upcoming 1.4.1 version --- docs/news.rst | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 577c93b8e..7ecf22470 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,53 @@ Release notes ============= +Scrapy 1.4.1 (2017-XX-XX) +------------------------- + +New features +~~~~~~~~~~~~ + +- Support ```` tags in ``Response.follow`` (:issue:`2785`) +- Support for ``ptpython`` REPL (:issue:`2654`) +- Populate spider variable when using ``shell.inspect_response`` (:issue:`2812`) +- Handle HTTP 308 Permanent Redirect (:issue:`2844`) +- Add 522 and 524 to ``RETRY_HTTP_CODES`` (:issue:`2851`) +- Log versions information at startup (:issue:`2857`) +- Add template for a downloader middleware (:issue:`2755`) +- Explicit message for NotImplementedError when parse callback not defined (:issue:`2831`) + +Bug fixes +~~~~~~~~~ + +- Fix PyPy test failures (:issue:`2793`) +- Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`) +- Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`) +- Add verification to check if Request callback is callable (:issue:`2766`) +- Port ``extras/qpsclient.py`` to Python 3 (:issue:`2849`) +- Use getfullargspec under the scenes for Python 3 to stop DeprecationWarning (:issue:`2862`) +- Update deprecated test aliases (:issue:`2876`) +- Fix ``SitemapSpider`` support for alternate links (:issue:`2853`) +- Fix logging of settings overridden by ``custom_settings``; + **this is technically backwards-incompatible** because the logger + changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please + update your log parsers if needed (:issue:`1343`) + +Docs +~~~~ + +- Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting. (:issue:`2756`) +- Include references to Scrapy subreddit in the docs (:issue:`2762`) +- Use https:// for readthedocs links +- Document CloseSpider extension better (:issue:`2759`) +- Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example (:issue:`2781`) +- Spelling mistake and typos (:issue:`2828`, :issue:`2837`, :issue:`#2884`) +- Clarify ``CSVFeedSpider.headers`` documentation (:issue:`2826`) +- Document ``DontCloseSpider`` exception and clarify ``spider_idle`` (:issue:`2791`) +- Update "Releases" section in README (:issue:`2764`) +- Fix rst syntax in ``DOWNLOAD_FAIL_ON_DATALOSS`` docs (:issue:`2763`) +- Small fix in description of startproject arguments (:issue:`2866`) + + Scrapy 1.4.0 (2017-05-18) ------------------------- From 45b0e1a0e4c51a773b39be14334a999cc5f0fe56 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 28 Dec 2017 07:33:43 +0500 Subject: [PATCH 306/362] DOC draft 1.5 release notes --- docs/news.rst | 78 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 62 insertions(+), 16 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 7ecf22470..df0f10a32 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,51 +3,97 @@ Release notes ============= -Scrapy 1.4.1 (2017-XX-XX) +Scrapy 1.5.0 (2017-XX-XX) ------------------------- +Supported Environments +---------------------- + +* Scrapy 1.5 drops support for Python 3.3; +* this release also improves Python 3.x support (especially 3.6); +* PyPy and PyPy3 are now supported officially, by running tests on CI. +* Ubuntu 12.04 and Debian 7.0 are baseline Linux distributions. + New features ~~~~~~~~~~~~ - Support ```` tags in ``Response.follow`` (:issue:`2785`) - Support for ``ptpython`` REPL (:issue:`2654`) +- Google Cloud Storage support for FilesPipeline and ImagesPipeline + (:issue:`2923`). +- New ``--meta`` option of the "scrapy parse" command allows to pass additional + request.meta (:issue:`2883`) - Populate spider variable when using ``shell.inspect_response`` (:issue:`2812`) - Handle HTTP 308 Permanent Redirect (:issue:`2844`) - Add 522 and 524 to ``RETRY_HTTP_CODES`` (:issue:`2851`) - Log versions information at startup (:issue:`2857`) - Add template for a downloader middleware (:issue:`2755`) -- Explicit message for NotImplementedError when parse callback not defined (:issue:`2831`) +- Explicit message for NotImplementedError when parse callback not defined + (:issue:`2831`) +- Connections to proxy servers are reused (:issue:`2743`) +- CrawlerProcess got an option to disable installation of root log handler + (:issue:`2921`) +- LinkExtractor now ignores ``m4v`` extension by default +- ``scrapy.mail.MailSender`` now works in Python 3 (it requires Twisted 17.9.0) +- Better log messages for responses over :setting:`DOWNLOAD_WARNSIZE` and + :setting:`DOWNLOAD_MAXSIZE` limits (:issue:`2927`) +- Show warning when a URL is put to ``Spider.allowed_domains`` instead of + a domain (:issue:`2250`). + Bug fixes ~~~~~~~~~ -- Fix PyPy test failures (:issue:`2793`) -- Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`) -- Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`) -- Add verification to check if Request callback is callable (:issue:`2766`) -- Port ``extras/qpsclient.py`` to Python 3 (:issue:`2849`) -- Use getfullargspec under the scenes for Python 3 to stop DeprecationWarning (:issue:`2862`) -- Update deprecated test aliases (:issue:`2876`) -- Fix ``SitemapSpider`` support for alternate links (:issue:`2853`) - Fix logging of settings overridden by ``custom_settings``; **this is technically backwards-incompatible** because the logger changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please update your log parsers if needed (:issue:`1343`) +- Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`). + **This is technically backwards-incompatible**; override + :setting:`USER_AGENT` if you relied on old value. +- Fix PyPy and PyPy3 test failures, support them officially + (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`, :issue:`3048`) +- Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`) +- Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`) +- Add verification to check if Request callback is callable (:issue:`2766`) +- Port ``extras/qpsclient.py`` to Python 3 (:issue:`2849`) +- Use getfullargspec under the scenes for Python 3 to stop DeprecationWarning + (:issue:`2862`) +- Update deprecated test aliases (:issue:`2876`) +- Fix ``SitemapSpider`` support for alternate links (:issue:`2853`) + Docs ~~~~ -- Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting. (:issue:`2756`) -- Include references to Scrapy subreddit in the docs (:issue:`2762`) -- Use https:// for readthedocs links +- Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY`` + setting. (:issue:`2756`) +- Update Contributing docs, document new support channels + (:issue:`2762`, issue:`3038`) +- Include references to Scrapy subreddit in the docs +- Fix broken links; use https:// for external links + (:issue:`2978`, :issue:`2982`, :issue:`2958`) - Document CloseSpider extension better (:issue:`2759`) -- Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example (:issue:`2781`) -- Spelling mistake and typos (:issue:`2828`, :issue:`2837`, :issue:`#2884`) +- Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example + (:issue:`2781`) +- Spelling mistake and typos + (:issue:`2828`, :issue:`2837`, :issue:`#2884`, :issue:`2924`) - Clarify ``CSVFeedSpider.headers`` documentation (:issue:`2826`) -- Document ``DontCloseSpider`` exception and clarify ``spider_idle`` (:issue:`2791`) +- Document ``DontCloseSpider`` exception and clarify ``spider_idle`` + (:issue:`2791`) - Update "Releases" section in README (:issue:`2764`) - Fix rst syntax in ``DOWNLOAD_FAIL_ON_DATALOSS`` docs (:issue:`2763`) - Small fix in description of startproject arguments (:issue:`2866`) +- Clarify data types in Response.body docs (:issue:`2922`) +- Add a note about ``request.meta['depth']`` to DepthMiddleware docs (:issue:`2374`) +- Add a note about ``request.meta['dont_merge_cookies']`` to CookiesMiddleware + docs (:issue:`2999`) +- Up-to-date example of project structure (:issue:`2964`, :issue:`2976`) +- A better example of ItemExporters usage (:issue:`2989`) +- Document ``from_crawler`` methods for spider and downloader middlewares + (:issue:`3019`) + + Scrapy 1.4.0 (2017-05-18) From d4e5671d07a8dcf18b665ed3ce4136dccae222fb Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 29 Dec 2017 07:06:00 +0500 Subject: [PATCH 307/362] make release docs more readable, add highlights --- docs/news.rst | 45 ++++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index df0f10a32..2283b00ae 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,13 +6,35 @@ Release notes Scrapy 1.5.0 (2017-XX-XX) ------------------------- -Supported Environments ----------------------- +This release brings small new features and improvements across the codebase. +Some highlights: -* Scrapy 1.5 drops support for Python 3.3; -* this release also improves Python 3.x support (especially 3.6); -* PyPy and PyPy3 are now supported officially, by running tests on CI. -* Ubuntu 12.04 and Debian 7.0 are baseline Linux distributions. +* Google Cloud Storage is supported in FilesPipeline and ImagesPipeline. +* Crawling with proxy servers becomes more efficient, as connections + to proxies can be reused now. +* Warnings, exception and logging messages are improved to make debugging + easier. +* ``scrapy parse`` command now allows to set custom request meta via + ``--meta`` argument. +* Compatibility with Python 3.6, PyPy and PyPy3 is improved; + PyPy and PyPy3 are now supported officially, by running tests on CI. +* Better default handling of HTTP 308, 522 and 524 status codes. +* Documentation is improved, as usual. + +Backwards Incompatible Changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Scrapy 1.5 drops support for Python 3.3. +* Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`). + **This is technically backwards-incompatible**; override + :setting:`USER_AGENT` if you relied on old value. +* Logging of settings overridden by ``custom_settings`` is fixed; + **this is technically backwards-incompatible** because the logger + changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``. If you're + parsing Scrapy logs, please update your log parsers (:issue:`1343`). +* LinkExtractor now ignores ``m4v`` extension by default, this is change + in behavior. +* 522 and 524 status codes are added to ``RETRY_HTTP_CODES`` (:issue:`2851`) New features ~~~~~~~~~~~~ @@ -27,20 +49,19 @@ New features - Handle HTTP 308 Permanent Redirect (:issue:`2844`) - Add 522 and 524 to ``RETRY_HTTP_CODES`` (:issue:`2851`) - Log versions information at startup (:issue:`2857`) +- ``scrapy.mail.MailSender`` now works in Python 3 (it requires Twisted 17.9.0) +- Connections to proxy servers are reused (:issue:`2743`) - Add template for a downloader middleware (:issue:`2755`) - Explicit message for NotImplementedError when parse callback not defined (:issue:`2831`) -- Connections to proxy servers are reused (:issue:`2743`) - CrawlerProcess got an option to disable installation of root log handler (:issue:`2921`) - LinkExtractor now ignores ``m4v`` extension by default -- ``scrapy.mail.MailSender`` now works in Python 3 (it requires Twisted 17.9.0) - Better log messages for responses over :setting:`DOWNLOAD_WARNSIZE` and :setting:`DOWNLOAD_MAXSIZE` limits (:issue:`2927`) - Show warning when a URL is put to ``Spider.allowed_domains`` instead of a domain (:issue:`2250`). - Bug fixes ~~~~~~~~~ @@ -52,7 +73,8 @@ Bug fixes **This is technically backwards-incompatible**; override :setting:`USER_AGENT` if you relied on old value. - Fix PyPy and PyPy3 test failures, support them officially - (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`, :issue:`3048`) + (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`, + :issue:`3048`) - Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`) - Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`) - Add verification to check if Request callback is callable (:issue:`2766`) @@ -62,7 +84,6 @@ Bug fixes - Update deprecated test aliases (:issue:`2876`) - Fix ``SitemapSpider`` support for alternate links (:issue:`2853`) - Docs ~~~~ @@ -94,8 +115,6 @@ Docs (:issue:`3019`) - - Scrapy 1.4.0 (2017-05-18) ------------------------- From c107059ef82a4b7b491b23b740b71353f03ab891 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 29 Dec 2017 07:07:43 +0500 Subject: [PATCH 308/362] DOC fix rst syntax --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 2283b00ae..65bccc12d 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -74,7 +74,7 @@ Bug fixes :setting:`USER_AGENT` if you relied on old value. - Fix PyPy and PyPy3 test failures, support them officially (:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`, - :issue:`3048`) + :issue:`3048`) - Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`) - Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`) - Add verification to check if Request callback is callable (:issue:`2766`) From d07fe11981a07e493faf7454db79b98c02a53118 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sat, 30 Dec 2017 02:09:41 +0500 Subject: [PATCH 309/362] set release date --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 65bccc12d..36ead3aba 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,7 +3,7 @@ Release notes ============= -Scrapy 1.5.0 (2017-XX-XX) +Scrapy 1.5.0 (2017-12-29) ------------------------- This release brings small new features and improvements across the codebase. From aa83e159c97b441167d0510064204681bbc93f21 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Sat, 30 Dec 2017 02:09:52 +0500 Subject: [PATCH 310/362] =?UTF-8?q?Bump=20version:=201.4.0=20=E2=86=92=201?= =?UTF-8?q?.5.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 21800f616..6e7be142e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.4.0 +current_version = 1.5.0 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 88c5fb891..bc80560fa 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.4.0 +1.5.0 From a0836b8fd9720a9439cb3b940aca53b6844a094b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Losada?= Date: Mon, 1 Jan 2018 15:59:38 +0000 Subject: [PATCH 311/362] Fix link in news.rst --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 36ead3aba..1629510b2 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -98,7 +98,7 @@ Docs - Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example (:issue:`2781`) - Spelling mistake and typos - (:issue:`2828`, :issue:`2837`, :issue:`#2884`, :issue:`2924`) + (:issue:`2828`, :issue:`2837`, :issue:`2884`, :issue:`2924`) - Clarify ``CSVFeedSpider.headers`` documentation (:issue:`2826`) - Document ``DontCloseSpider`` exception and clarify ``spider_idle`` (:issue:`2791`) From 61c0b1478284b02a4fcfd2cc4931587c348c5d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Losada?= Date: Mon, 1 Jan 2018 16:03:55 +0000 Subject: [PATCH 312/362] Fix typo in comment --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 8e9727093..038db7b47 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -379,7 +379,7 @@ class ScrapyAgent(object): {'size': expected_size, 'warnsize': warnsize, 'request': request}) def _cancel(_): - # Abort connection inmediately. + # Abort connection immediately. txresponse._transport._producer.abortConnection() d = defer.Deferred(_cancel) From 1d1581266c3df99ebf870d6c3e10ac09f2ee3673 Mon Sep 17 00:00:00 2001 From: Yash Sharma Date: Fri, 26 Jan 2018 01:42:17 +0530 Subject: [PATCH 313/362] Changed some documentations (#3089) DOC typo fix in defer_fail docstring --- scrapy/utils/defer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index aa6dcffda..bcf209511 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -11,7 +11,7 @@ def defer_fail(_failure): """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop - It delays by 100ms so reactor has a chance to go trough readers and writers + It delays by 100ms so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ d = defer.Deferred() From ba15b63ed696dbbd6aa6082c035754a15ca8e03c Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 26 Jan 2018 02:11:49 +0500 Subject: [PATCH 314/362] TST fix tests to account for changes in w3lib 1.19 --- tests/test_http_response.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index a36ec3af6..b228344b5 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -272,7 +272,10 @@ class TextResponseTest(BaseResponseTest): headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xef\xbb\xbfWORD\xe3\xab") self.assertEqual(r6.encoding, 'utf-8') - self.assertEqual(r6.text, u'WORD\ufffd\ufffd') + self.assertIn(r6.text, { + u'WORD\ufffd\ufffd', # w3lib < 1.19.0 + u'WORD\ufffd', # w3lib >= 1.19.0 + }) def test_bom_is_removed_from_body(self): # Inferring encoding from body also cache decoded body as sideeffect, From c1916626c1b5c06f1b3c89cd29db1b7b5bc9996c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Losada?= Date: Sat, 27 Jan 2018 21:24:15 +0000 Subject: [PATCH 315/362] Fix OS signal names --- scrapy/utils/ossignal.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index df4eee5ec..f87d5a803 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,17 +1,18 @@ from __future__ import absolute_import +import signal from twisted.internet import reactor -import signal signal_names = {} for signame in dir(signal): - if signame.startswith("SIG"): + if signame.startswith('SIG') and not signame.startswith('SIG_'): signum = getattr(signal, signame) if isinstance(signum, int): signal_names[signum] = signame + def install_shutdown_handlers(function, override_sigint=True): """Install the given function as a signal handler for all common shutdown signals (such as SIGINT, SIGTERM, etc). If override_sigint is ``False`` the @@ -24,5 +25,5 @@ def install_shutdown_handlers(function, override_sigint=True): override_sigint: signal.signal(signal.SIGINT, function) # Catch Ctrl-Break in windows - if hasattr(signal, "SIGBREAK"): + if hasattr(signal, 'SIGBREAK'): signal.signal(signal.SIGBREAK, function) From 6f264ab190882d9bfb375688a7e44d03716242ba Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 30 Jan 2018 05:47:28 +0500 Subject: [PATCH 316/362] more stats for RobotsTxtMiddleware --- scrapy/downloadermiddlewares/robotstxt.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index c3dfa7819..b86c09c14 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -41,10 +41,12 @@ class RobotsTxtMiddleware(object): return d def process_request_2(self, rp, request, spider): - if rp is not None and not rp.can_fetch( - to_native_str(self._useragent), request.url): + if rp is None: + return + if not rp.can_fetch(to_native_str(self._useragent), request.url): logger.debug("Forbidden by robots.txt: %(request)s", {'request': request}, extra={'spider': spider}) + self.crawler.stats.inc_value('robotstxt/forbidden') raise IgnoreRequest() def robot_parser(self, request, spider): @@ -63,6 +65,7 @@ class RobotsTxtMiddleware(object): dfd.addCallback(self._parse_robots, netloc) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) + self.crawler.stats.inc_value('robotstxt/request_count') if isinstance(self._parsers[netloc], Deferred): d = Deferred() @@ -83,11 +86,14 @@ class RobotsTxtMiddleware(object): return failure def _parse_robots(self, response, netloc): + self.crawler.stats.inc_value('robotstxt/response_count') + self.crawler.stats.inc_value( + 'robotstxt/response_status_count/{}'.format(response.status)) rp = robotparser.RobotFileParser(response.url) body = '' if hasattr(response, 'text'): body = response.text - else: # last effort try + else: # last effort try try: body = response.body.decode('utf-8') except UnicodeDecodeError: @@ -95,7 +101,7 @@ class RobotsTxtMiddleware(object): # but keep the lookup cached (in self._parsers) # Running rp.parse() will set rp state from # 'disallow all' to 'allow any'. - pass + self.crawler.stats.inc_value('robotstxt/unicode_error_count') # stdlib's robotparser expects native 'str' ; # with unicode input, non-ASCII encoded bytes decoding fails in Python2 rp.parse(to_native_str(body).splitlines()) @@ -105,6 +111,9 @@ class RobotsTxtMiddleware(object): rp_dfd.callback(rp) def _robots_error(self, failure, netloc): + if failure.type is not IgnoreRequest: + key = 'robotstxt/exception_count/{}'.format(failure.type) + self.crawler.stats.inc_value(key) rp_dfd = self._parsers[netloc] self._parsers[netloc] = None rp_dfd.callback(None) From 4d5e5378bd2ccea2879102614536410d9338b3f1 Mon Sep 17 00:00:00 2001 From: Wenbin Zhang Date: Wed, 7 Feb 2018 10:59:32 -0500 Subject: [PATCH 317/362] Update robotstxt.py Add message to IgnoreRequest exception so that it can be detectedin the errbak method of a spider --- scrapy/downloadermiddlewares/robotstxt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index c3dfa7819..4f3b5d64f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -45,7 +45,7 @@ class RobotsTxtMiddleware(object): to_native_str(self._useragent), request.url): logger.debug("Forbidden by robots.txt: %(request)s", {'request': request}, extra={'spider': spider}) - raise IgnoreRequest() + raise IgnoreRequest("Forbidden by robots.txt") def robot_parser(self, request, spider): url = urlparse_cached(request) From 0c374c00fb7c8d24a60bec8a94f7cbb06accb980 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 8 Feb 2018 05:09:02 +0500 Subject: [PATCH 318/362] use INFO log level to show telnet host/port --- scrapy/extensions/telnet.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 5ca0d19a0..e78afa1fc 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -55,9 +55,9 @@ class TelnetConsole(protocol.ServerFactory): def start_listening(self): self.port = listen_tcp(self.portrange, self.host, self) h = self.port.getHost() - logger.debug("Telnet console listening on %(host)s:%(port)d", - {'host': h.host, 'port': h.port}, - extra={'crawler': self.crawler}) + logger.info("Telnet console listening on %(host)s:%(port)d", + {'host': h.host, 'port': h.port}, + extra={'crawler': self.crawler}) def stop_listening(self): self.port.stopListening() From a56540877c2c24d3ba787cc43ca1f81d91b386fd Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 16 Jan 2018 16:14:35 -0300 Subject: [PATCH 319/362] Do not serialize unpickable objects (py3) --- scrapy/squeues.py | 7 ++++--- tests/test_squeues.py | 9 +++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 21520f454..0b8f6af7d 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -25,9 +25,10 @@ def _serializable_queue(queue_class, serialize, deserialize): def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) - # Python>=3.5 raises AttributeError here while - # Python<=3.4 raises pickle.PicklingError - except (pickle.PicklingError, AttributeError) as e: + # Python<=3.4 raises pickle.PicklingError here while + # Python>=3.5 raises AttributeError and + # Python>=3.6 raises TypeError + except (pickle.PicklingError, AttributeError, TypeError) as e: raise ValueError(str(e)) PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \ diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 3a24348b4..d2f721241 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,3 +1,4 @@ +from sys import version_info import pickle from queuelib.tests import test_queue as t @@ -5,6 +6,7 @@ from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFif from scrapy.item import Item, Field from scrapy.http import Request from scrapy.loader import ItemLoader +from scrapy.selector import Selector class TestItem(Item): name = Field() @@ -17,20 +19,23 @@ class TestLoader(ItemLoader): name_out = staticmethod(_test_procesor) def nonserializable_object_test(self): + q = self.queue() try: pickle.dumps(lambda x: x) except Exception: # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA - q = self.queue() self.assertRaises(ValueError, q.push, lambda x: x) else: # Use a different unpickleable object class A(object): pass a = A() a.__reduce__ = a.__reduce_ex__ = None - q = self.queue() self.assertRaises(ValueError, q.push, a) + if version_info.major == 3 and version_info.minor >= 6: + # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) + sel = Selector(text='

some text

') + self.assertRaises(ValueError, q.push, sel) class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): From e4558cb27e8eeec8432a06124acf8c2569784ccc Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 19 Jan 2018 10:51:30 -0300 Subject: [PATCH 320/362] Update test for unpickable objects --- tests/test_squeues.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index d2f721241..3ded5c027 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,4 +1,3 @@ -from sys import version_info import pickle from queuelib.tests import test_queue as t @@ -32,10 +31,9 @@ def nonserializable_object_test(self): a = A() a.__reduce__ = a.__reduce_ex__ = None self.assertRaises(ValueError, q.push, a) - if version_info.major == 3 and version_info.minor >= 6: - # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) - sel = Selector(text='

some text

') - self.assertRaises(ValueError, q.push, sel) + # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) + sel = Selector(text='

some text

') + self.assertRaises(ValueError, q.push, sel) class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): From 0d87e77afeb506c69f4717744917b95234a86650 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 8 Feb 2018 14:49:26 -0300 Subject: [PATCH 321/362] Bump parsel dependency --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 392f83dd6..2a94d742d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ queuelib six>=1.5.2 PyDispatcher>=2.0.5 service_identity -parsel>=1.1 +parsel>=1.4 diff --git a/setup.py b/setup.py index 06a36e2ba..c37919cda 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ setup( 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', - 'parsel>=1.1', + 'parsel>=1.4', 'PyDispatcher>=2.0.5', 'service_identity', ], From 6edd4114c4e715a3a0c440af455fff089a099620 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 8 Feb 2018 15:47:20 -0300 Subject: [PATCH 322/362] Clarify comment about Pyhton versions --- scrapy/squeues.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 0b8f6af7d..d2074a457 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -25,9 +25,9 @@ def _serializable_queue(queue_class, serialize, deserialize): def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) - # Python<=3.4 raises pickle.PicklingError here while - # Python>=3.5 raises AttributeError and - # Python>=3.6 raises TypeError + # Python <= 3.4 raises pickle.PicklingError here while + # 3.5 <= Python < 3.6 raises AttributeError and + # Python >= 3.6 raises TypeError except (pickle.PicklingError, AttributeError, TypeError) as e: raise ValueError(str(e)) From dc0304fde1b29b4973fabf9d189eb5c4084bf899 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 13 Feb 2018 19:47:41 +0500 Subject: [PATCH 323/362] fix docs building with recent sphinx: don't use deprecated sphinx options and imports --- docs/_ext/scrapydocs.py | 6 +++++- docs/conf.py | 4 ---- docs/requirements.txt | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 83b0d2cc6..192123473 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,6 +1,6 @@ from docutils.parsers.rst.roles import set_classes from docutils import nodes -from sphinx.util.compat import Directive +from docutils.parsers.rst import Directive from sphinx.util.nodes import make_refnode from operator import itemgetter @@ -110,24 +110,28 @@ def setup(app): app.connect('doctree-read', collect_scrapy_settings_refs) app.connect('doctree-resolved', replace_settingslist_nodes) + def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'https://github.com/scrapy/scrapy/blob/master/' + text set_classes(options) node = nodes.reference(rawtext, text, refuri=ref, **options) return [node], [] + def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'https://github.com/scrapy/scrapy/issues/' + text set_classes(options) node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options) return [node], [] + def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'https://github.com/scrapy/scrapy/commit/' + text set_classes(options) node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options) return [node], [] + def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]): ref = 'http://hg.scrapy.org/scrapy/changeset/' + text set_classes(options) diff --git a/docs/conf.py b/docs/conf.py index 007dc2788..594740f39 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -144,10 +144,6 @@ html_static_path = ['_static'] # using the given strftime format. html_last_updated_fmt = '%b %d, %Y' -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -html_use_smartypants = True - # Custom sidebar templates, maps document names to template names. #html_sidebars = {} diff --git a/docs/requirements.txt b/docs/requirements.txt index d3dcb97be..8e7611d21 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -Sphinx>=1.3 +Sphinx>=1.6 sphinx_rtd_theme \ No newline at end of file From 6954da136604ef2a5bf37f8d792e0de6ec90469a Mon Sep 17 00:00:00 2001 From: Anjali Jain Date: Thu, 15 Feb 2018 23:27:40 +0530 Subject: [PATCH 324/362] Updated contributing.rst Rectified grammatical errors --- docs/contributing.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 9a02634cb..44068baa9 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -6,7 +6,7 @@ Contributing to Scrapy .. important:: - Double check you are reading the most recent version of this document at + Double check that you are reading the most recent version of this document at https://doc.scrapy.org/en/master/contributing.html There are many ways to contribute to Scrapy. Here are some of them: @@ -18,7 +18,7 @@ There are many ways to contribute to Scrapy. Here are some of them: * Report bugs and request features in the `issue tracker`_, trying to follow the guidelines detailed in `Reporting bugs`_ below. -* Submit patches for new functionality and/or bug fixes. Please read +* Submit patches for new functionalities and/or bug fixes. Please read :ref:`writing-patches` and `Submitting patches`_ below for details on how to write and submit a patch. @@ -80,8 +80,8 @@ guidelines when reporting a new bug. Writing patches =============== -The better written a patch is, the higher chance that it'll get accepted and -the sooner that will be merged. +The better written a patch is, higher is the chance that it'll get accepted and +sooner it will be merged. Well-written patches should: From bbc2a3569f153b20bf306375883688d99b565cb3 Mon Sep 17 00:00:00 2001 From: Anjali Jain Date: Fri, 16 Feb 2018 23:33:10 +0530 Subject: [PATCH 325/362] further edited --- docs/contributing.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 44068baa9..f4f9e393f 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -80,8 +80,7 @@ guidelines when reporting a new bug. Writing patches =============== -The better written a patch is, higher is the chance that it'll get accepted and -sooner it will be merged. +The better a patch is written, the higher the chances that it'll get accepted and the sooner it will be merged. Well-written patches should: From acd2b8d43b5ebec7ffd364b6f335427041a0b98d Mon Sep 17 00:00:00 2001 From: NewUserHa <32261870+NewUserHa@users.noreply.github.com> Date: Thu, 22 Feb 2018 06:37:26 +0800 Subject: [PATCH 326/362] [MRG+1] Fix part of issue #3128 - None should not be a valid type for 'url' in Response.follow (#3131) * fix one issue of issue#3128 because @kmike posted: 'If url is '', Scrapy should follow the same page, this is an intended behavior.' * fix one issue of issue#3128 because @kmike posted: 'If url is '', Scrapy should follow the same page, this is an intended behavior.' --- scrapy/http/response/__init__.py | 2 ++ tests/test_http_response.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 434d87eab..1974259b5 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -120,6 +120,8 @@ class Response(object_ref): """ if isinstance(url, Link): url = url.url + elif url is None: + raise ValueError("url can't be None") url = self.urljoin(url) return Request(url, callback, method=method, diff --git a/tests/test_http_response.py b/tests/test_http_response.py index b228344b5..820758dc9 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -155,6 +155,10 @@ class BaseResponseTest(unittest.TestCase): self._assert_followed_url(Link('http://example.com/foo'), 'http://example.com/foo') + def test_follow_None_url(self): + r = self.response_class("http://example.com") + self.assertRaises(ValueError, r.follow, None) + def test_follow_whitespace_url(self): self._assert_followed_url('foo ', 'http://example.com/foo%20') From d5b7ebcfdcfd40d29990712b587893fcc6e84ce8 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 3 Mar 2018 18:17:49 +0530 Subject: [PATCH 327/362] Fixed bug FormRequest.from_response() clickdata ignores input[type=image] --- scrapy/http/request/form.py | 10 ++++++---- tests/test_http_request.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d9d178a3e..184ee2599 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -170,10 +170,12 @@ def _get_clickable(clickdata, form): """ clickables = [ el for el in form.xpath( - 'descendant::*[(self::input or self::button)' - ' and re:test(@type, "^submit$", "i")]' - '|descendant::button[not(@type)]', - namespaces={"re": "http://exslt.org/regular-expressions"}) + 'descendant::*[(self::input or self::button)' + ' and re:test(@type, "^submit$", "i")]' + '|descendant::*[(self::input or self::button)' + ' and re:test(@type, "^image$", "i")]' + '|descendant::button[not(@type)]', + namespaces={"re": "http://exslt.org/regular-expressions"}) ] if not clickables: return diff --git a/tests/test_http_request.py b/tests/test_http_request.py index fca8ff411..73a74cd5d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -532,6 +532,16 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) self.assertEqual(fs, {b'i1': [b'i1v']}) + + def test_from_response_clickdata_does_not_ignore_image(self): + response = _buildresponse( + """
+ + +
""") + req = self.request_class.from_response(response, dont_click=True) + fs = _qs(req) + self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) def test_from_response_dont_submit_reset_as_input(self): response = _buildresponse( From 65744c2199fc6a5bccfa11eec40a867c1401aee9 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 3 Mar 2018 20:07:50 +0530 Subject: [PATCH 328/362] Corrected Test --- tests/test_http_request.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 73a74cd5d..a042f03b6 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -532,16 +532,6 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) self.assertEqual(fs, {b'i1': [b'i1v']}) - - def test_from_response_clickdata_does_not_ignore_image(self): - response = _buildresponse( - """
- - -
""") - req = self.request_class.from_response(response, dont_click=True) - fs = _qs(req) - self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) def test_from_response_dont_submit_reset_as_input(self): response = _buildresponse( @@ -554,6 +544,16 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) + + def test_from_response_clickdata_does_not_ignore_image(self): + response = _buildresponse( + """
+ + +
""") + req = self.request_class.from_response(response) + fs = _qs(req) + self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) def test_from_response_multiple_clickdata(self): response = _buildresponse( From 13a74d77e2e4d1719bf984d99ca2ae6874752e41 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Mon, 12 Mar 2018 22:25:19 +0800 Subject: [PATCH 329/362] catch CertificateError in tls verification --- scrapy/core/downloader/tls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 498e3d60f..e1c4f4908 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -40,6 +40,7 @@ if twisted_version >= (14, 0, 0): from twisted.internet._sslverify import (ClientTLSOptions, verifyHostname, VerificationError) + from service_identity.exceptions import CertificateError if twisted_version < (17, 0, 0): from twisted.internet._sslverify import _maybeSetHostNameIndication @@ -65,7 +66,7 @@ if twisted_version >= (14, 0, 0): elif where & SSL_CB_HANDSHAKE_DONE: try: verifyHostname(connection, self._hostnameASCII) - except VerificationError as e: + except (CertificateError, VerificationError) as e: logger.warning( 'Remote certificate is not valid for hostname "{}"; {}'.format( self._hostnameASCII, e)) From e487100987496474de4d2b5fc509ff47d01cd0b6 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Tue, 13 Mar 2018 08:59:03 +0800 Subject: [PATCH 330/362] add a test case --- tests/keys/localhost-ip.gen.README | 21 ++++++++++++ tests/keys/localhost.crt | 36 ++++++++++----------- tests/keys/localhost.ip.crt | 20 ++++++++++++ tests/keys/localhost.ip.key | 28 ++++++++++++++++ tests/keys/localhost.key | 52 +++++++++++++++--------------- tests/test_downloader_handlers.py | 9 ++++++ 6 files changed, 122 insertions(+), 44 deletions(-) create mode 100644 tests/keys/localhost-ip.gen.README create mode 100644 tests/keys/localhost.ip.crt create mode 100644 tests/keys/localhost.ip.key diff --git a/tests/keys/localhost-ip.gen.README b/tests/keys/localhost-ip.gen.README new file mode 100644 index 000000000..8e94e1217 --- /dev/null +++ b/tests/keys/localhost-ip.gen.README @@ -0,0 +1,21 @@ +$ openssl req -x509 -sha256 -nodes -newkey rsa:2048 -days 365 -keyout localhost.key -out localhost.crt +Generating a 2048 bit RSA private key +...................................................................................................+++ +.....+++ +writing new private key to 'localhost.key' +----- +You are about to be asked to enter information that will be incorporated +into your certificate request. +What you are about to enter is what is called a Distinguished Name or a DN. +There are quite a few fields but you can leave some blank +For some fields there will be a default value, +If you enter '.', the field will be left blank. +----- +Country Name (2 letter code) [AU]:IE +State or Province Name (full name) [Some-State]:. +Locality Name (eg, city) []:. +Organization Name (eg, company) [Internet Widgits Pty Ltd]:Scrapy +Organizational Unit Name (eg, section) []:. +Common Name (e.g. server FQDN or YOUR name) []:127.0.0.1 +Email Address []:. + diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt index 13c5b5bd6..48d7bd9a3 100644 --- a/tests/keys/localhost.crt +++ b/tests/keys/localhost.crt @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV -BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x -NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD -VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K -o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y -njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY -ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/ -8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52 -gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O -BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j -xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES -9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i -jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+ -TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own -tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM -am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U -Mjybr/hQK3mnVdE= +MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x +ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL +JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev +y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV +ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR +v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4 +JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O +BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ +dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr +idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP +8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X +kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ +FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb +smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO +zh6AcsnjkLumOkM= -----END CERTIFICATE----- diff --git a/tests/keys/localhost.ip.crt b/tests/keys/localhost.ip.crt new file mode 100644 index 000000000..48d7bd9a3 --- /dev/null +++ b/tests/keys/localhost.ip.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x +ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL +JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev +y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV +ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR +v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4 +JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O +BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ +dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr +idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP +8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X +kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ +FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb +smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO +zh6AcsnjkLumOkM= +-----END CERTIFICATE----- diff --git a/tests/keys/localhost.ip.key b/tests/keys/localhost.ip.key new file mode 100644 index 000000000..1e12c1255 --- /dev/null +++ b/tests/keys/localhost.ip.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo ++g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC +T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb +rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR +3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi +JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj +uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES +FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc +BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud +JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD +bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT +ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx +KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N +yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA +hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+ +32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie +PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY +r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2 +X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz +ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ +Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ +g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5 +WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+ +inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u +BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz +m1Q+9xJdtRxlYc1YGTK1YGCq +-----END PRIVATE KEY----- diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key index da975e6d3..1e12c1255 100644 --- a/tests/keys/localhost.key +++ b/tests/keys/localhost.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5 -ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4 -JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz -vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2 -RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY -b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/ -XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0 -gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk -i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu -Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss -ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4 -hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD -7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa -EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf -uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG -HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR -UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO -j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri -fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ -6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI -xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g -DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f -npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM -8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6 -J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj -Z+pMCFLbwAbFZ9Qb/NFlUv0= +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo ++g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC +T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb +rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR +3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi +JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj +uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES +FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc +BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud +JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD +bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT +ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx +KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N +yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA +hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+ +32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie +PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY +r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2 +X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz +ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ +Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ +g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5 +WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+ +inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u +BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz +m1Q+9xJdtRxlYc1YGTK1YGCq -----END PRIVATE KEY----- diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index bd2c86292..ceb03f945 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -505,6 +505,15 @@ class Https11InvalidDNSId(Https11TestCase): super(Https11InvalidDNSId, self).setUp() self.host = '127.0.0.1' +class Https11InvalidDNSPattern(Https11TestCase): + """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" + + keyfile = 'keys/localhost.ip.key' + certfile = 'keys/localhost.ip.crt' + + def setUp(self): + super(Https11InvalidDNSPattern, self).setUp() + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" From d9e6c73fb3ef787e39474bda5f008b309b65c65b Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Tue, 13 Mar 2018 13:05:37 +0800 Subject: [PATCH 331/362] revert wrong changes --- tests/keys/localhost.crt | 36 ++++++++++++++-------------- tests/keys/localhost.key | 52 ++++++++++++++++++++-------------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/tests/keys/localhost.crt b/tests/keys/localhost.crt index 48d7bd9a3..13c5b5bd6 100644 --- a/tests/keys/localhost.crt +++ b/tests/keys/localhost.crt @@ -1,20 +1,20 @@ -----BEGIN CERTIFICATE----- -MIIDNzCCAh+gAwIBAgIJAKAIhM4nA8W7MA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV -BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTAeFw0x -ODAzMTIxNDMyMjlaFw0xOTAzMTIxNDMyMjlaMDIxCzAJBgNVBAYTAklFMQ8wDQYD -VQQKDAZTY3JhcHkxEjAQBgNVBAMMCTEyNy4wLjAuMTCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBAK7Vzr+zdsbAEej6D8XFBS5frHnfmqSivQS/zrRZcSVL -JgPwHJSRMyVCNvlpRV4ulu7I6zTY0ItzeAJPiH/euSokM8AkM87y9GAugljVtuev -y0uKLUfznPvPZxfYzaB7lyQtU9E6AF8Amtuta8eb7rdqsuqjRopKp3pIheBAfvjV -ewkMlxz3xcKZHs8T3UWdceWftLEZJSi13FHe/uoohRBiXVn/6DvycBjk1TC+zNpR -v8mSm+uqcYoG8/CFZ/r1T2EveBH4jZjNReIlM9zFwVHjtjAdunSdMLVY59kBGNE4 -JqxjJ021W2XqoW4VFf6XrIdg8ai4NxHDpWO4blOoMbcCAwEAAaNQME4wHQYDVR0O -BBYEFBZWEo9+kkTjdGxJdvRNGyhpWfjMMB8GA1UdIwQYMBaAFBZWEo9+kkTjdGxJ -dvRNGyhpWfjMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGjMcuVr -idLmbuu/Krxmqnebt0zPLgJXg1ACUEto7110mmEK3jsZg/brdLf74PP+FUa6B/ZP -8+FJCgF1KZLc3tS9w2OVRSdz+uZ2WYgN6R7uJiVs77BiD6TR6wRrEicRsS6Cq90X -kNVhqExG4cDr8wGLiCGNfVfFwea7wGhF2zCohF82u1mAgqR/1obas0ils5fh+soJ -FmTd5A9vCbRpZRXost9J7Z4LCj86MYATgyH9bZp7aN6NJ2nI4uKgeafDFT83c5Vb -smQ/R0HeP5oylIhpmWWliNjT+XPONPIPDWgQgeFBBofX/vuv82KXz1ZBYfqpArgO -zh6AcsnjkLumOkM= +MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV +BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x +NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD +VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K +o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y +njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY +ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/ +8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52 +gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O +BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j +xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES +9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i +jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+ +TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own +tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM +am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U +Mjybr/hQK3mnVdE= -----END CERTIFICATE----- diff --git a/tests/keys/localhost.key b/tests/keys/localhost.key index 1e12c1255..da975e6d3 100644 --- a/tests/keys/localhost.key +++ b/tests/keys/localhost.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCu1c6/s3bGwBHo -+g/FxQUuX6x535qkor0Ev860WXElSyYD8ByUkTMlQjb5aUVeLpbuyOs02NCLc3gC -T4h/3rkqJDPAJDPO8vRgLoJY1bbnr8tLii1H85z7z2cX2M2ge5ckLVPROgBfAJrb -rWvHm+63arLqo0aKSqd6SIXgQH741XsJDJcc98XCmR7PE91FnXHln7SxGSUotdxR -3v7qKIUQYl1Z/+g78nAY5NUwvszaUb/JkpvrqnGKBvPwhWf69U9hL3gR+I2YzUXi -JTPcxcFR47YwHbp0nTC1WOfZARjROCasYydNtVtl6qFuFRX+l6yHYPGouDcRw6Vj -uG5TqDG3AgMBAAECggEBAKaLO0g3j3SicC0rT60IEfhr4OOzkh80erQ0dpYsAXES -FeN4bfFEI6FhYvbRRegCn3pVYGDWDEpasz4YPyH3qxEurTFiCwwfOZUJmNdAtdwc -BJ8vwBSjRq5EkqMPvkkakg4/M3HCO6pD7EBJAbuCmbKU7FxBLqf7l3AP9594MLud -JE1zkioK8tz6auBq4qLwDUNJhqv7eug1CKEpfArA9ZqW3orWg21+Octac8R82ZyD -bt+Veh0vWd16MkcSX574vydqYzNiseY70yNjBRxHLD+/HA8BvWn7M6d0ULuEN1UT -ojm+NAMc65ms3MkXksdUeDQ3eFIF9M4+/rTRU8gHeAECgYEA487ERT3/qEDMezYx -KcUkLE2VwqqnW0+Sfd6fzOG+VGqeYgHG/d9sjo1RsJR/D/ZgzO3oeJ4lgov3HN5N -yfPIGyJfYd7p9WWml4AiWvj3YVg5V4vmwnDs7LBxHU60bLClgvMQx4iSZ4q4QrXA -hRLBDrJuNGvuLUqFb6jar8BtVbcCgYEAxHjORgNxsBfzuAs0ZfvVyTYai4f+92U+ -32tPxghpI4gHnQnz7MbUccJGy+SR23N8DLNJv8K+LbVm7UNIdsy6d5b9vazkYIie -PyS3ynRO3vgIL3NbMC2cc+uc2dL2n/FnMA8nrdZMTgXukmnCn8tzSLphoZBu7SaY -r9938XE8BAECgYEAmuXzCun3Nl6pK3ZTw4Uq7Xzrwevr0+itQSzpF5S/qAK/IwD2 -X5VV6TAqRZkTNLVgaLe0BJ/z/WpSYqy90/4RKHIczR2Xk6bEuesEcTssamJkyyRz -ie7jCqWGpFjp0aXjRMElvacddY4bcDDJcTKpVub4jGh/EQjE5oG4AR0kus0CgYBZ -Eed56C/PRFySUEoV/gCisquAHExjvfut8Al/XurDV/UTpaJ28oD3fbr4zoutcIKJ -g3JoxBHRyQ57e+hLK29RrhsktU/nz6fmOnA0EVx8SvfzAxoREmx+RQ+b1L9ILXm5 -WPWFIsT/DkNlDxtTtDl0fEKsqz0OuFO6T9YhmFM8AQKBgCFn6FV8AdzLBtdKrPT+ -inQASBr264pb5lp7g9JdBmaQZ3McrQ35VOA3ZfhyTAMhYtY1wk0xp8+fW1bV325u -BiLdJ/gAocPBRlw7rS0rq1+U1+zAQCgxutrm2aRQd1qEUrCRvCtCyIeuUntshHAz -m1Q+9xJdtRxlYc1YGTK1YGCq +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5 +ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4 +JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz +vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2 +RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY +b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/ +XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0 +gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk +i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu +Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss +ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4 +hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD +7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa +EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf +uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG +HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR +UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO +j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri +fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ +6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI +xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g +DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f +npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM +8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6 +J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj +Z+pMCFLbwAbFZ9Qb/NFlUv0= -----END PRIVATE KEY----- From 6a7cdf9a6c4162cf4dd91721d28974c80e0f68cd Mon Sep 17 00:00:00 2001 From: siulkilulki Date: Tue, 13 Mar 2018 08:35:27 +0100 Subject: [PATCH 332/362] [MRG+1] Add 'flv' to ignored video extensions. (#3165) --- scrapy/linkextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 2d7115cc5..c3c79cf25 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -28,7 +28,7 @@ IGNORED_EXTENSIONS = [ # video '3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv', - 'm4a', 'm4v', + 'm4a', 'm4v', 'flv', # office suites 'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg', From 1a2f0193a30c1aceb4743e33a90b3c264e0b09c4 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Tue, 13 Mar 2018 19:14:52 +0800 Subject: [PATCH 333/362] fix tests on jessie --- scrapy/core/downloader/tls.py | 11 +++++++++-- tests/test_downloader_handlers.py | 4 ++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index e1c4f4908..c97c6a9a9 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -40,7 +40,14 @@ if twisted_version >= (14, 0, 0): from twisted.internet._sslverify import (ClientTLSOptions, verifyHostname, VerificationError) - from service_identity.exceptions import CertificateError + try: + # XXX: this import would fail on Debian jessie with system installed + # service_identity library, due to lack of cryptography.x509 dependency + # See https://github.com/pyca/service_identity/issues/21 + from service_identity.exceptions import CertificateError + verification_errors = (CertificateError, VerificationError) + except ImportError: + verification_errors = VerificationError if twisted_version < (17, 0, 0): from twisted.internet._sslverify import _maybeSetHostNameIndication @@ -66,7 +73,7 @@ if twisted_version >= (14, 0, 0): elif where & SSL_CB_HANDSHAKE_DONE: try: verifyHostname(connection, self._hostnameASCII) - except (CertificateError, VerificationError) as e: + except verification_errors as e: logger.warning( 'Remote certificate is not valid for hostname "{}"; {}'.format( self._hostnameASCII, e)) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index ceb03f945..b34faa7e7 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -512,6 +512,10 @@ class Https11InvalidDNSPattern(Https11TestCase): certfile = 'keys/localhost.ip.crt' def setUp(self): + try: + from service_identity.exceptions import CertificateError + except ImportError: + raise unittest.SkipTest("cryptography lib is too old") super(Https11InvalidDNSPattern, self).setUp() From 2c58da19a6f85f4532d80d3a941cede8f9d0bab8 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Wed, 14 Mar 2018 09:27:59 +0800 Subject: [PATCH 334/362] update docstring of ScrapyClientTLSOptions --- scrapy/core/downloader/tls.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index c97c6a9a9..df8051182 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -63,8 +63,9 @@ if twisted_version >= (14, 0, 0): (for genuinely invalid certificates or bugs in verification code). Same as Twisted's private _sslverify.ClientTLSOptions, - except that VerificationError and ValueError exceptions are caught, - so that the connection is not closed, only logging warnings. + except that VerificationError, CertificateError and ValueError + exceptions are caught, so that the connection is not closed, only + logging warnings. """ def _identityVerifyingInfoCallback(self, connection, where, ret): From ff5f717f7a2aaf0a1a1101019485201427edd536 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 17 Mar 2018 18:17:48 +0530 Subject: [PATCH 335/362] Fixed formatting issues --- scrapy/http/request/form.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 184ee2599..22846ad77 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -168,15 +168,17 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ + print("form =", form.__dict__) clickables = [ el for el in form.xpath( - 'descendant::*[(self::input or self::button)' - ' and re:test(@type, "^submit$", "i")]' - '|descendant::*[(self::input or self::button)' - ' and re:test(@type, "^image$", "i")]' - '|descendant::button[not(@type)]', - namespaces={"re": "http://exslt.org/regular-expressions"}) + 'descendant::*[(self::input or self::button)' + ' and re:test(@type, "^submit$", "i")]' + '|descendant::*[(self::input or self::button)' + ' and re:test(@type, "^image$", "i")]' + '|descendant::button[not(@type)]', + namespaces={"re": "http://exslt.org/regular-expressions"}) ] + print("clickables =", clickables) if not clickables: return From e25e2afe174bbe70ecbf88ea684937890e8ba4d5 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Sat, 17 Mar 2018 18:20:14 +0530 Subject: [PATCH 336/362] Removed unnecessary print statements --- scrapy/http/request/form.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 22846ad77..238dd44b3 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -168,7 +168,6 @@ def _get_clickable(clickdata, form): if the latter is given. If not, it returns the first clickable element found """ - print("form =", form.__dict__) clickables = [ el for el in form.xpath( 'descendant::*[(self::input or self::button)' @@ -178,7 +177,6 @@ def _get_clickable(clickdata, form): '|descendant::button[not(@type)]', namespaces={"re": "http://exslt.org/regular-expressions"}) ] - print("clickables =", clickables) if not clickables: return From a5acc9373f8735e27c22de6fbe345fbed8f268c1 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Mon, 19 Mar 2018 18:19:39 +0530 Subject: [PATCH 337/362] Resolving Comments --- scrapy/http/request/form.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 238dd44b3..d033a830e 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -170,11 +170,8 @@ def _get_clickable(clickdata, form): """ clickables = [ el for el in form.xpath( - 'descendant::*[(self::input or self::button)' - ' and re:test(@type, "^submit$", "i")]' - '|descendant::*[(self::input or self::button)' - ' and re:test(@type, "^image$", "i")]' - '|descendant::button[not(@type)]', + 'descendant::input[re.test(@type, "^(submit|image)$", "i")]' + '|descendant::button[not(@type) or re.test(@type, "^submit$", "i")]', namespaces={"re": "http://exslt.org/regular-expressions"}) ] if not clickables: From dd064413a46356940151ac3f9ccd8a45bca2cbd8 Mon Sep 17 00:00:00 2001 From: Viral Mehta Date: Mon, 19 Mar 2018 19:28:41 +0530 Subject: [PATCH 338/362] corrected syntax error in XPath --- scrapy/http/request/form.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d033a830e..95b38e990 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -170,8 +170,8 @@ def _get_clickable(clickdata, form): """ clickables = [ el for el in form.xpath( - 'descendant::input[re.test(@type, "^(submit|image)$", "i")]' - '|descendant::button[not(@type) or re.test(@type, "^submit$", "i")]', + 'descendant::input[re:test(@type, "^(submit|image)$", "i")]' + '|descendant::button[not(@type) or re:test(@type, "^submit$", "i")]', namespaces={"re": "http://exslt.org/regular-expressions"}) ] if not clickables: From c6d20bdd826070c6c808421cd6331b5d208f8aa7 Mon Sep 17 00:00:00 2001 From: Steven Almeroth Date: Tue, 27 Mar 2018 16:21:07 -0400 Subject: [PATCH 339/362] Doc: update wording for COOKIES_ENABLED --- docs/topics/downloader-middleware.rst | 20 +++++++++----------- docs/topics/request-response.rst | 4 +++- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 863620900..dfe4c13b4 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -158,13 +158,13 @@ more of the following methods: :type spider: :class:`~scrapy.spiders.Spider` object .. method:: from_crawler(cls, crawler) - + If present, this classmethod is called to create a middleware instance from a :class:`~scrapy.crawler.Crawler`. It must return a new instance of the middleware. Crawler object provides access to all Scrapy core components like settings and signals; it is a way for middleware to access them and hook its functionality into Scrapy. - + :param crawler: crawler that uses this middleware :type crawler: :class:`~scrapy.crawler.Crawler` object @@ -237,16 +237,14 @@ Default: ``True`` Whether to enable the cookies middleware. If disabled, no cookies will be sent to web servers. -Notice that if the :class:`~scrapy.http.Request` -has ``meta['dont_merge_cookies']`` evaluated to ``True``. -despite the value of :setting:`COOKIES_ENABLED` the cookies will **not** be -sent to web servers and received cookies in -:class:`~scrapy.http.Response` will **not** be merged with the existing -cookies. - -For more detailed information see the ``cookies`` parameter in -:class:`~scrapy.http.Request` +Notice that despite the value of :setting:`COOKIES_ENABLED` setting if +``Request.``:reqmeta:`meta['dont_merge_cookies'] ` +evaluates to ``True`` the request cookies will **not** be sent to the +web server and received cookies in :class:`~scrapy.http.Response` will +**not** be merged with the existing cookies. +For more detailed information see the ``cookies`` parameter in +:class:`~scrapy.http.Request`. .. setting:: COOKIES_DEBUG diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 121abe6b5..e29914dbf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -80,6 +80,8 @@ Request objects attributes of the cookie. This is only useful if the cookies are saved for later requests. + .. reqmeta:: dont_merge_cookies + When some site returns cookies (in a response) those are stored in the cookies for that domain and will be sent again in future requests. That's the typical behaviour of any regular web browser. However, if, for some @@ -294,7 +296,7 @@ Those are: * :reqmeta:`dont_retry` * :reqmeta:`handle_httpstatus_list` * :reqmeta:`handle_httpstatus_all` -* ``dont_merge_cookies`` (see ``cookies`` parameter of :class:`Request` constructor) +* :reqmeta:`dont_merge_cookies` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` * :reqmeta:`redirect_urls` From 8e8994c6b55fa8e975ce30b25ae326f829a58aed Mon Sep 17 00:00:00 2001 From: rhoboro Date: Mon, 2 Apr 2018 15:36:47 +0900 Subject: [PATCH 340/362] add acl support for gcs --- scrapy/pipelines/files.py | 6 +++++- scrapy/pipelines/images.py | 1 + scrapy/settings/default_settings.py | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index ab18a727d..af1d5488a 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -207,6 +207,8 @@ class GCSFilesStore(object): GCS_PROJECT_ID = None CACHE_CONTROL = 'max-age=172800' + POLICY = 'projectPrivate' # Overriden from settings.FILES_STORE_GCS_ACL in + # FilesPipeline.from_settings. def __init__(self, uri): from google.cloud import storage @@ -239,7 +241,8 @@ class GCSFilesStore(object): return threads.deferToThread( blob.upload_from_string, data=buf.getvalue(), - content_type=self._get_content_type(headers) + content_type=self._get_content_type(headers), + predefined_acl=self.POLICY ) @@ -314,6 +317,7 @@ class FilesPipeline(MediaPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] store_uri = settings['FILES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index c5fc12afe..5cdddce49 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -93,6 +93,7 @@ class ImagesPipeline(FilesPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] + gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ead511473..7916b9704 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,6 +159,7 @@ FEED_EXPORTERS_BASE = { FEED_EXPORT_INDENT = 0 FILES_STORE_S3_ACL = 'private' +FILES_STORE_GCS_ACL = 'projectPrivate' FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' @@ -181,6 +182,7 @@ HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' +IMAGES_STORE_GCS_ACL = 'projectPrivate' ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' From 5254ac393bdf712db698bfa41caf5fb2e682f883 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Tue, 3 Apr 2018 18:00:08 +0900 Subject: [PATCH 341/362] added test for gcs policy --- scrapy/utils/test.py | 3 ++- tests/test_pipeline_files.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 60b931f48..4b935c51b 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -57,8 +57,9 @@ def get_gcs_content_and_delete(bucket, path): bucket = client.get_bucket(bucket) blob = bucket.get_blob(path) content = blob.download_as_string() + acl = list(blob.acl) # loads acl before it will be deleted bucket.delete_blob(path) - return content, blob + return content, acl, blob def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index c761bd606..728a74803 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -388,17 +388,20 @@ class TestGCSFilesStore(unittest.TestCase): meta = {'foo': 'bar'} path = 'full/filename' store = GCSFilesStore(uri) + store.POLICY = 'authenticatedRead' + expected_policy = {'role': 'READER', 'entity': 'allAuthenticatedUsers'} yield store.persist_file(path, buf, info=None, meta=meta, headers=None) s = yield store.stat_file(path, info=None) self.assertIn('last_modified', s) self.assertIn('checksum', s) self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==') u = urlparse(uri) - content, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) + content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) self.assertEqual(content, data) self.assertEqual(blob.metadata, {'foo': 'bar'}) self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) self.assertEqual(blob.content_type, 'application/octet-stream') + self.assertIn(expected_policy, acl) class ItemWithFiles(Item): From 74a9c65290888b3ce712c3cfa7132f91ffa3c576 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Tue, 3 Apr 2018 18:20:37 +0900 Subject: [PATCH 342/362] update docs for support gcs acl --- docs/topics/media-pipeline.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 41beebe98..284ec1a25 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -189,6 +189,8 @@ Google Cloud Storage --------------------- .. setting:: GCS_PROJECT_ID +.. setting:: FILES_STORE_GCS_ACL +.. setting:: IMAGES_STORE_GCS_ACL :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ ) @@ -204,6 +206,18 @@ For information about authentication, see this `documentation`_. .. _documentation: https://cloud.google.com/docs/authentication/production +You can modify the Access Control List (ACL) policy used for the stored files, +which is defined by the :setting:`FILES_STORE_GCS_ACL` and +:setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to +``projectPrivate``. To make the files publicly available use the ``publicRead`` +policy:: + + IMAGES_STORE_GCS_ACL = 'publicRead' + +For more information, see `Predefined ACLs`_ in the Google Cloud Platform Developer Guide. + +.. _Predefined ACLs: https://cloud.google.com/storage/docs/access-control/lists#predefined-acl + Usage example ============= From cb76b88331e1e0cff30de9a6961de3e28e94ff44 Mon Sep 17 00:00:00 2001 From: grammy-jiang Date: Wed, 4 Apr 2018 05:56:05 -0400 Subject: [PATCH 343/362] fix a mistake in topic spider-middleware.rst --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index c297ed556..1d451af21 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -116,7 +116,7 @@ following methods: method (from other spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.http.Response`, dict or + iterable of :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` objects. If it returns ``None``, Scrapy will continue processing this exception, From 464973489e9f457570d7dd1a2e4aab8c4c1778fd Mon Sep 17 00:00:00 2001 From: rhoboro Date: Fri, 13 Apr 2018 12:06:39 +0900 Subject: [PATCH 344/362] Using bucket's default object ACL --- docs/topics/media-pipeline.rst | 3 ++- scrapy/pipelines/files.py | 6 ++++-- scrapy/settings/default_settings.py | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 284ec1a25..0872ac0cd 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -209,7 +209,8 @@ For information about authentication, see this `documentation`_. You can modify the Access Control List (ACL) policy used for the stored files, which is defined by the :setting:`FILES_STORE_GCS_ACL` and :setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to -``projectPrivate``. To make the files publicly available use the ``publicRead`` +None which means that Cloud Storage applies the bucket's default object ACL to the object. +To make the files publicly available use the ``publicRead`` policy:: IMAGES_STORE_GCS_ACL = 'publicRead' diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index af1d5488a..8ea70e5d1 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -207,8 +207,10 @@ class GCSFilesStore(object): GCS_PROJECT_ID = None CACHE_CONTROL = 'max-age=172800' - POLICY = 'projectPrivate' # Overriden from settings.FILES_STORE_GCS_ACL in - # FilesPipeline.from_settings. + + # The bucket's default object ACL will be applied to the object. + # Overriden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings. + POLICY = None def __init__(self, uri): from google.cloud import storage diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7916b9704..d7ca8a835 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,7 +159,7 @@ FEED_EXPORTERS_BASE = { FEED_EXPORT_INDENT = 0 FILES_STORE_S3_ACL = 'private' -FILES_STORE_GCS_ACL = 'projectPrivate' +FILES_STORE_GCS_ACL = None FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' @@ -182,7 +182,7 @@ HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' -IMAGES_STORE_GCS_ACL = 'projectPrivate' +IMAGES_STORE_GCS_ACL = None ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' From 560ee623fd84c3db986fccacaef7e9a31a8ea02c Mon Sep 17 00:00:00 2001 From: rhoboro Date: Fri, 13 Apr 2018 19:00:27 +0900 Subject: [PATCH 345/362] set defalut value "" to FILES_STORE_GCS_ACL --- scrapy/pipelines/files.py | 2 +- scrapy/pipelines/images.py | 2 +- scrapy/settings/default_settings.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8ea70e5d1..510cc23c7 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -319,7 +319,7 @@ class FilesPipeline(MediaPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] - gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] + gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None store_uri = settings['FILES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 5cdddce49..95323c613 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -93,7 +93,7 @@ class ImagesPipeline(FilesPipeline): gcs_store = cls.STORE_SCHEMES['gs'] gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID'] - gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] + gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] or None store_uri = settings['IMAGES_STORE'] return cls(store_uri, settings=settings) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d7ca8a835..36e17ef6b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -159,7 +159,7 @@ FEED_EXPORTERS_BASE = { FEED_EXPORT_INDENT = 0 FILES_STORE_S3_ACL = 'private' -FILES_STORE_GCS_ACL = None +FILES_STORE_GCS_ACL = '' FTP_USER = 'anonymous' FTP_PASSWORD = 'guest' @@ -182,7 +182,7 @@ HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' -IMAGES_STORE_GCS_ACL = None +IMAGES_STORE_GCS_ACL = '' ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' From 6ef6585b5a187d4a8dcec99ba7cae9b6cee91b30 Mon Sep 17 00:00:00 2001 From: rhoboro Date: Fri, 13 Apr 2018 19:06:29 +0900 Subject: [PATCH 346/362] update docs --- docs/topics/media-pipeline.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 0872ac0cd..a1f518cbd 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -209,7 +209,7 @@ For information about authentication, see this `documentation`_. You can modify the Access Control List (ACL) policy used for the stored files, which is defined by the :setting:`FILES_STORE_GCS_ACL` and :setting:`IMAGES_STORE_GCS_ACL` settings. By default, the ACL is set to -None which means that Cloud Storage applies the bucket's default object ACL to the object. +``''`` (empty string) which means that Cloud Storage applies the bucket's default object ACL to the object. To make the files publicly available use the ``publicRead`` policy:: From 57b0e6b6955705efcbe2d3da5501b27716ef1baf Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Thu, 19 Apr 2018 13:35:46 +0800 Subject: [PATCH 347/362] improve document about functions as processors --- docs/topics/loaders.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index ad86dba63..cdb066e57 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -136,6 +136,20 @@ accept one (and only one) positional argument, which will be an iterator. containing the collected values (for that field). The result of the output processors is the value that will be finally assigned to the item. +If you want to use a plain function as a processor, make sure it receives +``self`` as the first argument:: + + def lowercase_processor(self, values): + for v in values: + yield v.lower() + + class MyItemLoader(ItemLoader): + name_in = lowercase_processor + +This is because whenever a function is assigned as a class variable, it becomes +a method and would be passed the instance as the the first argument when being +called. See `this answer on stackoverflow`_ for more details. + The other thing you need to keep in mind is that the values returned by input processors are collected internally (in lists) and then passed to output processors to populate the fields. @@ -143,6 +157,7 @@ processors to populate the fields. Last, but not least, Scrapy comes with some :ref:`commonly used processors ` built-in for convenience. +.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635 Declaring Item Loaders ====================== From e75f721c04446f8f28d3bdfcd69f967f65981407 Mon Sep 17 00:00:00 2001 From: Pengyu Chen Date: Mon, 23 Apr 2018 22:08:28 +0800 Subject: [PATCH 348/362] Added: Allowing optional arguments for `scrapy.http.cookies.CookieJar.clear` --- scrapy/http/cookies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index a1e95102e..4e8056750 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -58,8 +58,8 @@ class CookieJar(object): def clear_session_cookies(self, *args, **kwargs): return self.jar.clear_session_cookies(*args, **kwargs) - def clear(self): - return self.jar.clear() + def clear(self, domain=None, path=None, name=None): + return self.jar.clear(domain, path, name) def __iter__(self): return iter(self.jar) From 0d015e5c0f0dcb8936044aec163dd12e33482730 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 16 May 2018 09:36:07 +0000 Subject: [PATCH 349/362] blacklist twisted version with regression --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2a94d742d..7d857a8c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted>=13.1.0 +Twisted>=13.1.0,!=18.4.0 lxml pyOpenSSL cssselect>=0.9 From c5ddfddb7e5b42e45b6c8f86a381ee52fbb6f797 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 17 May 2018 08:53:42 +0000 Subject: [PATCH 350/362] blacklist twisted version with regression in constraints file --- requirements.txt | 2 +- tests/constraints.txt | 1 + tox.ini | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 tests/constraints.txt diff --git a/requirements.txt b/requirements.txt index 7d857a8c9..2a94d742d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Twisted>=13.1.0,!=18.4.0 +Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 diff --git a/tests/constraints.txt b/tests/constraints.txt new file mode 100644 index 000000000..3bc30de15 --- /dev/null +++ b/tests/constraints.txt @@ -0,0 +1 @@ +Twisted!=18.4.0 diff --git a/tox.ini b/tox.ini index 60ff8c15e..c2fa9af28 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = py27 [testenv] deps = + -ctests/constraints.txt -rrequirements.txt # Extras botocore From b364d27247b2d9b86c164569c7e0459fa3f8391b Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 23 May 2018 21:25:50 +0300 Subject: [PATCH 351/362] [MRG+1] Automatic port selection for servicies in unit tests (#3210) * ability to pass port as a parameter * try to find free ports * use environment variables to pass mock server address * get mock server address from environment variables * ability to select ports for proxy in runtime * use common method for URLs from mock server * https support * get mock server address * get mock address * replace hand-written mechanism by kernel-based one * use ephemeral ports in mockserver * strip EOL from addresses * use ephemeral port in proxy * no need to restore environment as it is restored in tearDown * decode bytes * use mockserver address as a variable * ability to pass address as variable * per test-case mockserver * use base class * remove obsolete environment manipulation * return usage of proxy for http cases * common method for broking proxy auth credentials * python version-independent url methods --- tests/mockserver.py | 25 +++++++++--- tests/spiders.py | 17 ++++++--- tests/test_closespider.py | 8 ++-- tests/test_crawl.py | 48 ++++++++++++------------ tests/test_downloader_handlers.py | 8 ++-- tests/test_feedexport.py | 3 +- tests/test_pipeline_crawl.py | 16 ++++---- tests/test_proxy_connect.py | 41 ++++++++++++-------- tests/test_spidermiddleware_httperror.py | 23 ++++++------ 9 files changed, 111 insertions(+), 78 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 98723846e..f36ce3c44 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -192,9 +192,15 @@ class MockServer(): def __enter__(self): from scrapy.utils.test import get_testenv + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'], stdout=PIPE, env=get_testenv()) - self.proc.stdout.readline() + http_address = self.proc.stdout.readline().strip().decode('ascii') + https_address = self.proc.stdout.readline().strip().decode('ascii') + + self.http_address = http_address + self.https_address = https_address + return self def __exit__(self, exc_type, exc_value, traceback): @@ -202,6 +208,12 @@ class MockServer(): self.proc.wait() time.sleep(0.2) + def url(self, path, is_secure=False): + host = self.http_address + if is_secure: + host = self.https_address + return host + path + def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt'): return ssl.DefaultOpenSSLContextFactory( @@ -213,14 +225,17 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c if __name__ == "__main__": root = Root() factory = Site(root) - httpPort = reactor.listenTCP(8998, factory) + httpPort = reactor.listenTCP(0, factory) contextFactory = ssl_context_factory() - httpsPort = reactor.listenSSL(8999, factory, contextFactory) + httpsPort = reactor.listenSSL(0, factory, contextFactory) def print_listening(): httpHost = httpPort.getHost() httpsHost = httpsPort.getHost() - print("Mock server running at http://%s:%d and https://%s:%d" % ( - httpHost.host, httpHost.port, httpsHost.host, httpsHost.port)) + httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port) + httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port) + print(httpAddress) + print(httpsAddress) + reactor.callWhenRunning(print_listening) reactor.run() diff --git a/tests/spiders.py b/tests/spiders.py index 1038b69de..7816bf7c7 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -11,7 +11,12 @@ from scrapy.item import Item from scrapy.linkextractors import LinkExtractor -class MetaSpider(Spider): +class MockServerSpider(Spider): + def __init__(self, mockserver=None, *args, **kwargs): + super(MockServerSpider, self).__init__(*args, **kwargs) + self.mockserver = mockserver + +class MetaSpider(MockServerSpider): name = 'meta' @@ -33,7 +38,7 @@ class FollowAllSpider(MetaSpider): self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} - url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) + url = self.mockserver.url("/follow?%s" % urlencode(qargs, doseq=1)) self.start_urls = [url] def parse(self, response): @@ -55,7 +60,7 @@ class DelaySpider(MetaSpider): def start_requests(self): self.t1 = time.time() - url = "http://localhost:8998/delay?n=%s&b=%s" % (self.n, self.b) + url = self.mockserver.url("/delay?n=%s&b=%s" % (self.n, self.b)) yield Request(url, callback=self.parse, errback=self.errback) def parse(self, response): @@ -121,7 +126,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): for s in range(100): qargs = {'total': 10, 'seed': s} - url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) + url = self.mockserver.url("/follow?%s") % urlencode(qargs, doseq=1) yield Request(url, meta={'seed': s}) if self.fail_yielding: 2 / 0 @@ -160,7 +165,7 @@ class SingleRequestSpider(MetaSpider): return self.errback_func(failure) -class DuplicateStartRequestsSpider(Spider): +class DuplicateStartRequestsSpider(MockServerSpider): dont_filter = True name = 'duplicatestartrequests' distinct_urls = 2 @@ -169,7 +174,7 @@ class DuplicateStartRequestsSpider(Spider): def start_requests(self): for i in range(0, self.distinct_urls): for j in range(0, self.dupe_factor): - url = "http://localhost:8998/echo?headers=1&body=test%d" % i + url = self.mockserver.url("/echo?headers=1&body=test%d" % i) yield Request(url, dont_filter=self.dont_filter) def __init__(self, url="http://localhost:8998", *args, **kwargs): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index fa0b48998..0eb1b7944 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -18,7 +18,7 @@ class TestCloseSpider(TestCase): def test_closespider_itemcount(self): close_on = 5 crawler = get_crawler(ItemSpider, {'CLOSESPIDER_ITEMCOUNT': close_on}) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_itemcount') itemcount = crawler.stats.get_value('item_scraped_count') @@ -28,7 +28,7 @@ class TestCloseSpider(TestCase): def test_closespider_pagecount(self): close_on = 5 crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_PAGECOUNT': close_on}) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_pagecount') pagecount = crawler.stats.get_value('response_received_count') @@ -38,7 +38,7 @@ class TestCloseSpider(TestCase): def test_closespider_errorcount(self): close_on = 5 crawler = get_crawler(ErrorSpider, {'CLOSESPIDER_ERRORCOUNT': close_on}) - yield crawler.crawl(total=1000000) + yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') key = 'spider_exceptions/{name}'\ @@ -50,7 +50,7 @@ class TestCloseSpider(TestCase): def test_closespider_timeout(self): close_on = 0.1 crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_TIMEOUT': close_on}) - yield crawler.crawl(total=1000000) + yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_timeout') stats = crawler.stats diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d5babdded..3fc13eeb7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -26,7 +26,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_follow_all(self): crawler = self.runner.create_crawler(FollowAllSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url @defer.inlineCallbacks @@ -42,7 +42,7 @@ class CrawlTestCase(TestCase): def _test_delay(self, delay, randomize): settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize} crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider) - yield crawler.crawl(maxlatency=delay * 2) + yield crawler.crawl(maxlatency=delay * 2, mockserver=self.mockserver) t = crawler.spider.times totaltime = t[-1] - t[0] avgd = totaltime / (len(t) - 1) @@ -53,7 +53,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_success(self): crawler = self.runner.create_crawler(DelaySpider) - yield crawler.crawl(n=0.5) + yield crawler.crawl(n=0.5, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 > 0) self.assertTrue(crawler.spider.t2 > crawler.spider.t1) @@ -61,13 +61,13 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_failure(self): crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider) - yield crawler.crawl(n=0.5) + yield crawler.crawl(n=0.5, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) self.assertTrue(crawler.spider.t2_err > 0) self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) # server hangs after receiving response headers - yield crawler.crawl(n=0.5, b=1) + yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) self.assertTrue(crawler.spider.t2_err > 0) @@ -77,14 +77,14 @@ class CrawlTestCase(TestCase): def test_retry_503(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/status?n=503") + yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks def test_retry_conn_failed(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:65432/status?n=503") + yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks @@ -92,14 +92,14 @@ class CrawlTestCase(TestCase): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: # try to fetch the homepage of a non-existent domain - yield crawler.crawl("http://dns.resolution.invalid./") + yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): with LogCapture('scrapy', level=logging.ERROR) as l: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl(fail_before_yield=1) + yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) self.assertEqual(len(l.records), 1) record = l.records[0] @@ -110,7 +110,7 @@ class CrawlTestCase(TestCase): def test_start_requests_bug_yielding(self): with LogCapture('scrapy', level=logging.ERROR) as l: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl(fail_yielding=1) + yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) self.assertEqual(len(l.records), 1) record = l.records[0] @@ -121,7 +121,7 @@ class CrawlTestCase(TestCase): def test_start_requests_lazyness(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) #self.assertTrue(False, crawler.spider.seedsseen) #self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99), # crawler.spider.seedsseen) @@ -130,10 +130,10 @@ class CrawlTestCase(TestCase): def test_start_requests_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider) - yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3) + yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver) self.assertEqual(crawler.spider.visited, 6) - yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4) + yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4, mockserver=self.mockserver) self.assertEqual(crawler.spider.visited, 3) @defer.inlineCallbacks @@ -160,7 +160,7 @@ with multiples lines '''}) crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/raw?{0}".format(query)) + yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver) self.assertEqual(str(l).count("Got response 200"), 1) @defer.inlineCallbacks @@ -168,7 +168,7 @@ with multiples lines # connection lost after receiving data crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/drop?abort=0") + yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks @@ -176,7 +176,7 @@ with multiples lines # connection lost before receiving data crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/drop?abort=1") + yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver) self._assert_retried(l) def _assert_retried(self, log): @@ -186,7 +186,7 @@ with multiples lines @defer.inlineCallbacks def test_referer_header(self): """Referer header is set by RefererMiddleware unless it is already set""" - req0 = Request('http://localhost:8998/echo?headers=1&body=0', dont_filter=1) + req0 = Request(self.mockserver.url('/echo?headers=1&body=0'), dont_filter=1) req1 = req0.replace() req2 = req0.replace(headers={'Referer': None}) req3 = req0.replace(headers={'Referer': 'http://example.com'}) @@ -194,7 +194,7 @@ with multiples lines req1.meta['next'] = req2 req2.meta['next'] = req3 crawler = self.runner.create_crawler(SingleRequestSpider) - yield crawler.crawl(seed=req0) + yield crawler.crawl(seed=req0, mockserver=self.mockserver) # basic asserts in case of weird communication errors self.assertIn('responses', crawler.spider.meta) self.assertNotIn('failures', crawler.spider.meta) @@ -220,7 +220,7 @@ with multiples lines est.append(get_engine_status(crawler.engine)) crawler = self.runner.create_crawler(SingleRequestSpider) - yield crawler.crawl(seed='http://localhost:8998/', callback_func=cb) + yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) self.assertEqual(len(est), 1, est) s = dict(est[0]) self.assertEqual(s['engine.spider.name'], crawler.spider.name) @@ -244,7 +244,7 @@ with multiples lines raise TestError crawler = self.runner.create_crawler(FaultySpider) - yield self.assertFailure(crawler.crawl(), TestError) + yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError) self.assertFalse(crawler.crawling) @defer.inlineCallbacks @@ -256,7 +256,7 @@ with multiples lines } crawler = CrawlerRunner(settings).create_crawler(SimpleSpider) yield self.assertFailure( - self.runner.crawl(crawler, "http://localhost:8998/status?n=200"), + self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver), ZeroDivisionError) self.assertFalse(crawler.crawling) @@ -264,13 +264,13 @@ with multiples lines def test_crawlerrunner_accepts_crawler(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as log: - yield self.runner.crawl(crawler, "http://localhost:8998/status?n=200") + yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) self.assertIn("Got response 200", str(log)) @defer.inlineCallbacks def test_crawl_multiple(self): - self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=200") - self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=503") + self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver) with LogCapture() as log: yield self.runner.join() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index b34faa7e7..c91be2c0c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -534,14 +534,14 @@ class Http11MockServerTestCase(unittest.TestCase): crawler = get_crawler(SingleRequestSpider) # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid # download it - yield crawler.crawl(seed=Request(url='http://localhost:8998/partial', meta={'download_maxsize': 1000})) + yield crawler.crawl(seed=Request(url=self.mockserver.url('/partial'), meta={'download_maxsize': 1000})) failure = crawler.spider.meta['failure'] self.assertIsInstance(failure.value, defer.CancelledError) @defer.inlineCallbacks def test_download(self): crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl(seed=Request(url='http://localhost:8998')) + yield crawler.crawl(seed=Request(url=self.mockserver.url(''))) failure = crawler.spider.meta.get('failure') self.assertTrue(failure == None) reason = crawler.spider.meta['close_reason'] @@ -551,7 +551,7 @@ class Http11MockServerTestCase(unittest.TestCase): def test_download_gzip_response(self): crawler = get_crawler(SingleRequestSpider) body = b'1' * 100 # PayloadResource requires body length to be 100 - request = Request('http://localhost:8998/payload', method='POST', + request = Request(self.mockserver.url('/payload'), method='POST', body=body, meta={'download_maxsize': 50}) yield crawler.crawl(seed=request) failure = crawler.spider.meta['failure'] @@ -560,7 +560,7 @@ class Http11MockServerTestCase(unittest.TestCase): if six.PY2: request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') - request = request.replace(url='http://localhost:8998/xpayload') + request = request.replace(url=self.mockserver.url('/xpayload')) yield crawler.crawl(seed=request) # download_maxsize = 50 is enough for the gzipped response failure = crawler.spider.meta.get('failure') diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f55927121..0d9f1e83c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -179,6 +179,7 @@ class FeedExportTest(unittest.TestCase): try: with MockServer() as s: runner = CrawlerRunner(Settings(defaults)) + spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) with open(res_name, 'rb') as f: @@ -194,7 +195,6 @@ class FeedExportTest(unittest.TestCase): """ class TestSpider(scrapy.Spider): name = 'testspider' - start_urls = ['http://localhost:8998/'] def parse(self, response): for item in items: @@ -210,7 +210,6 @@ class FeedExportTest(unittest.TestCase): """ class TestSpider(scrapy.Spider): name = 'testspider' - start_urls = ['http://localhost:8998/'] def parse(self, response): pass diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 9b81f827d..5985a6f3e 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -46,7 +46,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): def _process_url(self, url): return add_or_replace_parameter( - 'http://localhost:8998/redirect-to', + self.mockserver.url('/redirect-to'), 'goto', url) @@ -134,7 +134,7 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media(self): crawler = self._create_crawler(MediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_downloaded(self.items, str(log)) @@ -143,7 +143,7 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_wrong_urls(self): crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_download_failure(crawler, self.items, 404, str(log)) @@ -152,9 +152,10 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_redirected_default_failure(self): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, - media_urls_key=self.media_urls_key) + media_urls_key=self.media_urls_key, + mockserver=self.mockserver) self._assert_files_download_failure(crawler, self.items, 302, str(log)) @defer.inlineCallbacks @@ -165,9 +166,10 @@ class FileDownloadCrawlTestCase(TestCase): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, - media_urls_key=self.media_urls_key) + media_urls_key=self.media_urls_key, + mockserver=self.mockserver) self._assert_files_downloaded(self.items, str(log)) self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 6213a51e8..ae1236bcb 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -2,6 +2,7 @@ import json import os import time +from six.moves.urllib.parse import urlsplit, urlunsplit from threading import Thread from libmproxy import controller, proxy from netlib import http_auth @@ -17,7 +18,7 @@ from tests.mockserver import MockServer class HTTPSProxy(controller.Master, Thread): - def __init__(self, port): + def __init__(self): password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy') authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), @@ -25,10 +26,19 @@ class HTTPSProxy(controller.Master, Thread): server = proxy.ProxyServer(proxy.ProxyConfig( authenticator = authenticator, cacert = cert_path), - port) + 0) + self.server = server Thread.__init__(self) controller.Master.__init__(self, server) + def http_address(self): + return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname() + + +def _wrong_credentials(proxy_url): + bad_auth_proxy = list(urlsplit(proxy_url)) + bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@') + return urlunsplit(bad_auth_proxy) class ProxyConnectTestCase(TestCase): @@ -36,12 +46,14 @@ class ProxyConnectTestCase(TestCase): self.mockserver = MockServer() self.mockserver.__enter__() self._oldenv = os.environ.copy() - self._proxy = HTTPSProxy(8888) + + self._proxy = HTTPSProxy() self._proxy.start() + # Wait for the proxy to start. time.sleep(1.0) - os.environ['http_proxy'] = 'http://scrapy:scrapy@localhost:8888' - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' + os.environ['https_proxy'] = self._proxy.http_address() + os.environ['http_proxy'] = self._proxy.http_address() def tearDown(self): self.mockserver.__exit__(None, None, None) @@ -52,17 +64,17 @@ class ProxyConnectTestCase(TestCase): def test_https_connect_tunnel(self): crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) @defer.inlineCallbacks def test_https_noconnect(self): - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888?noconnect' + proxy = os.environ['https_proxy'] + os.environ['https_proxy'] = proxy + '?noconnect' crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' @defer.inlineCallbacks def test_https_connect_tunnel_error(self): @@ -73,18 +85,17 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_tunnel_auth_error(self): - os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888' + os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) # The proxy returns a 407 error code but it does not reach the client; # he just sees a TunnelError. self._assert_got_tunnel_error(l) - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' @defer.inlineCallbacks def test_https_tunnel_without_leak_proxy_authorization_header(self): - request = Request("https://localhost:8999/echo") + request = Request(self.mockserver.url("/echo", is_secure=True)) crawler = get_crawler(SingleRequestSpider) with LogCapture() as l: yield crawler.crawl(seed=request) @@ -94,10 +105,10 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_noconnect_auth_error(self): - os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888?noconnect' + os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(407, l) def _assert_got_response_code(self, code, log): diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 19e6bbdcd..dacd0147f 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -11,20 +11,21 @@ from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError from scrapy.settings import Settings +from tests.spiders import MockServerSpider -class _HttpErrorSpider(Spider): +class _HttpErrorSpider(MockServerSpider): name = 'httperror' - start_urls = [ - "http://localhost:8998/status?n=200", - "http://localhost:8998/status?n=404", - "http://localhost:8998/status?n=402", - "http://localhost:8998/status?n=500", - ] bypass_status_codes = set() def __init__(self, *args, **kwargs): super(_HttpErrorSpider, self).__init__(*args, **kwargs) + self.start_urls = [ + self.mockserver.url("/status?n=200"), + self.mockserver.url("/status?n=404"), + self.mockserver.url("/status?n=402"), + self.mockserver.url("/status?n=500"), + ] self.failed = set() self.skipped = set() self.parsed = set() @@ -169,7 +170,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): @defer.inlineCallbacks def test_middleware_works(self): crawler = get_crawler(_HttpErrorSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) assert not crawler.spider.skipped, crawler.spider.skipped self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) @@ -184,7 +185,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): def test_logging(self): crawler = get_crawler(_HttpErrorSpider) with LogCapture() as log: - yield crawler.crawl(bypass_status_codes={402}) + yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402}) self.assertEqual(crawler.spider.parsed, {'200', '402'}) self.assertEqual(crawler.spider.skipped, {'402'}) self.assertEqual(crawler.spider.failed, {'404', '500'}) @@ -199,7 +200,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): # HttpError logs ignored responses with level INFO crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.INFO) as log: - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) @@ -211,7 +212,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): # with level WARNING, we shouldn't capture anything from HttpError crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.WARNING) as log: - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) From ffa7bede17088e5eebd50305704906bf1451ab3a Mon Sep 17 00:00:00 2001 From: Kevin Tewouda Date: Wed, 30 May 2018 06:33:18 +0200 Subject: [PATCH 352/362] Update spiders.rst I changed URLs to :class:`~scrapy.http.Request` in start_urls explanation of the default spider --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index c2c271245..697732b47 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -88,7 +88,7 @@ scrapy.Spider A list of URLs where the spider will begin to crawl from, when no particular URLs are specified. So, the first pages downloaded will be those - listed here. The subsequent URLs will be generated successively from data + listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data contained in the start URLs. .. attribute:: custom_settings From ecdd888ff4614a0e994c9dadb31b7c9c85c88e8a Mon Sep 17 00:00:00 2001 From: Chris Slothouber Date: Fri, 1 Jun 2018 09:25:34 -0400 Subject: [PATCH 353/362] Minor edits to contributing.rst Corrected minor grammatical issues and increased clarity of instructions. --- docs/contributing.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index f4f9e393f..6615840f7 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -48,9 +48,9 @@ guidelines when reporting a new bug. `Stack Overflow `__ (use "scrapy" tag). -* check the `open issues`_ to see if it has already been reported. If it has, - don't dismiss the report, but check the ticket history and comments. If you - have additional useful information, please leave a comment, or consider +* check the `open issues`_ to see if the issue has already been reported. If it + has, don't dismiss the report, but check the ticket history and comments. If + you have additional useful information, please leave a comment, or consider :ref:`sending a pull request ` with a fix. * search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has @@ -122,7 +122,7 @@ conversation in the `Scrapy subreddit`_ to discuss your idea first. Sometimes there is an existing pull request for the problem you'd like to solve, which is stalled for some reason. Often the pull request is in a right direction, but changes are requested by Scrapy maintainers, and the -original pull request author haven't had time to address them. +original pull request author hasn't had time to address them. In this case consider picking up this pull request: open a new pull request with all commits from the original pull request, as well as additional changes to address the raised issues. Doing so helps a lot; it is @@ -143,7 +143,7 @@ instead of "Fix for #411". Complete titles make it easy to skim through the issue tracker. Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports -removal, etc) in separate commits than functional changes. This will make pull +removal, etc) in separate commits from functional changes. This will make pull requests easier to review and more likely to get merged. Coding style @@ -170,7 +170,7 @@ Documentation policies **do** provide a docstring, but make sure sphinx documentation uses autodoc_ extension to pull the docstring. For example, the :meth:`ItemLoader.add_value` method should be either - documented only in the sphinx documentation (not it a docstring), or + documented only in the sphinx documentation (not as a docstring), or it should have a docstring which is pulled to sphinx documentation using autodoc_ extension. From 6a2d2c3b77bde1d74b46d7dbfb9488cb06e5021f Mon Sep 17 00:00:00 2001 From: Fredrik Bergenlid Date: Fri, 1 Jun 2018 21:38:07 +0200 Subject: [PATCH 354/362] Improve gunzip performance for big files --- scrapy/utils/gz.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 16c9ce539..ec3949651 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -30,25 +30,25 @@ def gunzip(data): This is resilient to CRC checksum errors. """ f = GzipFile(fileobj=BytesIO(data)) - output = b'' + output_list = [] chunk = b'.' while chunk: try: chunk = read1(f, 8196) - output += chunk + output_list.append(chunk) except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - # some pages are quite small so output is '' and f.extrabuf + # some pages are quite small so output_list is empty and f.extrabuf # contains the whole page content - if output or getattr(f, 'extrabuf', None): + if output_list or getattr(f, 'extrabuf', None): try: - output += f.extrabuf[-f.extrasize:] + output_list.append(f.extrabuf[-f.extrasize:]) finally: break else: raise - return output + return b''.join(output_list) _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search From 98d9093dc7241b32b927e79ff2f8a475ae5659a6 Mon Sep 17 00:00:00 2001 From: Colton Herinckx Date: Mon, 14 May 2018 13:37:16 -0700 Subject: [PATCH 355/362] minor grammatical fixes in CODE_OF_CONDUCT.md --- CODE_OF_CONDUCT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 162602248..d477168eb 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -3,7 +3,7 @@ ## Our Pledge In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and +contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and From 9bd5444a42e82a05d2791fd10a205a9dd99303a6 Mon Sep 17 00:00:00 2001 From: Colton Herinckx Date: Mon, 14 May 2018 13:48:28 -0700 Subject: [PATCH 356/362] added oxford commas to LICENSE --- LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 6ead05ece..4d0a0863a 100644 --- a/LICENSE +++ b/LICENSE @@ -5,10 +5,10 @@ Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. + this list of conditions, and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the + notice, this list of conditions, and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of Scrapy nor the names of its contributors may be used From 12d10eec2cb10fb13cf199d73363093478e80f9a Mon Sep 17 00:00:00 2001 From: Colton Herinckx Date: Mon, 14 May 2018 13:53:53 -0700 Subject: [PATCH 357/362] changed Twisted >= 17.9.0 to Twisted>=17.9.0 --- requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 2aae3ae65..1f342cfbb 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted >= 17.9.0 +Twisted>=17.9.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 From 596f39600dfa94504b585ef65b1b04571a631447 Mon Sep 17 00:00:00 2001 From: Colton Herinckx Date: Sat, 19 May 2018 16:32:55 -0700 Subject: [PATCH 358/362] reversed earlier change that seemed to cause Travis CI build failure --- requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 1f342cfbb..2aae3ae65 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,4 +1,4 @@ -Twisted>=17.9.0 +Twisted >= 17.9.0 lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 From d4511667fb5df63058accfe731e3d4160e795ee8 Mon Sep 17 00:00:00 2001 From: mugayoshi Date: Sat, 9 Jun 2018 18:17:11 +0900 Subject: [PATCH 359/362] Update debugging memory leaks section in the docs Add Python3 tools description. --- docs/topics/leaks.rst | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 92590c180..af14d14e8 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -202,6 +202,7 @@ memory leaks (Requests, Responses, Items, and Selectors). However, there are other cases where the memory leaks could come from other (more or less obscure) objects. If this is your case, and you can't find your leaks using ``trackref``, you still have another resource: the `Guppy library`_. +If you're using Python3, see :ref:`topics-leaks-muppy`. .. _Guppy library: https://pypi.python.org/pypi/guppy @@ -253,6 +254,50 @@ knowledge about Python internals. For more info about Guppy, refer to the .. _Guppy documentation: http://guppy-pe.sourceforge.net/ +.. _topics-leaks-muppy: + +Debugging memory leaks with muppy +================================= +If you're using Python 3, you can use muppy from `Pympler`_. + +.. _Pympler: https://pypi.org/project/Pympler/ + +If you use ``pip``, you can install muppy with the following command:: + + pip install Pympler + +Here's an example to view all Python objects available in +the heap using muppy:: + + >>> from pympler import muppy + >>> all_objects = muppy.get_objects() + >>> len(all_objects) + 28667 + >>> from pympler import summary + >>> suml = summary.summarize(all_objects) + >>> summary.print_(suml) + types | # objects | total size + ==================================== | =========== | ============ + Date: Thu, 14 Jun 2018 17:58:48 +0300 Subject: [PATCH 360/362] Return non-zero exit code from scrapy commands in case of spider bootstrap errors * method to detect spider creation in crawler * correct method name * method to know if crawlers has spiders * we do not need to issue requests * set exit code accordingly to spiders in crawlers * more portable way to check ofr exceptions * more clear way * test cases for several spiders per crawler * grammatically correct name for method * method is private * grammatically correct name for method * method is private * remove unused import * correct order of imports * changes mechanism of obtaining spider status from method to object member * rename tests --- scrapy/commands/crawl.py | 3 ++ scrapy/commands/runspider.py | 3 ++ scrapy/crawler.py | 2 ++ tests/test_commands.py | 12 +++++++ tests/test_crawler.py | 64 +++++++++++++++++++++++++++++++++++- 5 files changed, 83 insertions(+), 1 deletion(-) diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 4b986bf9d..8093fd402 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -56,3 +56,6 @@ class Command(ScrapyCommand): self.crawler_process.crawl(spname, **opts.spargs) self.crawler_process.start() + + if self.crawler_process.bootstrap_failed: + self.exitcode = 1 diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index a98033dd1..376d3c84e 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -87,3 +87,6 @@ class Command(ScrapyCommand): self.crawler_process.crawl(spidercls, **opts.spargs) self.crawler_process.start() + + if self.crawler_process.bootstrap_failed: + self.exitcode = 1 diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 5cbc2d7c5..04aee18ed 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -137,6 +137,7 @@ class CrawlerRunner(object): self.spider_loader = _get_spider_loader(settings) self._crawlers = set() self._active = set() + self.bootstrap_failed = False @property def spiders(self): @@ -178,6 +179,7 @@ class CrawlerRunner(object): def _done(result): self.crawlers.discard(crawler) self._active.discard(d) + self.bootstrap_failed |= not getattr(crawler, 'spider', None) return result return d.addBoth(_done) diff --git a/tests/test_commands.py b/tests/test_commands.py index cb1301c95..7d9071b64 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,3 +1,4 @@ +import inspect import os import sys import subprocess @@ -17,6 +18,7 @@ from scrapy.utils.python import retry_on_eintr from scrapy.utils.test import get_testenv from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest +from tests.test_crawler import ExceptionSpider, NoRequestsSpider class ProjectTest(unittest.TestCase): @@ -220,6 +222,16 @@ class MySpider(scrapy.Spider): self.assertIn("INFO: Closing spider (finished)", log) self.assertIn("INFO: Spider closed (finished)", log) + def test_run_fail_spider(self): + proc = self.runspider("import scrapy\n" + inspect.getsource(ExceptionSpider)) + ret = proc.returncode + self.assertNotEqual(ret, 0) + + def test_run_good_spider(self): + proc = self.runspider("import scrapy\n" + inspect.getsource(NoRequestsSpider)) + ret = proc.returncode + self.assertEqual(ret, 0) + def test_runspider_log_level(self): log = self.get_log(self.debug_log_spider, args=('-s', 'LOG_LEVEL=INFO')) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ba0d709ff..d3b80f460 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -4,6 +4,9 @@ import tempfile import warnings import unittest +from twisted.internet import defer +import twisted.trial.unittest + import scrapy from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess from scrapy.settings import Settings, default_settings @@ -11,9 +14,9 @@ from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object +from scrapy.utils.test import get_crawler from scrapy.extensions.throttle import AutoThrottle - class BaseCrawlerTest(unittest.TestCase): def assertOptionIsDefault(self, settings, key): @@ -181,3 +184,62 @@ class CrawlerProcessTest(BaseCrawlerTest): def test_crawler_process_accepts_None(self): runner = CrawlerProcess() self.assertOptionIsDefault(runner.settings, 'RETRY_ENABLED') + + +class ExceptionSpider(scrapy.Spider): + name = 'exception' + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + raise ValueError('Exception in from_crawler method') + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +class CrawlerRunnerHasSpider(twisted.trial.unittest.TestCase): + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_successful(self): + runner = CrawlerRunner() + yield runner.crawl(NoRequestsSpider) + self.assertEqual(runner.bootstrap_failed, False) + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_successful_for_several(self): + runner = CrawlerRunner() + yield runner.crawl(NoRequestsSpider) + yield runner.crawl(NoRequestsSpider) + self.assertEqual(runner.bootstrap_failed, False) + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_failed(self): + runner = CrawlerRunner() + + try: + yield runner.crawl(ExceptionSpider) + except ValueError: + pass + else: + self.fail('Exception should be raised from spider') + + self.assertEqual(runner.bootstrap_failed, True) + + @defer.inlineCallbacks + def test_crawler_runner_bootstrap_failed_for_several(self): + runner = CrawlerRunner() + + try: + yield runner.crawl(ExceptionSpider) + except ValueError: + pass + else: + self.fail('Exception should be raised from spider') + + yield runner.crawl(NoRequestsSpider) + + self.assertEqual(runner.bootstrap_failed, True) From 7a601d76de7adc37571815a7d08f84e1e26f7507 Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 19 Jun 2018 10:51:55 +0200 Subject: [PATCH 361/362] fix typo extractred --> extracted --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 1629510b2..1b8d121a1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -156,7 +156,7 @@ attributes with ``FormRequest``. **Please also note that link extractors do not canonicalize URLs by default anymore.** This was puzzling users every now and then, and it's not what -browsers do in fact, so we removed that extra transformation on extractred +browsers do in fact, so we removed that extra transformation on extracted links. For those of you wanting more control on the ``Referer:`` header that Scrapy From 88bd067912ee94e2a6d2e1ba5b0d5db1241b7621 Mon Sep 17 00:00:00 2001 From: Grammy Jiang Date: Wed, 20 Jun 2018 16:56:46 +0800 Subject: [PATCH 362/362] fix the test case name of HttpProxyMiddleware --- tests/test_downloadermiddleware_httpproxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 17be875c1..537126613 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -13,7 +13,7 @@ from scrapy.settings import Settings spider = Spider('foo') -class TestDefaultHeadersMiddleware(TestCase): +class TestHttpProxyMiddleware(TestCase): failureException = AssertionError