From 4eec05351fd505dcd5e1060d25fb268675b13901 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Mon, 1 Aug 2016 11:37:51 -0300 Subject: [PATCH 01/91] remove bumpversion prerelease configuration I propose to remove the prerelease configuration from bumpversion, because I think its behavior is just too confusing. The rational for this is that making the release procedure predictable is more important than facilitating making pre-releases, which are sort of the exception in the workflow. The current configuration makes most common cases confusing: * bug fix releases require you have to remember to use `--serialize "{major}.{minor}.{patch}"` * to start a pre-release cycle, you actually use `minor` or `patch` * to do the actual minor or patch release, you use `prerel` Also, `prerel` breaks if you run it on a branch with a final release, because it can't parse the prerelease information. Therefore, I propose keeping the bumpversion defaults, and do the prereleases (dev1, dev2, rc1, etc) manually (with `--new-version`), which makes for a more predictable and intuitive behavior. * `bumpversion minor` and `bumpversion patch` will work as expected * pre-releases will be manually handled, but this seems a small overhead than remembering the details I mention above. If you're happy with this, I'll also update [the wiki][1] with new instructions. [1]: https://github.com/scrapy/scrapy/wiki/Scrapy-release-procedure --- .bumpversion.cfg | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 49a7e239f..12f5cb16c 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -3,27 +3,5 @@ current_version = 1.2.0dev2 commit = True tag = True tag_name = {new_version} -parse = ^ - (?P\d+)\.(?P\d+)\.(?P\d+) - (?:(?P[abc]|rc|dev)(?P\d+))? -serialize = - {major}.{minor}.{patch}{prerel}{prerelversion} - {major}.{minor}.{patch} [bumpversion:file:scrapy/VERSION] - -[bumpversion:part:prerel] -optional_value = gamma -values = - dev - rc - gamma - -[bumpversion:part:prerelversion] -values = - 1 - 2 - 3 - 4 - 5 - From c7dfb5eb88e2cb9c489725e5ad81bc1c69f7b3fc Mon Sep 17 00:00:00 2001 From: gustavodeandrade Date: Fri, 21 Oct 2016 00:08:08 -0200 Subject: [PATCH 02/91] Fix issue 1704 --- docs/topics/selectors.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 39ec9b73c..293628953 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -626,6 +626,11 @@ Built-in Selectors reference ``regex`` can be either a compiled regular expression or a string which will be compiled to a regular expression using ``re.compile(regex)`` + .. note:: + + Note that ``re`` and ``re_first`` both escape HTML entities. If you want to + get raw unescaped content, use extract() or extract_first() + .. method:: register_namespace(prefix, uri) Register the given namespace to be used in this :class:`Selector`. From f2e49bc23cd2044551721bf81f6443a391b30f6f Mon Sep 17 00:00:00 2001 From: Gustavo de Andrade Date: Tue, 1 Nov 2016 21:32:17 -0200 Subject: [PATCH 03/91] Update selectors.rst Decode instead escape, exceptions < and & (kmike) Second sentence droped (Digenis) --- docs/topics/selectors.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 293628953..9f8143db1 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -628,8 +628,7 @@ Built-in Selectors reference .. note:: - Note that ``re`` and ``re_first`` both escape HTML entities. If you want to - get raw unescaped content, use extract() or extract_first() + Note that ``re()`` and ``re_first()`` both decode HTML entities (except ``<`` and ``&``). .. method:: register_namespace(prefix, uri) From e7c7e055ff3c661cd60b15fe5ad910a2765db707 Mon Sep 17 00:00:00 2001 From: Raul Gallegos Date: Sat, 24 Dec 2016 11:55:04 -0500 Subject: [PATCH 04/91] settings: fixing name of the pipeline template --- scrapy/templates/project/module/settings.py.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 72f25ebef..486df6b71 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -65,7 +65,7 @@ ROBOTSTXT_OBEY = True # Configure item pipelines # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html #ITEM_PIPELINES = { -# '$project_name.pipelines.SomePipeline': 300, +# '$project_name.pipelines.${ProjectName}Pipeline': 300, #} # Enable and configure the AutoThrottle extension (disabled by default) From 8a86574394172a8221a95d9e2fcc229a26be2103 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 28 Dec 2016 14:10:50 +0000 Subject: [PATCH 05/91] .devN release suffix must be preceded with a dot https://packaging.python.org/distributing/#standards-compliance-for-interoperability --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 506f3779b..ac79a0425 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ branches: only: - master - /^\d\.\d+$/ - - /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/ + - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ env: - TOXENV=py27 - TOXENV=jessie @@ -35,4 +35,4 @@ deploy: on: tags: true repo: scrapy/scrapy - condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|dev[0-9]+)?$" + condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$" From a21473147160e4b176fedb32c92e1b68b4af04f6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 6 Jan 2017 15:38:35 +0100 Subject: [PATCH 06/91] Add Python 3.6 tox env + Travis CI build for it --- .travis.yml | 1 + tox.ini | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 506f3779b..b0ac3afde 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ env: - TOXENV=jessie - TOXENV=py33 - TOXENV=py35 + - TOXENV=py36 - TOXENV=docs install: - pip install -U tox twine wheel codecov diff --git a/tox.ini b/tox.ini index 812302b4c..bdc14a128 100644 --- a/tox.ini +++ b/tox.ini @@ -70,6 +70,10 @@ deps = {[testenv:py33]deps} basepython = python3.5 deps = {[testenv:py33]deps} +[testenv:py36] +basepython = python3.6 +deps = {[testenv:py33]deps} + [docs] changedir = docs deps = From 40851a3d4cde6b44cc92e03b7272b3acd33ed7f6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 6 Jan 2017 15:44:32 +0100 Subject: [PATCH 07/91] Use Python 3.6-dev on Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b0ac3afde..5c9eb8cc7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -python: 3.5 +python: 3.6-dev sudo: false branches: only: From 53769245f553355ad1907ad0b47a05093ec65cbe Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 6 Jan 2017 16:02:14 +0100 Subject: [PATCH 08/91] Use python 3.6 directly --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5c9eb8cc7..4c4adb948 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -python: 3.6-dev +python: 3.6 sudo: false branches: only: From 6b838b02966902311ab869bb6a35e023265ed274 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 6 Jan 2017 16:10:14 +0100 Subject: [PATCH 09/91] Use matrix build config --- .travis.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4c4adb948..52fbf02ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,24 @@ language: python -python: 3.6 sudo: false branches: only: - master - /^\d\.\d+$/ - /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/ -env: - - TOXENV=py27 - - TOXENV=jessie - - TOXENV=py33 - - TOXENV=py35 - - TOXENV=py36 - - TOXENV=docs +matrix: + include: + - python: 2.7 + env: TOXENV=py27 + - python: 2.7 + env: TOXENV=jessie + - python: 3.3 + env: TOXENV=py33 + - python: 3.5 + env: TOXENV=py35 + - python: 3.6 + env: TOXENV=py36 + - python: 3.6 + env: TOXENV=docs install: - pip install -U tox twine wheel codecov script: tox From b3406677b980656751af9a12ddf2d33ad884fbc1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 9 Jan 2017 14:40:02 +0100 Subject: [PATCH 10/91] Update classifiers in setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f673b1dc4..388cf0dec 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ setup( 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', From df1a42419f8bce48b605087937320af1ec968116 Mon Sep 17 00:00:00 2001 From: Raul Gallegos Date: Sat, 14 Jan 2017 20:45:20 -0500 Subject: [PATCH 11/91] adding formid to FormRequest documentation --- docs/topics/request-response.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 664a7239f..a1bd1e146 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -358,7 +358,7 @@ fields with form data from :class:`Response` objects. The :class:`FormRequest` objects support the following class method in addition to the standard :class:`Request` methods: - .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) + .. classmethod:: FormRequest.from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) Returns a new :class:`FormRequest` object with its form field values pre-populated with those found in the HTML ``
`` element contained @@ -383,6 +383,9 @@ fields with form data from :class:`Response` objects. :param formname: if given, the form with name attribute set to this value will be used. :type formname: string + :param formid: if given, the form with id attribute set to this value will be used. + :type formid: string + :param formxpath: if given, the first form that matches the xpath will be used. :type formxpath: string @@ -421,6 +424,9 @@ fields with form data from :class:`Response` objects. .. versionadded:: 1.1.0 The ``formcss`` parameter. + .. versionadded:: 1.1.0 + The ``formid`` parameter. + Request usage examples ---------------------- From b279bc8546994f5610836e63490b4a7a262b3d77 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 19 Jan 2017 16:28:52 +0100 Subject: [PATCH 12/91] Fix view command against new --no-redirect option --- scrapy/commands/view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 4eb44f77d..59592d08b 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -11,7 +11,7 @@ class Command(fetch.Command): "contents in a browser" def add_options(self, parser): - ScrapyCommand.add_options(self, parser) + super(Command, self).add_options(parser) parser.add_option("--spider", dest="spider", help="use this spider") From 299544416a05c126838db3c75b6fc154b6a08de1 Mon Sep 17 00:00:00 2001 From: Raul Gallegos Date: Sun, 22 Jan 2017 19:23:44 -0500 Subject: [PATCH 13/91] changing README to README.rst --- artwork/{README => README.rst} | 2 ++ docs/{README => README.rst} | 2 ++ sep/{README => README.rst} | 2 ++ 3 files changed, 6 insertions(+) rename artwork/{README => README.rst} (97%) rename docs/{README => README.rst} (99%) rename sep/{README => README.rst} (95%) diff --git a/artwork/README b/artwork/README.rst similarity index 97% rename from artwork/README rename to artwork/README.rst index c185d57da..016462f2c 100644 --- a/artwork/README +++ b/artwork/README.rst @@ -1,3 +1,5 @@ +:orphan: + Scrapy artwork ============== diff --git a/docs/README b/docs/README.rst similarity index 99% rename from docs/README rename to docs/README.rst index cf04965ac..733af2af4 100644 --- a/docs/README +++ b/docs/README.rst @@ -1,3 +1,5 @@ +:orphan: + ====================================== Scrapy documentation quick start guide ====================================== diff --git a/sep/README b/sep/README.rst similarity index 95% rename from sep/README rename to sep/README.rst index 668772492..e2d2e6274 100644 --- a/sep/README +++ b/sep/README.rst @@ -1,3 +1,5 @@ +:orphan: + Scrapy Enhancement Proposals ============================ From 53757e51e56ace34bef8616451cca921da16791c Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 24 Jan 2017 11:29:11 -0300 Subject: [PATCH 14/91] Preserve request class when converting to/from dicts (utils.reqser) --- scrapy/utils/reqser.py | 6 +++++- tests/test_utils_reqser.py | 13 ++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 7e1e99e48..2fceb0d94 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -5,6 +5,7 @@ import six from scrapy.http import Request from scrapy.utils.python import to_unicode, to_native_str +from scrapy.utils.misc import load_object def request_to_dict(request, spider=None): @@ -32,6 +33,8 @@ def request_to_dict(request, spider=None): 'priority': request.priority, 'dont_filter': request.dont_filter, } + if type(request) is not Request: + d['_class'] = request.__module__ + '.' + request.__class__.__name__ return d @@ -47,7 +50,8 @@ def request_from_dict(d, spider=None): eb = d['errback'] if eb and spider: eb = _get_method(spider, eb) - return Request( + request_cls = load_object(d['_class']) if '_class' in d else Request + return request_cls( url=to_native_str(d['url']), callback=cb, errback=eb, diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index a62f13e21..5b889ab5d 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import unittest -from scrapy.http import Request +from scrapy.http import Request, FormRequest from scrapy.spiders import Spider from scrapy.utils.reqser import request_to_dict, request_from_dict @@ -42,6 +42,7 @@ class RequestSerializationTest(unittest.TestCase): self._assert_same_request(request, request2) def _assert_same_request(self, r1, r2): + self.assertEqual(r1.__class__, r2.__class__) self.assertEqual(r1.url, r2.url) self.assertEqual(r1.callback, r2.callback) self.assertEqual(r1.errback, r2.errback) @@ -54,6 +55,12 @@ class RequestSerializationTest(unittest.TestCase): self.assertEqual(r1.priority, r2.priority) self.assertEqual(r1.dont_filter, r2.dont_filter) + def test_request_class(self): + r = FormRequest("http://www.example.com") + self._assert_serializes_ok(r, spider=self.spider) + r = CustomRequest("http://www.example.com") + self._assert_serializes_ok(r, spider=self.spider) + def test_callback_serialization(self): r = Request("http://www.example.com", callback=self.spider.parse_item, errback=self.spider.handle_error) @@ -77,3 +84,7 @@ class TestSpider(Spider): def handle_error(self, failure): pass + + +class CustomRequest(Request): + pass From bae12870bbb4e6f589887ad6036e50e5db64369f Mon Sep 17 00:00:00 2001 From: Michael Fladischer Date: Tue, 24 Jan 2017 22:20:37 +0100 Subject: [PATCH 15/91] Fix spelling error in scrapy.1 manpage. The word "intepreted" should be "interpreted". --- extras/scrapy.1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extras/scrapy.1 b/extras/scrapy.1 index a4f29569b..84f693fc3 100644 --- a/extras/scrapy.1 +++ b/extras/scrapy.1 @@ -28,16 +28,16 @@ Query Scrapy settings Print raw setting value .TP .I --getbool=SETTING -Print setting value, intepreted as a boolean +Print setting value, interpreted as a boolean .TP .I --getint=SETTING -Print setting value, intepreted as an integer +Print setting value, interpreted as an integer .TP .I --getfloat=SETTING -Print setting value, intepreted as an float +Print setting value, interpreted as an float .TP .I --getlist=SETTING -Print setting value, intepreted as an float +Print setting value, interpreted as an float .TP .I --init Print initial setting value (before loading extensions and spiders) From 87472346df6ce0ca63842e50d548a423b606767d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 25 Jan 2017 11:28:20 +0100 Subject: [PATCH 16/91] Update scrapy.1 --- extras/scrapy.1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extras/scrapy.1 b/extras/scrapy.1 index 84f693fc3..2fa8d8231 100644 --- a/extras/scrapy.1 +++ b/extras/scrapy.1 @@ -34,10 +34,10 @@ Print setting value, interpreted as a boolean Print setting value, interpreted as an integer .TP .I --getfloat=SETTING -Print setting value, interpreted as an float +Print setting value, interpreted as a float .TP .I --getlist=SETTING -Print setting value, interpreted as an float +Print setting value, interpreted as a float .TP .I --init Print initial setting value (before loading extensions and spiders) From fc07711614549ce8f0464b3fca5b0f4acd746681 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 30 Jan 2017 15:54:28 +0100 Subject: [PATCH 17/91] Remove unused --headers option for view command --- scrapy/commands/view.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 59592d08b..59e665016 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -12,8 +12,7 @@ class Command(fetch.Command): def add_options(self, parser): super(Command, self).add_options(parser) - parser.add_option("--spider", dest="spider", - help="use this spider") + parser.remove_option("--headers") def _print_response(self, response, opts): open_in_browser(response) From 4156a86148f07c108d3ea4248a2d4a7e2a58ffa9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 30 Jan 2017 15:57:37 +0100 Subject: [PATCH 18/91] Update docs on view command --- docs/topics/commands.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 6636c30cb..eaeeee113 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -358,6 +358,12 @@ Opens the given URL in a browser, as your Scrapy spider would "see" it. Sometimes spiders see pages differently from regular users, so this can be used to check what the spider "sees" and confirm it's what you expect. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) + Usage example:: $ scrapy view http://www.example.com/some/page.html From ae6d8d728e12e8efd704ba529de1db2eacfb494f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 30 Jan 2017 16:33:08 +0100 Subject: [PATCH 19/91] Support 'True' and 'False' strings as boolean settings values --- scrapy/settings/__init__.py | 16 +++++++++++++--- tests/test_settings/__init__.py | 12 ++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 7b7808959..28446a372 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -114,8 +114,8 @@ class BaseSettings(MutableMapping): """ Get a setting value as a boolean. - ``1``, ``'1'``, and ``True`` return ``True``, while ``0``, ``'0'``, - ``False`` and ``None`` return ``False``. + ``1``, ``'1'``, `True`` and ``'True'`` return ``True``, + while ``0``, ``'0'``, ``False``, ``'False'`` and ``None`` return ``False``. For example, settings populated through environment variables set to ``'0'`` will return ``False`` when using this method. @@ -126,7 +126,17 @@ class BaseSettings(MutableMapping): :param default: the value to return if no setting is found :type default: any """ - return bool(int(self.get(name, default))) + got = self.get(name, default) + try: + return bool(int(got)) + except ValueError: + if got in ("True", "true"): + return True + if got in ("False", "false"): + return False + raise ValueError("Supported values for boolean settings " + "are 0/1, True/False, '0'/'1', " + "'True'/'False' and 'true'/'false'") def getint(self, name, default=0): """ diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 4acf22cba..863684075 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -211,9 +211,15 @@ class BaseSettingsTest(unittest.TestCase): 'TEST_ENABLED1': '1', 'TEST_ENABLED2': True, 'TEST_ENABLED3': 1, + 'TEST_ENABLED4': 'True', + 'TEST_ENABLED5': 'true', + 'TEST_ENABLED_WRONG': 'on', 'TEST_DISABLED1': '0', 'TEST_DISABLED2': False, 'TEST_DISABLED3': 0, + 'TEST_DISABLED4': 'False', + 'TEST_DISABLED5': 'false', + 'TEST_DISABLED_WRONG': 'off', 'TEST_INT1': 123, 'TEST_INT2': '123', 'TEST_FLOAT1': 123.45, @@ -231,11 +237,15 @@ class BaseSettingsTest(unittest.TestCase): self.assertTrue(settings.getbool('TEST_ENABLED1')) self.assertTrue(settings.getbool('TEST_ENABLED2')) self.assertTrue(settings.getbool('TEST_ENABLED3')) + self.assertTrue(settings.getbool('TEST_ENABLED4')) + self.assertTrue(settings.getbool('TEST_ENABLED5')) self.assertFalse(settings.getbool('TEST_ENABLEDx')) self.assertTrue(settings.getbool('TEST_ENABLEDx', True)) self.assertFalse(settings.getbool('TEST_DISABLED1')) self.assertFalse(settings.getbool('TEST_DISABLED2')) self.assertFalse(settings.getbool('TEST_DISABLED3')) + self.assertFalse(settings.getbool('TEST_DISABLED4')) + self.assertFalse(settings.getbool('TEST_DISABLED5')) self.assertEqual(settings.getint('TEST_INT1'), 123) self.assertEqual(settings.getint('TEST_INT2'), 123) self.assertEqual(settings.getint('TEST_INTx'), 0) @@ -258,6 +268,8 @@ class BaseSettingsTest(unittest.TestCase): self.assertEqual(settings.getdict('TEST_DICT3'), {}) self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5}) self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1') + self.assertRaises(ValueError, settings.getbool, 'TEST_ENABLED_WRONG') + self.assertRaises(ValueError, settings.getbool, 'TEST_DISABLED_WRONG') def test_getpriority(self): settings = BaseSettings({'key': 'value'}, priority=99) From d2e9ea0c88b7578c5fc8d4d37e5df9d078e9b884 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 13 Jan 2017 16:17:51 +0100 Subject: [PATCH 20/91] Enforce DNS resolution timeout --- scrapy/resolver.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 3954fd977..4f4f0b04f 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -16,8 +16,11 @@ class CachingThreadedResolver(ThreadedResolver): def getHostByName(self, name, timeout=None): if name in dnscache: return defer.succeed(dnscache[name]) - if not timeout: - timeout = self.timeout + # in Twisted<=16.6, getHostByName() is always called with + # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple), + # so the input argument above is simply overridden + # to enforce Scrapy's DNS_TIMEOUT setting's value + timeout = (self.timeout,) d = super(CachingThreadedResolver, self).getHostByName(name, timeout) d.addCallback(self._cache_result, name) return d From a58624375824d821f0bdadb7bc04fad53171ca70 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 13 Jan 2017 15:51:44 +0100 Subject: [PATCH 21/91] Buffer CONNECT response bytes from proxy until all HTTP headers are received --- scrapy/core/downloader/handlers/http11.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index ecd7f90d3..4b02cb16f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -105,6 +105,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): self._tunneledHost = host self._tunneledPort = port self._contextFactory = contextFactory + self._connectBuffer = b'' def requestTunnel(self, protocol): """Asks the proxy to open a tunnel.""" @@ -121,8 +122,16 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): created, notifies the client that we are ready to send requests. If not raises a TunnelError. """ + self._connectBuffer += rcvd_bytes + # make sure that enough (all) bytes are consumed + # and that we've got all HTTP headers (ending with a blank line) + # from the proxy so that we don't send those bytes to the TLS layer + # + # see https://github.com/scrapy/scrapy/issues/2491 + if b'\r\n\r\n' not in self._connectBuffer: + return self._protocol.dataReceived = self._protocolDataReceived - respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(rcvd_bytes) + respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) if respm and int(respm.group('status')) == 200: try: # this sets proper Server Name Indication extension From 8c4f614d2101c21d3560fe68dcec6d3910417cb7 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 1 Sep 2016 15:25:38 +0200 Subject: [PATCH 22/91] Enable PyPy tests on Travis --- .travis.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6f3ab511f..2df02ea43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,10 +17,30 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 + - python: 2.7 + env: TOXENV=pypy - python: 3.6 env: TOXENV=docs + allow_failures: + - python: 2.7 + env: TOXENV=pypy install: - - pip install -U tox twine wheel codecov + - | + if [ "$TOXENV" = "pypy" ]; then + export PYENV_ROOT="$HOME/.pyenv" + if [ -f "$PYENV_ROOT/bin/pyenv" ]; then + pushd "$PYENV_ROOT" && git pull && popd + else + rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT" + fi + # get latest PyPy from pyenv directly (thanks to natural version sort option -V) + export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-[0-9][\.0-9]*$' |sort -V |tail -1` + "$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION" + virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION" + source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" + fi + - pip install -U tox twine wheel codecov + script: tox after_success: - codecov From 70f260d3537d788e4efaf4a060fce7f0fc183ed9 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 1 Feb 2017 15:14:32 +0100 Subject: [PATCH 23/91] Don't run coverage stats when on PyPy --- tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index bdc14a128..0fdea11bb 100644 --- a/tox.ini +++ b/tox.ini @@ -54,6 +54,11 @@ commands = pip install -U https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib py.test --cov=scrapy --cov-report= {posargs:scrapy tests} +[testenv:pypy] +basepython = pypy +commands = + py.test {posargs:scrapy tests} + [testenv:py33] basepython = python3.3 deps = From 55742c0392e8c582ed622f0ea28d8d62ece2c401 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 1 Feb 2017 22:43:28 +0500 Subject: [PATCH 24/91] DOC mention LevelDB cache storage backend --- docs/topics/downloader-middleware.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 3b9a5335a..912671d19 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -318,10 +318,11 @@ HttpCacheMiddleware This middleware provides low-level cache to all HTTP requests and responses. It has to be combined with a cache storage backend as well as a cache policy. - Scrapy ships with two HTTP cache storage backends: + Scrapy ships with three HTTP cache storage backends: * :ref:`httpcache-storage-fs` * :ref:`httpcache-storage-dbm` + * :ref:`httpcache-storage-leveldb` You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE` setting. Or you can also implement your own storage backend. From 0cf6344cc22a193079ea1fd2fbe28187cc212c7a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 9 Dec 2016 17:58:38 +0100 Subject: [PATCH 25/91] Support kwargs for response.xpath() --- scrapy/http/response/text.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index afa430329..5a6507aa8 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -111,8 +111,8 @@ class TextResponse(Response): self._cached_selector = Selector(self) return self._cached_selector - def xpath(self, query): - return self.selector.xpath(query) + def xpath(self, query, **kwargs): + return self.selector.xpath(query, **kwargs) def css(self, query): return self.selector.css(query) From 803d8c4b5709d792f1c314bbde124e69903ff23f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 20 Dec 2016 11:26:42 +0100 Subject: [PATCH 26/91] Add tests for passing kwargs on response .xpath() shortcut --- tests/test_http_response.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 7624aa4c4..9df3bf6e7 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -320,6 +320,20 @@ class TextResponseTest(BaseResponseTest): response.selector.css("title::text").extract(), ) + def test_selector_shortcuts_kwargs(self): + body = b"Some page

A nice paragraph.

" + response = self.response_class("http://www.example.com", body=body) + + self.assertEqual( + response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(), + response.xpath("normalize-space(//p[@class=\"content\"])").extract(), + ) + self.assertEqual( + response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()", + pclass="content", pcount=1).extract(), + response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(), + ) + def test_urljoin_with_base_url(self): """Test urljoin shortcut which also evaluates base-url through get_base_url().""" body = b'' @@ -428,3 +442,21 @@ class XmlResponseTest(TextResponseTest): response.xpath("//elem/text()").extract(), response.selector.xpath("//elem/text()").extract(), ) + + def test_selector_shortcuts_kwargs(self): + body = b''' + + value + ''' + response = self.response_class("http://www.example.com", body=body) + + self.assertEqual( + response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(), + response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(), + ) + + response.selector.register_namespace('s2', 'http://scrapy.org') + self.assertEqual( + response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(), + response.selector.xpath("//s2:elem/text()").extract(), + ) From 1c0b8053579e9680e4bd4c788a0a2bdf88a8f175 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 11 Jan 2017 19:44:21 +0100 Subject: [PATCH 27/91] DOC Mention XPath variables in Selectors section --- docs/topics/selectors.rst | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 39ec9b73c..43370d479 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -283,6 +283,40 @@ XPath specification. .. _Location Paths: https://www.w3.org/TR/xpath#location-paths +.. _topics-selectors-xpath-variables: + +Variables in XPath expressions +------------------------------ + +XPath allows you to reference variables in your XPath expressions, using +the ``$somevariable`` syntax. This is somewhat similar to parameterized +queries or prepared statements in the SQL world where you replace +some arguments in your queries with placeholders like ``?``, +which are then substituted with values passed with the query. + +Here's an example to match an element based on its "id" attribute value, +without hard-coding it (that was shown previously):: + + >>> # `$val` used in the expression, a `val` argument needs to be passed + >>> response.xpath('//div[@id=$val]/a/text()', val='images').extract_first() + u'Name: My image 1 ' + +Here's another example, to find the "id" attribute of a ``
`` tag containing +five ```` children (here we pass the value ``5`` as an integer):: + + >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).extract_first() + u'images' + +All variable references must have a binding value when calling ``.xpath()`` +(otherwise you'll get a ``ValueError: XPath error:`` exception). +This is done by passing as many named arguments as necessary. + +`parsel`_, the library powering Scrapy selectors, has more details and examples +on `XPath variables`_. + +.. _parsel: https://parsel.readthedocs.io/ +.. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions + Using EXSLT extensions ---------------------- From 1295c17a26544f451261f4c6558b344162f3c4d4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Feb 2017 17:47:22 +0100 Subject: [PATCH 28/91] Bump parsel requirement to at least parsel v1.1 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 64b6e771c..f92603d3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ queuelib six>=1.5.2 PyDispatcher>=2.0.5 service_identity -parsel>=0.9.5 +parsel>=1.1 diff --git a/setup.py b/setup.py index 388cf0dec..a6e6f9615 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ setup( 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', - 'parsel>=0.9.5', + 'parsel>=1.1', 'PyDispatcher>=2.0.5', 'service_identity', ], From 3358254c5c78a1b8de1d02cb3b1db85fe824c798 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Feb 2017 17:53:28 +0100 Subject: [PATCH 29/91] Make DNS retry test compatible with Twisted 17+ --- tests/test_crawl.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 1b4a4b3b0..c7f4c0e35 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -91,12 +91,11 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_retry_dns_error(self): - with mock.patch('socket.gethostbyname', - side_effect=socket.gaierror(-5, 'No address associated with hostname')): - crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl("http://example.com/") - self._assert_retried(l) + crawler = self.runner.create_crawler(SimpleSpider) + with LogCapture() as l: + # try to fetch the homepage of a non-existent domain + yield crawler.crawl("http://dns.resolution.invalid/") + self._assert_retried(l) @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): From e604c0f3abeafdd896606960170ab5bad699d79d Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Feb 2017 18:26:41 +0100 Subject: [PATCH 30/91] Remove unused imports --- tests/test_crawl.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index c7f4c0e35..0c64948fa 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,5 +1,4 @@ import json -import socket import logging from testfixtures import LogCapture @@ -9,7 +8,6 @@ from twisted.trial.unittest import TestCase from scrapy.http import Request from scrapy.crawler import CrawlerRunner from scrapy.utils.python import to_unicode -from tests import mock from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \ BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider from tests.mockserver import MockServer From 02e1d2b1fd0257a926f438a7758a9e65a0114325 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 2 Feb 2017 22:28:37 +0100 Subject: [PATCH 31/91] Add trailing dot --- tests/test_crawl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 0c64948fa..d5babdded 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -92,7 +92,7 @@ class CrawlTestCase(TestCase): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: # try to fetch the homepage of a non-existent domain - yield crawler.crawl("http://dns.resolution.invalid/") + yield crawler.crawl("http://dns.resolution.invalid./") self._assert_retried(l) @defer.inlineCallbacks From 09643796b4d6ef25b23c1d7366d22ca55bd091d8 Mon Sep 17 00:00:00 2001 From: Lukas Anzinger Date: Fri, 3 Feb 2017 20:05:17 +0100 Subject: [PATCH 32/91] Fix typo in downloader-middleware.rst. --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 3b9a5335a..2e32eb280 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -748,7 +748,7 @@ REDIRECT_MAX_TIMES Default: ``20`` -The maximum number of redirections that will be follow for a single request. +The maximum number of redirections that will be followed for a single request. MetaRefreshMiddleware --------------------- From 3021084f376a643f129388f76f039ab1301abb11 Mon Sep 17 00:00:00 2001 From: djrobust Date: Sat, 4 Feb 2017 20:07:05 -0800 Subject: [PATCH 33/91] Use 'yield' when parsing multiple responses Use 'yield' consistently across examples of parse functions. --- docs/topics/request-response.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 664a7239f..3853f5935 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -207,12 +207,12 @@ different fields from different pages:: request = scrapy.Request("http://www.example.com/some_page.html", callback=self.parse_page2) request.meta['item'] = item - return request + yield request def parse_page2(self, response): item = response.meta['item'] item['other_url'] = response.url - return item + yield item .. _topics-request-response-ref-errbacks: From fcb3daf4fa3e4d97e9e66462395a021b3bc4363d Mon Sep 17 00:00:00 2001 From: Takehiro Shiozaki Date: Mon, 6 Feb 2017 14:03:41 +0900 Subject: [PATCH 34/91] fix typo --- docs/topics/spider-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 604f1864f..8360827e8 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -112,7 +112,7 @@ following methods: .. method:: process_spider_exception(response, exception, spider) - This method is called when when a spider or :meth:`process_spider_input` + This method is called when a spider or :meth:`process_spider_input` method (from other spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an From 16c4b4e184c1b213eafacecca0097ff44effa696 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Mon, 6 Feb 2017 11:41:08 +0100 Subject: [PATCH 35/91] [httpcompression] add support for br - brotli content encoding --- requirements.txt | 1 + scrapy/downloadermiddlewares/httpcompression.py | 6 +++++- tests/sample_data/compressed/html-br.bin | Bin 0 -> 4027 bytes .../test_downloadermiddleware_httpcompression.py | 13 ++++++++++++- 4 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 tests/sample_data/compressed/html-br.bin diff --git a/requirements.txt b/requirements.txt index f92603d3d..362d05013 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ six>=1.5.2 PyDispatcher>=2.0.5 service_identity parsel>=1.1 +brotlipy==0.6 diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index bcf20f10c..9202fd8da 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,5 +1,7 @@ import zlib +import brotli + from scrapy.utils.gz import gunzip, is_gzipped from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes @@ -17,7 +19,7 @@ class HttpCompressionMiddleware(object): return cls() def process_request(self, request, spider): - request.headers.setdefault('Accept-Encoding', 'gzip,deflate') + request.headers.setdefault('Accept-Encoding', 'gzip,deflate,br') def process_response(self, request, response, spider): @@ -55,5 +57,7 @@ class HttpCompressionMiddleware(object): # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx # http://www.gzip.org/zlib/zlib_faq.html#faq38 body = zlib.decompress(body, -15) + if encoding == b"br": + body = brotli.decompress(body) return body diff --git a/tests/sample_data/compressed/html-br.bin b/tests/sample_data/compressed/html-br.bin new file mode 100644 index 0000000000000000000000000000000000000000..c7eea4bb826a1a1851d047e64dd98423138cf996 GIT binary patch literal 4027 zcmV;s4@B@=R3tGHVT)KTxseDvm_{W4YMH#TqFGI-3`~9)H9xksFwjQ_FSnd3&0Y8= z@_ThUt$zy9SegPwSgY7y5IE>Uo9&E0Ix0^mRs=<3xGiyIT}Yr9fO2kCiRXNW-0{i+Lu3MI!yIoUSf-Z226AD}r0^ zdA#v**@y{+8$OqPhc=zMEOz;5{^K$+BPXfRJ1|wzaHbBHwx3!Jh1BbRnT%?EOzxB` zc@o=4lizjLI=g3zEFMR~-@xbc=^ZH>n1uLgXP=Rip&%H2-eg z$?YM9bB=&I5l()^leA~+n16V4g_^DL-W4w{9=Ts$j!|s*p|9)8b6}Tq?$r33#pv6QdYC{6y7(EBcDcrg)1%XlDFWLE zIxI2y!eR*(8>O@}u3Hx&$4JYjr`~J*l_%PUM_=!22LDWBb0`w>VTQ3!mL%lo=0kh> z;=f_ZiDy6Qi&t|`{$gtJAKjVrTHwK@8XlMYrk*v45K zR(OuASJjF_HuS)^Z3_;}-x*kbam!jCxmOBSZRDe}&Ao{=EGz4)%9_~X?Reh+%sac7 zGuk9hsij`&OiZ72ftz#WrA9ZS6`R$ubDx3drEzSlcoi3GIyjXkI?`smByM(WtKW-H zWt7nbG9EvTV{l{PV!0a@LvYsd{#$Oc7=PHO*Tvu0%JJ>8p$>ryS1;SQzlXd;_<=@yK5Wg8buXuZIS?L!lfDPB6Z^`*eg zutTl2T*u?5VH;d&i+*2*Ep~8PI;ShzaWl4DOzYKVhmHR_p&Q&md=hBvsdBnWMe#f`~|qjj-zg?1*K|BSawH>RT%cD#arQ7^TI z6KvdkI?!mmV|~lieEkzilzWsWq#SbmLLQWoW|ul_2z*TXMh3@&sY}_3+cF$~=~WaRuLRH zUEPy7hC`>~;lwtyrg)YdG@M|=iOVCKn3mX%);O^=MJtD6aba>hXdREsw_u5NhA+4< z^sKWQ6|@y+i+U>g{@V(_mUeYdG6iL&*_d)M3>B2UhLP}Q%IjQ-WWddTE~_uq2D4R*$|#Mo;&1hp{C*{ib`vZ;l|QC2kzlp zcW8qz7Ow>GBmI5eG5r=Ij!~P~x`Wh@iT^RIF&vn>rIiM8`^4oaS=lALY>4}Rd9{?x z2~pZ28W!2T#&BcGbi5XgW9h~am$ddO1B-hYldX6)DLps$m>P*^fkBPe_i$Q=Q0L+p zOUieVrI$zYIdT_{Z#*_}8RPXWf?%)XCZqB)4+m?*aBFFV!egD_UB)01g%>2+HtLY+JuQ7OeW5{mY#yF<9sft#; zm%%GGhDL{_4vFSs8EZt&f=bTu_)ucGPgpfh2}P@v-36Ci;&yf@H5?~v3Z8`Zj(6eG zjSk}BrDb}So@WuVuC8=js+hJX*q5eTw&5(OwWr;~oDPn5VsJ{uu7xQSKm$c*Z({v2&xOGa_`jLDebo8PVxh;rI z2UgQBA(pbl-m4reT$bA~mr(motI4NglL8WtC&D?=w#mzb8)5KR1N}Ykemgi!- z3hdhf_2FiLf%Ql}hp$6dS8>!O%UY!A9R`oCJ6Jks!6_NlBmI5QXm&re)=b=%c&$oo zvx-AU(;S);+M$Ei>6-SN1vad4`F?b$H7Xd}F3~E3_3QWlz>ediMS+aksu44b-GoQ) z_&Q)MV!KOdT4mwF(zEQ2%TJ&mUDTA{EHF4Q_NzmuL(_ro5-$?Arx)GPNe8L3<839- zdL-WmUT$2P)9h1}+xS^HyHySzOzlc=0=?-@fxP-Kv%toX^4065!>~oPTM_!b1ZeB- z;e;NwQvhF$YSuK1-NKDqR~;hW0_t^vs}xL3%}HHQes@Ud%o7b~feS<9!x?UK58t-5 zSC{MkX>8-n*BAU+-r&+iR~?29txo6-4RlD~c>J{JU};j)>^G(5!G7VwrHMu{Y-(iw zsbJ`xoInkJa=s}Cx9;kkOioxY z9R%4f##`(GFWl~~E}$ul`~eUzbZ=NMu<51=K(I^Fdd;(|*!Khv9lLfQIJ&2LFDE&? z)~qCzEv7b`BAX`>Uc%H}_i=W!ho#^j20V6#$7;*ba;7oQZ!n*1cd1l&a4}Tb_O0hT zT%VKq=MOB;ZtceUZb|pcSC4UwJ5Pvze0Ak9i`Z1ZCA-|C{*PIiO2tBNT!#C4Np@;! zlWnIr#`Pw-uM&~yvo21Luh=1OOKh@t-LC5~S*)BW^o$9e`+u%i$?RI&WS@1B|35Cd zrc$ACKL1O#xqi7`l5w@P$(}gwD^6UMh(^D4dyLCDC%(RK{hw_a?7nk7C1)!)4n9}2 zUNMV#MaHNd+ijA)cfCqpRXQ(T9UhF=;$`!{+5eGeYej!9r=E}L9BZSJIWS$^KJgyO z?5-{OOOAIQHs?8axl@UJa9g(NK6c34F1BG%J!c)MtSQ{n23odvO^y#?e^HmQeB(eFigv0KW8rG_HO78cg@E%tYcSQJK1o1Sbbw2>9n`k zXS*!Jb{$K*5^>NQR==~1V6~^ewmy{XuR*ib+Bo|Dt;c^*OW7L*yOfBC|Czkr9rW1Fk z6^QfxUoq#j8RM;@w718$4HtZ#JG%YXIrYc?t7+D1(&&&Twr~8$A|sXz0u_EqyI~jtXW;#@LC?1Q?(w~+C-vH zG%sUQ%NbAANSU*0+v7GZmtEC{Ypt_jOSwnPJMJ1KC)cdXwp=d5WmT2!T4QdwgsADf zJmac@lrgJi6MbViF5*^;%e97R^hBF!P4x6!&4$kD+&ndBdp(A2I#uH_*OCXeI+xYC z*NZ;y|5NAe>e_D0uq=<|byX@aOjU+eS+QMm(&f$S*e=5`+?HW^-D?iPKcoNK9I}nR1?6ZQJp>9NXnJOnXiF@OqZ=HLuIy^o!wIrES}$9BR+`f3`L;*}GKd3%s7D z{hHf!+_uMDg>9LZlo79GsdsjA;q?^F*G#v^Wt!}#a<*Z(U50HKhOw4NeEQCV0i4eb zyY;$Ws$|XT+OKD=cs`cpcs-TMg2&@ha?bSH>dadm+m>^$*LK-%(_V9KxZRe=Ww=e3 zYqf22)^=0hYx4H&5AP~Vm2f*N*>IV*+wGW!*S#v+aJj51uzF(~Ym9@wH>2|YglgW( zYj~xMT`imGIm=~n`kZZ6Dla^aVHmdKHjGu+_PpEFJ@ZxbRBkWy=e(SL^Wyxt9JoEE zV_6=Tr&hKlXT)omwqYA<$p_PI&r`py!R~WX&aAHOvRt<1vb<)Y&I1@Tx2ZdGGYr#X z7^_3~o~x##w(q&*l;qtt=YnInEXy%0Z>?;W^uy8!P|j4f_LbK_s`0D? Date: Mon, 6 Feb 2017 12:17:45 +0100 Subject: [PATCH 36/91] [httpcompression] import brotli when available --- requirements.txt | 1 - scrapy/downloadermiddlewares/httpcompression.py | 17 ++++++++++++----- ...test_downloadermiddleware_httpcompression.py | 3 ++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 362d05013..f92603d3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,3 @@ six>=1.5.2 PyDispatcher>=2.0.5 service_identity parsel>=1.1 -brotlipy==0.6 diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 9202fd8da..04c9e355d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,5 @@ import zlib -import brotli from scrapy.utils.gz import gunzip, is_gzipped from scrapy.http import Response, TextResponse @@ -8,10 +7,18 @@ from scrapy.responsetypes import responsetypes from scrapy.exceptions import NotConfigured +ACCEPTED_ENCODINGS = [b'gzip', b'deflate'] + +try: + import brotli + ACCEPTED_ENCODINGS.append(b'br') +except ImportError: + pass + + class HttpCompressionMiddleware(object): """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): @@ -19,7 +26,8 @@ class HttpCompressionMiddleware(object): return cls() def process_request(self, request, spider): - request.headers.setdefault('Accept-Encoding', 'gzip,deflate,br') + request.headers.setdefault('Accept-Encoding', + ",".join(ACCEPTED_ENCODINGS)) def process_response(self, request, response, spider): @@ -57,7 +65,6 @@ class HttpCompressionMiddleware(object): # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx # http://www.gzip.org/zlib/zlib_faq.html#faq38 body = zlib.decompress(body, -15) - if encoding == b"br": + if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: body = brotli.decompress(body) return body - diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index c47de54ed..b47b267e2 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,6 +1,6 @@ from io import BytesIO from unittest import TestCase -from os.path import join, abspath, dirname +from os.path import join from gzip import GzipFile from scrapy.spiders import Spider @@ -20,6 +20,7 @@ FORMAT = { 'br': ('html-br.bin', 'br') } + class HttpCompressionTest(TestCase): def setUp(self): From 3daf473686aab89aa03ebd0ebc59a73b4b6e00f1 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Mon, 6 Feb 2017 12:29:33 +0100 Subject: [PATCH 37/91] [httpcompression] skip test if no brotli --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- tests/requirements.txt | 1 + tests/test_downloadermiddleware_httpcompression.py | 11 ++++++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 04c9e355d..2fc1bb8eb 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -27,7 +27,7 @@ class HttpCompressionMiddleware(object): def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', - ",".join(ACCEPTED_ENCODINGS)) + b",".join(ACCEPTED_ENCODINGS)) def process_response(self, request, response, spider): diff --git a/tests/requirements.txt b/tests/requirements.txt index 9d0c3c996..9baa4be21 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,6 +6,7 @@ pytest==2.9.2 pytest-twisted pytest-cov==2.2.1 jmespath +brotlipy==0.6 testfixtures # optional for shell wrapper tests bpython diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index b47b267e2..7924fb3b5 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,11 +1,12 @@ from io import BytesIO -from unittest import TestCase +from unittest import TestCase, SkipTest from os.path import join from gzip import GzipFile from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse -from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware +from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \ + ACCEPTED_ENCODINGS from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -53,7 +54,7 @@ class HttpCompressionTest(TestCase): assert 'Accept-Encoding' not in request.headers self.mw.process_request(request, self.spider) self.assertEqual(request.headers.get('Accept-Encoding'), - b'gzip,deflate,br') + b','.join(ACCEPTED_ENCODINGS)) def test_process_response_gzip(self): response = self._getresponse('gzip') @@ -66,6 +67,10 @@ class HttpCompressionTest(TestCase): assert 'Content-Encoding' not in newresponse.headers def test_process_response_br(self): + try: + import brotli + except ImportError: + raise SkipTest("no brotli") response = self._getresponse('br') request = response.request self.assertEqual(response.headers['Content-Encoding'], b'br') From af802bad14f833178bc4e03e3db3a86c44dcb735 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Mon, 6 Feb 2017 15:45:21 +0100 Subject: [PATCH 38/91] [httpcompression] add brotlipy for python 3 --- tests/requirements-py3.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index ed189c66c..d73a2300f 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -8,3 +8,4 @@ botocore # optional for shell wrapper tests bpython ipython +brotlipy==0.6 From f73eb715ac4374b1eafb1da3c1311c5a7c153b5e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 02:16:06 +0500 Subject: [PATCH 39/91] =?UTF-8?q?LinkExtractor:=20don=E2=80=99t=20check=20?= =?UTF-8?q?all=20regexes=20if=20one=20of=20them=20matches?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/linkextractors/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index f51934b00..e5d21e174 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -40,7 +40,7 @@ IGNORED_EXTENSIONS = [ _re_type = type(re.compile("", 0)) -_matches = lambda url, regexs: any((r.search(url) for r in regexs)) +_matches = lambda url, regexs: any(r.search(url) for r in regexs) _is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'} @@ -93,8 +93,8 @@ class FilteringLinkExtractor(object): if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): return False - allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True] - denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else [] + allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True] + denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] return any(allowed) and not any(denied) def _process_links(self, links): From 85a124970ad406c2680402256853dd3ff86d191f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 03:32:54 +0500 Subject: [PATCH 40/91] Enable memusage extension by default. Fixes GH-2187. --- docs/topics/settings.rst | 10 ++++++---- scrapy/settings/default_settings.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0515a9e0d..c1f488f73 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -827,13 +827,15 @@ Example:: MEMUSAGE_ENABLED ---------------- -Default: ``False`` +Default: ``True`` Scope: ``scrapy.extensions.memusage`` -Whether to enable the memory usage extension that will shutdown the Scrapy -process when it exceeds a memory limit, and also notify by email when that -happened. +Whether to enable the memory usage extension. This extension keeps track of +a peak memory used by the process (it writes it to stats). It can also +optionally shutdown the Scrapy process when it exceeds a memory limit +(see :setting:`MEMUSAGE_LIMIT_MB`), and notify by email when that happened +(see :setting:`MEMUSAGE_NOTIFY_MAIL`). See :ref:`topics-extensions-ref-memusage`. diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 24714a7a8..1cc169c9a 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -207,7 +207,7 @@ MEMDEBUG_ENABLED = False # enable memory debugging MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0 -MEMUSAGE_ENABLED = False +MEMUSAGE_ENABLED = True MEMUSAGE_LIMIT_MB = 0 MEMUSAGE_NOTIFY_MAIL = [] MEMUSAGE_REPORT = False From fb4ef21a1dcdc0c70fa80443e7379150639c42f6 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Tue, 7 Feb 2017 10:22:42 +0100 Subject: [PATCH 41/91] [httpcompression] minor style edits --- scrapy/downloadermiddlewares/httpcompression.py | 1 - tests/requirements-py3.txt | 2 +- tests/requirements.txt | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 2fc1bb8eb..19d6345e4 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,5 @@ import zlib - from scrapy.utils.gz import gunzip, is_gzipped from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index d73a2300f..51a25f5e5 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -8,4 +8,4 @@ botocore # optional for shell wrapper tests bpython ipython -brotlipy==0.6 +brotlipy diff --git a/tests/requirements.txt b/tests/requirements.txt index 9baa4be21..c1576a2e7 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,7 +6,7 @@ pytest==2.9.2 pytest-twisted pytest-cov==2.2.1 jmespath -brotlipy==0.6 +brotlipy testfixtures # optional for shell wrapper tests bpython From 24e82bfe75ad1a7b5ceb7a2840ecbae82d05268d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 23 Dec 2016 12:58:29 -0300 Subject: [PATCH 42/91] Validate values for components order --- scrapy/utils/conf.py | 9 +++++++++ tests/test_utils_conf.py | 21 +++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index e8af90f11..435e9a6b3 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,5 +1,6 @@ import os import sys +import numbers from operator import itemgetter import six @@ -34,6 +35,13 @@ def build_component_list(compdict, custom=None, convert=update_classpath): _check_components(compdict) return {convert(k): v for k, v in six.iteritems(compdict)} + def _validate_values(compdict): + """Fail if a value in the components dict is not a real number or None.""" + for name, value in six.iteritems(compdict): + if value is not None and not isinstance(value, numbers.Real): + raise ValueError('Invalid value {} for component {}, please provide ' \ + 'a real number or None instead'.format(value, name)) + # BEGIN Backwards compatibility for old (base, custom) call signature if isinstance(custom, (list, tuple)): _check_components(custom) @@ -43,6 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): compdict.update(custom) # END Backwards compatibility + _validate_values(compdict) compdict = without_none_values(_map_keys(compdict)) return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))] diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index dab41ac8d..f203c32ef 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -62,6 +62,27 @@ class BuildComponentListTest(unittest.TestCase): self.assertRaises(ValueError, build_component_list, duplicate_bs, convert=lambda x: x.lower()) + def test_valid_numbers(self): + # work well with None and numeric values + d = {'a': 10, 'b': None, 'c': 15, 'd': 5.0} + self.assertEqual(build_component_list(d, convert=lambda x: x), + ['d', 'a', 'c']) + d = {'a': 33333333333333333333, 'b': 11111111111111111111, 'c': 22222222222222222222} + self.assertEqual(build_component_list(d, convert=lambda x: x), + ['b', 'c', 'a']) + # raise exception for invalid values + d = {'one': '5'} + self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) + d = {'one': '1.0'} + self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) + d = {'one': [1, 2, 3]} + self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) + d = {'one': {'a': 'a', 'b': 2}} + self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) + d = {'one': 'lorem ipsum',} + self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) + + class UtilsConfTestCase(unittest.TestCase): From eaf62ab69c23459fa36dafe298bcfabc5952b7f7 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 17:56:43 +0500 Subject: [PATCH 43/91] cleanup MetaRefreshMiddleware: remove redundant check --- scrapy/downloadermiddlewares/redirect.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index db276eefb..26677e527 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -53,8 +53,10 @@ class BaseRedirectMiddleware(object): class RedirectMiddleware(BaseRedirectMiddleware): - """Handle redirection of requests based on response status and meta-refresh html tag""" - + """ + Handle redirection of requests based on response status + and meta-refresh html tag. + """ def process_response(self, request, response, spider): if (request.meta.get('dont_redirect', False) or response.status in getattr(spider, 'handle_httpstatus_list', []) or @@ -92,10 +94,9 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): not isinstance(response, HtmlResponse): return response - if isinstance(response, HtmlResponse): - interval, url = get_meta_refresh(response) - if url and interval < self._maxdelay: - redirected = self._redirect_request_using_get(request, url) - return self._redirect(redirected, request, spider, 'meta refresh') + interval, url = get_meta_refresh(response) + if url and interval < self._maxdelay: + redirected = self._redirect_request_using_get(request, url) + return self._redirect(redirected, request, spider, 'meta refresh') return response From 04b2f79e7a1316d29b2cb94c8fb2623e040b64ec Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 22:30:58 +0500 Subject: [PATCH 44/91] Remove code required to support ancient twisted versions. See GH-1887. --- scrapy/core/downloader/handlers/http.py | 8 +--- tests/mockserver.py | 39 +++++-------------- tests/test_downloader_handlers.py | 49 +++++++++--------------- tests/test_downloadermiddleware_retry.py | 5 +-- 4 files changed, 30 insertions(+), 71 deletions(-) diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 81da2615a..e4a7d8564 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,10 +1,6 @@ -from scrapy import twisted_version +from __future__ import absolute_import from .http10 import HTTP10DownloadHandler - -if twisted_version >= (11, 1, 0): - from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler -else: - HTTPDownloadHandler = HTTP10DownloadHandler +from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler # backwards compatibility diff --git a/tests/mockserver.py b/tests/mockserver.py index a40e2e501..e611cc3ec 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -2,34 +2,18 @@ from __future__ import print_function import sys, time, random, os, json from six.moves.urllib.parse import urlencode from subprocess import Popen, PIPE + from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource -from twisted.internet import reactor, defer, ssl -from scrapy import twisted_version +from twisted.web.test.test_webclient import PayloadResource +from twisted.web.server import GzipEncoderFactory +from twisted.web.resource import EncodingResourceWrapper +from twisted.internet import reactor, ssl +from twisted.internet.task import deferLater + from scrapy.utils.python import to_bytes, to_unicode -if twisted_version < (11, 0, 0): - def deferLater(clock, delay, func, *args, **kw): - def _cancel_method(): - _cancel_cb(None) - d.errback(Exception()) - - def _cancel_cb(result): - if cl.active(): - cl.cancel() - return result - - d = defer.Deferred() - d.cancel = _cancel_method - d.addCallback(lambda ignored: func(*args, **kw)) - d.addBoth(_cancel_cb) - cl = clock.callLater(delay, d.callback, None) - return d -else: - from twisted.internet.task import deferLater - - def getarg(request, name, default=None, type=None): if name in request.args: value = request.args[name][0] @@ -174,13 +158,8 @@ class Root(Resource): self.putChild(b"drop", Drop()) self.putChild(b"raw", Raw()) self.putChild(b"echo", Echo()) - - if twisted_version > (12, 3, 0): - from twisted.web.test.test_webclient import PayloadResource - from twisted.web.server import GzipEncoderFactory - from twisted.web.resource import EncodingResourceWrapper - self.putChild(b"payload", PayloadResource()) - self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) + self.putChild(b"payload", PayloadResource()) + self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) def getChild(self, name, request): return self diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 8d3b49d6a..6333efceb 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -17,7 +17,6 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \ from twisted.cred import portal, checkers, credentials from w3lib.url import path_to_file_uri -from scrapy import twisted_version from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler @@ -281,8 +280,6 @@ class Https10TestCase(Http10TestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" download_handler_cls = HTTP11DownloadHandler - if twisted_version < (11, 1, 0): - skip = 'HTTP1.1 not supported in twisted < 11.1.0' def test_download_without_maxsize_limit(self): request = Request(self.getURL('file')) @@ -366,8 +363,6 @@ class Https11InvalidDNSId(Https11TestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" - if twisted_version < (11, 1, 0): - skip = 'HTTP1.1 not supported in twisted < 11.1.0' def setUp(self): self.mockserver = MockServer() @@ -396,31 +391,27 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download_gzip_response(self): + crawler = get_crawler(SingleRequestSpider) + body = b'1' * 100 # PayloadResource requires body length to be 100 + request = Request('http://localhost:8998/payload', method='POST', + body=body, meta={'download_maxsize': 50}) + yield crawler.crawl(seed=request) + failure = crawler.spider.meta['failure'] + # download_maxsize < 100, hence the CancelledError + self.assertIsInstance(failure.value, defer.CancelledError) - if twisted_version > (12, 3, 0): - - crawler = get_crawler(SingleRequestSpider) - body = b'1'*100 # PayloadResource requires body length to be 100 - request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50}) + if six.PY2: + request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') + request = request.replace(url='http://localhost:8998/xpayload') yield crawler.crawl(seed=request) - failure = crawler.spider.meta['failure'] - # download_maxsize < 100, hence the CancelledError - self.assertIsInstance(failure.value, defer.CancelledError) - - if six.PY2: - request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') - request = request.replace(url='http://localhost:8998/xpayload') - yield crawler.crawl(seed=request) - # download_maxsize = 50 is enough for the gzipped response - failure = crawler.spider.meta.get('failure') - self.assertTrue(failure == None) - reason = crawler.spider.meta['close_reason'] - self.assertTrue(reason, 'finished') - else: - # See issue https://twistedmatrix.com/trac/ticket/8175 - raise unittest.SkipTest("xpayload only enabled for PY2") + # download_maxsize = 50 is enough for the gzipped response + failure = crawler.spider.meta.get('failure') + self.assertTrue(failure == None) + reason = crawler.spider.meta['close_reason'] + self.assertTrue(reason, 'finished') else: - raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0") + # See issue https://twistedmatrix.com/trac/ticket/8175 + raise unittest.SkipTest("xpayload only enabled for PY2") class UriResource(resource.Resource): @@ -500,8 +491,6 @@ class Http10ProxyTestCase(HttpProxyTestCase): class Http11ProxyTestCase(HttpProxyTestCase): download_handler_cls = HTTP11DownloadHandler - if twisted_version < (11, 1, 0): - skip = 'HTTP1.1 not supported in twisted < 11.1.0' @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): @@ -692,8 +681,6 @@ class FTPTestCase(unittest.TestCase): username = "scrapy" password = "passwd" - if twisted_version < (10, 2, 0): - skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home" if six.PY3: skip = "Twisted missing ftp support for PY3" diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index eb17974bf..e129b71f8 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -5,7 +5,6 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionLost, TCPTimedOutError from twisted.web.client import ResponseFailed -from scrapy import twisted_version from scrapy.downloadermiddlewares.retry import RetryMiddleware from scrapy.spiders import Spider from scrapy.http import Request, Response @@ -74,9 +73,7 @@ class RetryTest(unittest.TestCase): def test_twistederrors(self): exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, - ConnectError, ConnectionLost] - if twisted_version >= (11, 1, 0): # http11 available - exceptions.append(ResponseFailed) + ConnectError, ConnectionLost, ResponseFailed] for exc in exceptions: req = Request('http://www.scrapytest.org/%s' % exc.__name__) From 4e765acaed7a914630ee5320fa6f6523890a2b9d Mon Sep 17 00:00:00 2001 From: Rolando Espinoza Date: Tue, 24 Jan 2017 10:30:47 -0400 Subject: [PATCH 45/91] BUG: Fix __classcell__ propagation. Python 3.6 added simpler customization of class creation but this requires to propagate correctly the __classcell__ attribute in custom __new__ methods. See https://docs.python.org/3.6/whatsnew/3.6.html#pep-487-simpler- customization-of-class-creation --- scrapy/item.py | 3 +++ tests/test_item.py | 52 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/scrapy/item.py b/scrapy/item.py index 138728a9a..aa05e9c69 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -25,6 +25,7 @@ class Field(dict): class ItemMeta(ABCMeta): def __new__(mcs, class_name, bases, attrs): + classcell = attrs.pop('__classcell__', None) new_bases = tuple(base._class for base in bases if hasattr(base, '_class')) _class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs) @@ -39,6 +40,8 @@ class ItemMeta(ABCMeta): new_attrs['fields'] = fields new_attrs['_class'] = _class + if classcell is not None: + new_attrs['__classcell__'] = classcell return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs) diff --git a/tests/test_item.py b/tests/test_item.py index dcb169c3a..85a554de0 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,8 +1,14 @@ +import sys import unittest -from scrapy.item import Item, Field import six +from scrapy.item import ABCMeta, Item, ItemMeta, Field +from tests import mock + + +PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6) + class ItemTest(unittest.TestCase): @@ -244,5 +250,49 @@ class ItemTest(unittest.TestCase): self.assertNotEqual(item['name'], copied_item['name']) +class ItemMetaTest(unittest.TestCase): + + def test_new_method_propagates_classcell(self): + new_mock = mock.Mock(side_effect=ABCMeta.__new__) + base = ItemMeta.__bases__[0] + + with mock.patch.object(base, '__new__', new_mock): + + class MyItem(Item): + if not PY36_PLUS: + # This attribute is an internal attribute in Python 3.6+ + # and must be propagated properly. See + # https://docs.python.org/3.6/reference/datamodel.html#creating-the-class-object + # In <3.6, we add a dummy attribute just to ensure the + # __new__ method propagates it correctly. + __classcell__ = object() + + def f(self): + # For rationale of this see: + # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222 + return __class__ + + MyItem() + + (first_call, second_call) = new_mock.call_args_list[-2:] + + mcs, class_name, bases, attrs = first_call[0] + assert '__classcell__' not in attrs + mcs, class_name, bases, attrs = second_call[0] + assert '__classcell__' in attrs + + +class ItemMetaClassCellRegression(unittest.TestCase): + + def test_item_meta_classcell_regression(self): + class MyItem(six.with_metaclass(ItemMeta, Item)): + def __init__(self, *args, **kwargs): + # This call to super() trigger the __classcell__ propagation + # requirement. When not done properly raises an error: + # TypeError: __class__ set to + # defining 'MyItem' as + super(MyItem, self).__init__(*args, **kwargs) + + if __name__ == "__main__": unittest.main() From 48c8c679de72da295aa753ffd9ed68b3958d3cbb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 6 Feb 2017 18:03:48 +0100 Subject: [PATCH 46/91] Update changelog for upcoming 1.3.1 release --- docs/news.rst | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index cce46599b..41374f970 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,54 @@ Release notes ============= +Scrapy 1.3.1 (2017-02-XX) +------------------------- + +New features +~~~~~~~~~~~~ + +- Support ``'True'`` and ``'False'`` string values for boolean settings (:issue:`2519`); + you can now do something like ``scrapy crawl myspider -s REDIRECT_ENABLED=False``. +- Support kwargs with ``response.xpath()`` to use :ref:`XPath variables ` + and ad-hoc namespaces declarations ; + this requires at least Parsel v1.1 (:issue:`2457`). +- Add support for Python 3.6 (:issue:`2485`). +- Run tests on PyPy (warning: some tests still fail, so PyPy is not supported yet). + +Bug fixes +~~~~~~~~~ + +- Enforce ``DNS_TIMEOUT`` setting (:issue:`2496`). +- Fix :command:`view` command ; it was a regression in v1.3.0 (:issue:`2503`). +- Fix tests regarding ``*_EXPIRES settings`` with Files/Images pipelines (:issue:`2460`). +- Fix name of generated pipeline class when using basic project template (:issue:`2466`). +- Fix compatiblity with Twisted 17+ (:issue:`2496`, :issue:`2528`). +- Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`). +- Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``, + ``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). + +Documentation +~~~~~~~~~~~~~ + +- Reword Code of Coduct section and upgrade to Contributor Covenant v1.4 + (:issue:`2469`). +- Clarify that passing spider arguments converts them to spider attributes + (:issue:`2483`). +- Document ``formid`` argument on ``FormRequest.from_response()`` (:issue:`2497`). +- Add .rst extension to README files (:issue:`2507`). +- Mention LevelDB cache storage backend (:issue:`2525`). +- Use ``yield`` in sample callback code (:issue:`2533`). +- Add note about HTML entities decoding with ``.re()/.re_first()`` (:issue:`1704`). +- Typos (:issue:`2512`, :issue:`2534`, :issue:`2531`). + +Cleanups +~~~~~~~~ + +- Remove reduntant check in ``MetaRefreshMiddleware`` (:issue:`2542`). +- Faster checks in ``LinkExtractor`` for allow/deny patterns (:issue:`2538`). +- Remove dead code supporting old Twisted versions (:issue:`2544`). + + Scrapy 1.3.0 (2016-12-21) ------------------------- From ff8a564b1a775ec16143b7c00199d40875868337 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 8 Feb 2017 17:05:06 +0100 Subject: [PATCH 47/91] Set date for 1.3.1 release --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index 41374f970..3c1d24561 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,7 +3,7 @@ Release notes ============= -Scrapy 1.3.1 (2017-02-XX) +Scrapy 1.3.1 (2017-02-08) ------------------------- New features From af55a8713e561015b4e50eb5dd7061709770c67a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 8 Feb 2017 17:08:19 +0100 Subject: [PATCH 48/91] =?UTF-8?q?Bump=20version:=201.3.0=20=E2=86=92=201.3?= =?UTF-8?q?.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 57ff603fa..0a8a71e8e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.3.0 +current_version = 1.3.1 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index f0bb29e76..3a3cd8cc8 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.3.0 +1.3.1 From 9c0aae724ed821fd954a14db83902a86f7fe7731 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 3 Feb 2017 10:32:36 -0300 Subject: [PATCH 49/91] Use credentials from request.meta['proxy'] if present --- docs/topics/downloader-middleware.rst | 5 ++- scrapy/downloadermiddlewares/httpproxy.py | 24 +++++++---- tests/test_downloadermiddleware_httpproxy.py | 43 +++++++++++++++++--- 3 files changed, 57 insertions(+), 15 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1ca78ccc6..f0ff3c77c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -681,7 +681,10 @@ HttpProxyMiddleware * ``no_proxy`` You can also set the meta key ``proxy`` per-request, to a value like - ``http://some_proxy_server:port``. + ``http://username:password@some_proxy_server:port``. Keep in mind + this value will take precedence over ``http_proxy``/``https_proxy`` + environment variables, and it will also ignore ``no_proxy`` environment + variable. .. _urllib: https://docs.python.org/2/library/urllib.html .. _urllib2: https://docs.python.org/2/library/urllib2.html diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 98c87aa9c..edc1c52ed 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -8,7 +8,6 @@ except ImportError: from six.moves.urllib.parse import urlunparse from scrapy.utils.httpobj import urlparse_cached -from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_bytes @@ -20,23 +19,23 @@ class HttpProxyMiddleware(object): for type, url in getproxies().items(): self.proxies[type] = self._get_proxy(url, type) - if not self.proxies: - raise NotConfigured - @classmethod def from_crawler(cls, crawler): auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING') return cls(auth_encoding) + def _basic_auth_header(self, username, password): + user_pass = to_bytes( + '%s:%s' % (unquote(username), unquote(password)), + encoding=self.auth_encoding) + return base64.b64encode(user_pass).strip() + def _get_proxy(self, url, orig_type): proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', '')) if user: - user_pass = to_bytes( - '%s:%s' % (unquote(user), unquote(password)), - encoding=self.auth_encoding) - creds = base64.b64encode(user_pass).strip() + creds = self._basic_auth_header(user, password) else: creds = None @@ -45,6 +44,15 @@ class HttpProxyMiddleware(object): def process_request(self, request, spider): # ignore if proxy is already set if 'proxy' in request.meta: + if request.meta['proxy'] is None: + return + # extract credentials if present + creds, proxy_url = self._get_proxy(request.meta['proxy'], '') + request.meta['proxy'] = proxy_url + if creds and not request.headers.get('Proxy-Authorization'): + request.headers['Proxy-Authorization'] = b'Basic ' + creds + return + elif not self.proxies: return parsed = urlparse_cached(request) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 2b26431a4..dd09e4dd0 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -20,10 +20,6 @@ class TestDefaultHeadersMiddleware(TestCase): def tearDown(self): os.environ = self._oldenv - def test_no_proxies(self): - os.environ = {} - self.assertRaises(NotConfigured, HttpProxyMiddleware) - def test_no_enviroment_proxies(self): os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} mw = HttpProxyMiddleware() @@ -47,6 +43,13 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.url, url) self.assertEquals(req.meta.get('proxy'), proxy) + def test_proxy_precedence_meta(self): + os.environ['http_proxy'] = 'https://proxy.com' + mw = HttpProxyMiddleware() + req = Request('http://scrapytest.org', meta={'proxy': 'https://new.proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://new.proxy:3128'}) + def test_proxy_auth(self): os.environ['http_proxy'] = 'https://user:pass@proxy:3128' mw = HttpProxyMiddleware() @@ -54,6 +57,11 @@ class TestDefaultHeadersMiddleware(TestCase): assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') def test_proxy_auth_empty_passwd(self): os.environ['http_proxy'] = 'https://user:@proxy:3128' @@ -62,6 +70,11 @@ class TestDefaultHeadersMiddleware(TestCase): assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6') def test_proxy_auth_encoding(self): # utf-8 encoding @@ -72,6 +85,12 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') + # proxy from request.meta + req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==') + # default latin-1 encoding mw = HttpProxyMiddleware(auth_encoding='latin-1') req = Request('http://scrapytest.org') @@ -79,15 +98,21 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') + # proxy from request.meta, latin-1 encoding + req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz') + def test_proxy_already_seted(self): - os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' + os.environ['http_proxy'] = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() req = Request('http://noproxy.com', meta={'proxy': None}) assert mw.process_request(req, spider) is None assert 'proxy' in req.meta and req.meta['proxy'] is None def test_no_proxy(self): - os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' + os.environ['http_proxy'] = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() os.environ['no_proxy'] = '*' @@ -104,3 +129,9 @@ class TestDefaultHeadersMiddleware(TestCase): req = Request('http://noproxy.com') assert mw.process_request(req, spider) is None assert 'proxy' not in req.meta + + # proxy from meta['proxy'] takes precedence + os.environ['no_proxy'] = '*' + req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'}) + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'http://proxy.com'}) From 29e60213db19030907dc8afd50173aa4421132df Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 9 Feb 2017 10:41:21 +0100 Subject: [PATCH 50/91] Use consistent selectors for author field in tutorial --- docs/intro/tutorial.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 8e14d1b7c..3dc5ad2ed 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -225,7 +225,7 @@ You will see something like:: [s] shelp() Shell help (print this help) [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser - >>> + >>> Using the shell, you can try selecting elements using `CSS`_ with the response object:: @@ -423,7 +423,7 @@ in the callback, as you can see below:: for quote in response.css('div.quote'): yield { 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('span small::text').extract_first(), + 'author': quote.css('small.author::text').extract_first(), 'tags': quote.css('div.tags a.tag::text').extract(), } @@ -522,7 +522,7 @@ page, extracting data from it:: for quote in response.css('div.quote'): yield { 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('span small::text').extract_first(), + 'author': quote.css('small.author::text').extract_first(), 'tags': quote.css('div.tags a.tag::text').extract(), } @@ -568,7 +568,7 @@ this time for scraping author information:: def parse(self, response): # follow links to author pages - for href in response.css('.author+a::attr(href)').extract(): + for href in response.css('.author + a::attr(href)').extract(): yield scrapy.Request(response.urljoin(href), callback=self.parse_author) @@ -624,7 +624,7 @@ option when running them:: scrapy crawl quotes -o quotes-humor.json -a tag=humor These arguments are passed to the Spider's ``__init__`` method and become -spider attributes by default. +spider attributes by default. In this example, the value provided for the ``tag`` argument will be available via ``self.tag``. You can use this to make your spider fetch only quotes @@ -647,7 +647,7 @@ with a specific tag, building the URL based on the argument:: for quote in response.css('div.quote'): yield { 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('span small a::text').extract_first(), + 'author': quote.css('small.author::text').extract_first(), } next_page = response.css('li.next a::attr(href)').extract_first() From 9956f198db54fa181856ad788e2ad8ee76ae5437 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 13 Feb 2017 18:40:53 +0500 Subject: [PATCH 51/91] add a couple more lines to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b116640b4..406146e5f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ dist .idea htmlcov/ .coverage +.coverage.* +.cache/ # Windows Thumbs.db From de65ad3fb1e90f6fcffbcc74bfa6aff8ff65e14f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 13 Feb 2017 18:44:39 +0500 Subject: [PATCH 52/91] TST replace Ubuntu 12.04 tox environment with 14.04 --- tox.ini | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 0fdea11bb..bbf50b733 100644 --- a/tox.ini +++ b/tox.ini @@ -21,16 +21,16 @@ passenv = commands = py.test --cov=scrapy --cov-report= {posargs:scrapy tests} -[testenv:precise] +[testenv:trusty] basepython = python2.7 deps = pyOpenSSL==0.13 - lxml==2.3.2 - Twisted==11.1.0 - boto==2.2.2 - Pillow<2.0 + lxml==3.3.3 + Twisted==13.2.0 + boto==2.20.1 + Pillow==2.3.0 cssselect==0.9.1 - zope.interface==3.6.1 + zope.interface==4.0.5 -rtests/requirements.txt [testenv:jessie] From 1bc4d8b6b6cf84c1785a6ad69abf37b4f0114bb3 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 13 Feb 2017 20:03:53 +0500 Subject: [PATCH 53/91] fixed tls in Twisted 17+ --- scrapy/core/downloader/tls.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 955b7630c..498e3d60f 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -1,6 +1,8 @@ import logging from OpenSSL import SSL +from scrapy import twisted_version + logger = logging.getLogger(__name__) @@ -18,11 +20,17 @@ openssl_methods = { METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only } -# ClientTLSOptions requires a recent-enough version of Twisted -try: +if twisted_version >= (14, 0, 0): + # ClientTLSOptions requires a recent-enough version of Twisted. + # Not having ScrapyClientTLSOptions should not matter for older + # Twisted versions because it is not used in the fallback + # ScrapyClientContextFactory. # taken from twisted/twisted/internet/_sslverify.py + try: + # XXX: this try-except is not needed in Twisted 17.0.0+ because + # it requires pyOpenSSL 0.16+. from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START except ImportError: SSL_CB_HANDSHAKE_START = 0x10 @@ -30,10 +38,17 @@ try: from twisted.internet.ssl import AcceptableCiphers from twisted.internet._sslverify import (ClientTLSOptions, - _maybeSetHostNameIndication, verifyHostname, VerificationError) + if twisted_version < (17, 0, 0): + from twisted.internet._sslverify import _maybeSetHostNameIndication + set_tlsext_host_name = _maybeSetHostNameIndication + else: + def set_tlsext_host_name(connection, hostNameBytes): + connection.set_tlsext_host_name(hostNameBytes) + + class ScrapyClientTLSOptions(ClientTLSOptions): """ SSL Client connection creator ignoring certificate verification errors @@ -46,7 +61,7 @@ try: def _identityVerifyingInfoCallback(self, connection, where, ret): if where & SSL_CB_HANDSHAKE_START: - _maybeSetHostNameIndication(connection, self._hostnameBytes) + set_tlsext_host_name(connection, self._hostnameBytes) elif where & SSL_CB_HANDSHAKE_DONE: try: verifyHostname(connection, self._hostnameASCII) @@ -62,8 +77,3 @@ try: self._hostnameASCII, repr(e))) DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT') - -except ImportError: - # ImportError should not matter for older Twisted versions - # as the above is not used in the fallback ScrapyClientContextFactory - pass From 9315e944a225a4010870333ff751fb2ed512f489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 13 Feb 2017 14:56:29 -0300 Subject: [PATCH 54/91] Release notes for 1.3.2 --- docs/news.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 3c1d24561..ff1e4ce03 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,16 @@ Release notes ============= +Scrapy 1.3.2 (2017-02-13) +------------------------- + +Bug fixes +~~~~~~~~~ + +- Preserve crequest class when converting to/from dicts (utils.reqser) (:issue:`2510`). +- Use consistent selectors for author field in tutorial (:issue:`2551`). +- Fix TLS compatibility in Twisted 17+ (:issue:`2558`) + Scrapy 1.3.1 (2017-02-08) ------------------------- From 7dd7646e6563798d408b8062059e826f08256b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Mon, 13 Feb 2017 14:57:55 -0300 Subject: [PATCH 55/91] =?UTF-8?q?Bump=20version:=201.3.1=20=E2=86=92=201.3?= =?UTF-8?q?.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0a8a71e8e..b95e0bad5 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.3.1 +current_version = 1.3.2 commit = True tag = True tag_name = {new_version} diff --git a/scrapy/VERSION b/scrapy/VERSION index 3a3cd8cc8..1892b9267 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.3.1 +1.3.2 From 5b31dfe3c970f7e16dc911bb8b028a0fff54a7f9 Mon Sep 17 00:00:00 2001 From: terut Date: Mon, 13 Feb 2017 23:51:43 -0800 Subject: [PATCH 56/91] Separate building request from _requests_to_follow in CrawlSpider You just overwrite buiding request if you can use another request class because of something like splash-plugin. --- scrapy/spiders/crawl.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 031f649d6..e5ac72e18 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -48,6 +48,11 @@ class CrawlSpider(Spider): def process_results(self, response, results): return results + def _build_request(self, rule, link): + r = Request(url=link.url, callback=self._response_downloaded) + r.meta.update(rule=rule, link_text=link.text) + return r + def _requests_to_follow(self, response): if not isinstance(response, HtmlResponse): return @@ -59,8 +64,7 @@ class CrawlSpider(Spider): links = rule.process_links(links) for link in links: seen.add(link) - r = Request(url=link.url, callback=self._response_downloaded) - r.meta.update(rule=n, link_text=link.text) + r = self._build_request(n, link) yield rule.process_request(r) def _response_downloaded(self, response): From ae0ea31abd4e65f7decb4482d1669d7af17bca0a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 8 Feb 2017 13:21:10 -0300 Subject: [PATCH 57/91] Add HTTPPROXY_ENABLED setting (default True) --- docs/topics/downloader-middleware.rst | 15 +++++++++++---- scrapy/downloadermiddlewares/httpproxy.py | 3 +++ scrapy/settings/default_settings.py | 1 + tests/test_downloadermiddleware_httpproxy.py | 8 ++++++++ 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index f0ff3c77c..0ef3fb071 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -681,10 +681,9 @@ HttpProxyMiddleware * ``no_proxy`` You can also set the meta key ``proxy`` per-request, to a value like - ``http://username:password@some_proxy_server:port``. Keep in mind - this value will take precedence over ``http_proxy``/``https_proxy`` - environment variables, and it will also ignore ``no_proxy`` environment - variable. + ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``. + Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` + environment variables, and it will also ignore ``no_proxy`` environment variable. .. _urllib: https://docs.python.org/2/library/urllib.html .. _urllib2: https://docs.python.org/2/library/urllib2.html @@ -952,8 +951,16 @@ enable it for :ref:`broad crawls `. HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. setting:: HTTPPROXY_ENABLED .. setting:: HTTPPROXY_AUTH_ENCODING +HTTPPROXY_ENABLED +^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether or not to enable the :class:`HttpProxyMiddleware`. + HTTPPROXY_AUTH_ENCODING ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index edc1c52ed..0d5320bf8 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -8,6 +8,7 @@ except ImportError: from six.moves.urllib.parse import urlunparse from scrapy.utils.httpobj import urlparse_cached +from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_bytes @@ -21,6 +22,8 @@ class HttpProxyMiddleware(object): @classmethod def from_crawler(cls, crawler): + if not crawler.settings.getbool('HTTPPROXY_ENABLED'): + raise NotConfigured auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING') return cls(auth_encoding) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 24714a7a8..cb88bc2bf 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -174,6 +174,7 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False +HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = 'latin-1' IMAGES_STORE_S3_ACL = 'private' diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index dd09e4dd0..c77179ceb 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -1,11 +1,14 @@ import os import sys +from functools import partial from twisted.trial.unittest import TestCase, SkipTest from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request from scrapy.spiders import Spider +from scrapy.crawler import Crawler +from scrapy.settings import Settings spider = Spider('foo') @@ -20,6 +23,11 @@ class TestDefaultHeadersMiddleware(TestCase): def tearDown(self): os.environ = self._oldenv + def test_not_enabled(self): + settings = Settings({'HTTPPROXY_ENABLED': False}) + crawler = Crawler(spider, settings) + self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler)) + def test_no_enviroment_proxies(self): os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'} mw = HttpProxyMiddleware() From 922d3fec54f40b0349c8c98dd8441aee3075cd65 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 14 Feb 2017 12:11:06 -0300 Subject: [PATCH 58/91] Doc: binary mode is required for exporters --- docs/topics/exporters.rst | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index af469eb7b..85c73222d 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -225,7 +225,8 @@ XmlItemExporter Exports Items in XML format to the specified file object. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) :param root_element: The name of root element in the exported XML. :type root_element: str @@ -281,7 +282,8 @@ CsvItemExporter CSV columns and their order. The :attr:`export_empty_fields` attribute has no effect on this exporter. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) :param include_headers_line: If enabled, makes the exporter output a header line with the field names taken from @@ -312,7 +314,8 @@ PickleItemExporter Exports Items in pickle format to the given file-like object. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) :param protocol: The pickle protocol to use. :type protocol: int @@ -333,7 +336,8 @@ PprintItemExporter Exports Items in pretty print format to the specified file object. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) The additional keyword arguments of this constructor are passed to the :class:`BaseItemExporter` constructor. @@ -356,7 +360,8 @@ JsonItemExporter arguments to the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_ constructor argument to customize this exporter. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) A typical output of this exporter would be:: @@ -386,7 +391,8 @@ JsonLinesItemExporter the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_ constructor argument to customize this exporter. - :param file: the file-like object to use for exporting the data. + :param file: the file-like object to use for exporting the data. Its ``write`` method should + accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) A typical output of this exporter would be:: From e1ceaf3b5fa29326f032c4ed3f50943384b9e63d Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Mon, 13 Feb 2017 21:06:05 +0500 Subject: [PATCH 59/91] require w3lib 1.17+ --- requirements-py3.txt | 2 +- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 08ccf1958..cc0a7f644 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -3,5 +3,5 @@ lxml>=3.2.4 pyOpenSSL>=0.13.1 cssselect>=0.9 queuelib>=1.1.1 -w3lib>=1.14.2 +w3lib>=1.17.0 service_identity diff --git a/requirements.txt b/requirements.txt index f92603d3d..392f83dd6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ Twisted>=13.1.0 lxml pyOpenSSL cssselect>=0.9 -w3lib>=1.15.0 +w3lib>=1.17.0 queuelib six>=1.5.2 PyDispatcher>=2.0.5 diff --git a/setup.py b/setup.py index a6e6f9615..086ab8142 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ setup( ], install_requires=[ 'Twisted>=13.1.0', - 'w3lib>=1.15.0', + 'w3lib>=1.17.0', 'queuelib', 'lxml', 'pyOpenSSL', From 877057fac0d47e5ece95a55594706c91c8855883 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 06:00:09 +0500 Subject: [PATCH 60/91] initial response.follow implementation --- docs/intro/overview.rst | 3 +- docs/intro/tutorial.rst | 46 +++++++++++++++++++------ docs/topics/request-response.rst | 4 +++ scrapy/http/response/text.py | 59 ++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 12 deletions(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 7195017ff..1da1a4059 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -40,8 +40,7 @@ http://quotes.toscrape.com, following the pagination:: next_page = response.css('li.next a::attr("href")').extract_first() if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, callback=self.parse) + yield response.follow(next_page, self.parse) Put this in a text file, name it to something like ``quotes_spider.py`` diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 3dc5ad2ed..d47bf69e5 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -551,13 +551,40 @@ In our example, it creates a sort of loop, following all the links to the next p until it doesn't find one -- handy for crawling blogs, forums and other sites with pagination. +As a shortcut for creating Request objects you can use +:meth:`response.follow ` method:: + + import scrapy + + + class QuotesSpider(scrapy.Spider): + name = "quotes" + start_urls = [ + 'http://quotes.toscrape.com/page/1/', + ] + + def parse(self, response): + for quote in response.css('div.quote'): + yield { + 'text': quote.css('span.text::text').extract_first(), + 'author': quote.css('span small::text').extract_first(), + 'tags': quote.css('div.tags a.tag::text').extract(), + } + + for href in response.css('li.next a::attr(href)'): + yield response.follow(href, callback=self.parse) + +Unlike scrapy.Request, ``response.follow`` supports +relative URLs directly; you can also pass a selector to it instead of +a string. Note that ``response.follow`` just returns a Request instance; +you still have to yield this Request. + More examples and patterns -------------------------- Here is another spider that illustrates callbacks and following links, this time for scraping author information:: - import scrapy @@ -568,15 +595,12 @@ this time for scraping author information:: def parse(self, response): # follow links to author pages - for href in response.css('.author + a::attr(href)').extract(): - yield scrapy.Request(response.urljoin(href), - callback=self.parse_author) + for href in response.css('.author + a::attr(href)'): + yield response.follow(href, self.parse_author) # follow pagination links - next_page = response.css('li.next a::attr(href)').extract_first() - if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, callback=self.parse) + for href in response.css('li.next a::attr(href)'): + yield response.follow(href, self.parse) def parse_author(self, response): def extract_with_css(query): @@ -592,6 +616,9 @@ This spider will start from the main page, it will follow all the links to the authors pages calling the ``parse_author`` callback for each of them, and also the pagination links with the ``parse`` callback as we saw before. +Here we're passing callbacks to ``response.follow`` as positional arguments +to make the code shorter; it also works for ``scrapy.Request``. + The ``parse_author`` callback defines a helper function to extract and cleanup the data from a CSS query and yields the Python dict with the author data. @@ -652,8 +679,7 @@ with a specific tag, building the URL based on the argument:: next_page = response.css('li.next a::attr(href)').extract_first() if next_page is not None: - next_page = response.urljoin(next_page) - yield scrapy.Request(next_page, self.parse) + yield response.follow(next_page, self.parse) If you pass the ``tag=humor`` argument to this spider, you'll notice that it diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1fdd26043..71050fddd 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -683,6 +683,10 @@ TextResponse objects response.css('p') + .. method:: TextResponse.follow(url, ...) + + Return a scrapy.Request instance to follow a link ``url``. + .. method:: TextResponse.body_as_unicode() The same as :attr:`text`, but available as a method. This method is diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 5a6507aa8..1718b1f3b 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -8,8 +8,12 @@ See documentation in docs/topics/request-response.rst import six from six.moves.urllib.parse import urljoin +import parsel from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding + +from scrapy.link import Link +from scrapy.http.request import Request from scrapy.http.response import Response from scrapy.utils.response import get_base_url from scrapy.utils.python import memoizemethod_noargs, to_native_str @@ -116,3 +120,58 @@ class TextResponse(Response): def css(self, query): return self.selector.css(query) + + def follow(self, url, callback=None, method='GET', headers=None, body=None, + cookies=None, meta=None, encoding=None, priority=0, + dont_filter=False, errback=None): + # type: (...) -> Request + """ + Return a scrapy.Request instance to follow a link ``url``. + + ``url`` can be: + + * absolute URL; + * relative URL; + * scrapy.link.Link object (e.g. a link extractor result); + * attribute Selector (not SelectorList) - e.g. + ``response.css('a::attr(href)')[0]`` or + ``response.xpath('//img/@src')[0]``. + * a Selector for ```` element, e.g. + ``response.css('a.my_link')[0]``. + """ + if isinstance(url, Link): + url = url.url + elif isinstance(url, parsel.Selector): + url = _url_from_selector(url) + elif isinstance(url, parsel.SelectorList): + raise ValueError("Please pass either string") + + + encoding = self.encoding if encoding is None else encoding + url = self.urljoin(url) + return Request(url, callback, + method=method, + headers=headers, + body=body, + cookies=cookies, + meta=meta, + encoding=encoding, + priority=priority, + dont_filter=dont_filter, + errback=errback) + + +def _url_from_selector(sel): + # type: (parsel.Selector) -> str + if isinstance(sel.root, six.string_types): + # e.g. ::attr(href) result + return sel.root + if not hasattr(sel.root, 'tag'): + raise ValueError("Unsupported selector: %s" % sel) + if sel.root.tag != 'a': + raise ValueError("Only elements are supported; got <%s>" % + sel.root.tag) + href = sel.root.get('href') + if href is None: + raise ValueError(" element has no href attribute: %s" % sel) + return href From 71dd5d0bf9d0c41d70e72b0fd0a89528ef246065 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 06:11:08 +0500 Subject: [PATCH 61/91] strip URL extracted from selectors (as per html5 standard) --- scrapy/http/response/text.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 1718b1f3b..5bfd2debb 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -142,10 +142,9 @@ class TextResponse(Response): if isinstance(url, Link): url = url.url elif isinstance(url, parsel.Selector): - url = _url_from_selector(url) + url = _url_from_selector(url).strip() elif isinstance(url, parsel.SelectorList): - raise ValueError("Please pass either string") - + raise ValueError("SelectorList is not supported") encoding = self.encoding if encoding is None else encoding url = self.urljoin(url) From 608c3f0c452bd508aa24bc9d4bf759e0b11e1683 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Feb 2017 04:17:41 +0500 Subject: [PATCH 62/91] handle whitespace in response.follow; add tests --- scrapy/http/response/text.py | 7 ++- tests/__init__.py | 7 ++- tests/test_http_response.py | 105 +++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 5 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 5bfd2debb..6eacfbd35 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -11,6 +11,7 @@ from six.moves.urllib.parse import urljoin import parsel from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding +from w3lib.html import strip_html5_whitespace from scrapy.link import Link from scrapy.http.request import Request @@ -142,7 +143,7 @@ class TextResponse(Response): if isinstance(url, Link): url = url.url elif isinstance(url, parsel.Selector): - url = _url_from_selector(url).strip() + url = _url_from_selector(url) elif isinstance(url, parsel.SelectorList): raise ValueError("SelectorList is not supported") @@ -164,7 +165,7 @@ def _url_from_selector(sel): # type: (parsel.Selector) -> str if isinstance(sel.root, six.string_types): # e.g. ::attr(href) result - return sel.root + return strip_html5_whitespace(sel.root) if not hasattr(sel.root, 'tag'): raise ValueError("Unsupported selector: %s" % sel) if sel.root.tag != 'a': @@ -173,4 +174,4 @@ def _url_from_selector(sel): href = sel.root.get('href') if href is None: raise ValueError(" element has no href attribute: %s" % sel) - return href + return strip_html5_whitespace(href) diff --git a/tests/__init__.py b/tests/__init__.py index d940f28ea..c2e4fd2bf 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -26,9 +26,12 @@ try: except ImportError: import mock -tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data') +tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'sample_data') + def get_testdata(*paths): """Return test data""" path = os.path.join(tests_datadir, *paths) - return open(path, 'rb').read() + with open(path, 'rb') as f: + return f.read() diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 9df3bf6e7..2a9baf5ed 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import unittest import six @@ -8,6 +9,8 @@ from scrapy.http import (Request, Response, TextResponse, HtmlResponse, from scrapy.selector import Selector from scrapy.utils.python import to_native_str from scrapy.exceptions import NotSupported +from scrapy.link import Link +from tests import get_testdata class BaseResponseTest(unittest.TestCase): @@ -356,6 +359,11 @@ class HtmlResponseTest(TextResponseTest): response_class = HtmlResponse + def _links_response(self): + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + resp = self.response_class('http://example.com/index', body=body) + return resp + def test_html_encoding(self): body = b"""Some page @@ -388,6 +396,103 @@ class HtmlResponseTest(TextResponseTest): r1 = self.response_class("http://www.example.com", body=body) self._assert_response_values(r1, 'gb2312', body) + def assert_followed_url(self, follow_obj, target_url, response=None): + if response is None: + response = self._links_response() + req = response.follow(follow_obj) + self.assertEqual(req.url, target_url) + return req + + def test_follow_url_absolute(self): + self.assert_followed_url('http://foo.example.com', + 'http://foo.example.com') + + def test_follow_url_relative(self): + self.assert_followed_url('foo', + 'http://example.com/foo') + + def test_follow_link(self): + self.assert_followed_url(Link('http://example.com/foo'), + 'http://example.com/foo') + + def test_follow_selector(self): + resp = self._links_response() + urls = [ + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample3.html', + 'http://www.google.com/something', + 'http://example.com/innertag.html' + ] + + # select elements + for sellist in [resp.css('a'), resp.xpath('//a')]: + for sel, url in zip(sellist, urls): + self.assert_followed_url(sel, url, response=resp) + + # href attributes should work + for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]: + for sel, url in zip(sellist, urls): + self.assert_followed_url(sel, url, response=resp) + + # non-a elements are not supported + self.assertRaises(ValueError, resp.follow, resp.css('div')[0]) + + def test_follow_selector_list(self): + resp = self._links_response() + self.assertRaisesRegex(ValueError, 'SelectorList', + resp.follow, resp.css('a')) + + def test_follow_selector_attribute(self): + resp = self._links_response() + for src in resp.css('img::attr(src)'): + self.assert_followed_url(src, 'http://example.com/sample2.jpg') + + def test_follow_whitespace_url(self): + self.assert_followed_url('foo ', + 'http://example.com/foo%20') + + def test_follow_whitespace_link(self): + self.assert_followed_url(Link('http://example.com/foo '), + 'http://example.com/foo%20') + + def test_follow_whitespace_selector(self): + resp = self.response_class( + 'http://example.com', + body=b'''click me''' + ) + self.assert_followed_url(resp.css('a')[0], + 'http://example.com/foo', + response=resp) + self.assert_followed_url(resp.css('a::attr(href)')[0], + 'http://example.com/foo', + response=resp) + + def test_follow_encoding(self): + resp1 = self.response_class( + 'http://example.com', + encoding='utf8', + body='click me'.encode('utf8') + ) + req = self.assert_followed_url( + resp1.css('a')[0], + 'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82', + response=resp1, + ) + self.assertEqual(req.encoding, 'utf8') + + resp2 = self.response_class( + 'http://example.com', + encoding='cp1251', + body='click me'.encode('cp1251') + ) + req = self.assert_followed_url( + resp2.css('a')[0], + 'http://example.com/foo?%EF%F0%E8%E2%E5%F2', + response=resp2, + ) + self.assertEqual(req.encoding, 'cp1251') + class XmlResponseTest(TextResponseTest): From 2674f317df9e4970f5953db3a6df04331246d8c9 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Feb 2017 04:39:47 +0500 Subject: [PATCH 63/91] Response.follow --- scrapy/http/response/__init__.py | 29 +++++ scrapy/http/response/text.py | 27 ++-- tests/test_http_response.py | 204 +++++++++++++++---------------- 3 files changed, 143 insertions(+), 117 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 58ad414f1..e5fb4eef8 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -6,7 +6,9 @@ See documentation in docs/topics/request-response.rst """ from six.moves.urllib.parse import urljoin +from scrapy.http.request import Request from scrapy.http.headers import Headers +from scrapy.link import Link from scrapy.utils.trackref import object_ref from scrapy.http.common import obsolete_setter from scrapy.exceptions import NotSupported @@ -101,3 +103,30 @@ class Response(object_ref): is text (subclasses of TextResponse). """ raise NotSupported("Response content isn't text") + + def follow(self, url, callback=None, method='GET', headers=None, body=None, + cookies=None, meta=None, encoding='utf-8', priority=0, + dont_filter=False, errback=None): + # type: (...) -> Request + """ + Return a scrapy.Request instance to follow a link ``url``. + + ``url`` can be: + + * absolute URL; + * relative URL; + * scrapy.link.Link object. + """ + if isinstance(url, Link): + url = url.url + url = self.urljoin(url) + return Request(url, callback, + method=method, + headers=headers, + body=body, + cookies=cookies, + meta=meta, + encoding=encoding, + priority=priority, + dont_filter=dont_filter, + errback=errback) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 6eacfbd35..3c360bcf9 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -140,25 +140,22 @@ class TextResponse(Response): * a Selector for ```` element, e.g. ``response.css('a.my_link')[0]``. """ - if isinstance(url, Link): - url = url.url - elif isinstance(url, parsel.Selector): + if isinstance(url, parsel.Selector): url = _url_from_selector(url) elif isinstance(url, parsel.SelectorList): raise ValueError("SelectorList is not supported") - encoding = self.encoding if encoding is None else encoding - url = self.urljoin(url) - return Request(url, callback, - method=method, - headers=headers, - body=body, - cookies=cookies, - meta=meta, - encoding=encoding, - priority=priority, - dont_filter=dont_filter, - errback=errback) + return super(TextResponse, self).follow(url, callback, + method=method, + headers=headers, + body=body, + cookies=cookies, + meta=meta, + encoding=encoding, + priority=priority, + dont_filter=dont_filter, + errback=errback + ) def _url_from_selector(sel): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 2a9baf5ed..e64d0eeba 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -143,6 +143,38 @@ class BaseResponseTest(unittest.TestCase): r.css('body') r.xpath('//body') + def test_follow_url_absolute(self): + self._assert_followed_url('http://foo.example.com', + 'http://foo.example.com') + + def test_follow_url_relative(self): + self._assert_followed_url('foo', + 'http://example.com/foo') + + def test_follow_link(self): + self._assert_followed_url(Link('http://example.com/foo'), + 'http://example.com/foo') + + def test_follow_whitespace_url(self): + self._assert_followed_url('foo ', + 'http://example.com/foo%20') + + def test_follow_whitespace_link(self): + self._assert_followed_url(Link('http://example.com/foo '), + 'http://example.com/foo%20') + + def _assert_followed_url(self, follow_obj, target_url, response=None): + if response is None: + response = self._links_response() + req = response.follow(follow_obj) + self.assertEqual(req.url, target_url) + return req + + def _links_response(self): + body = get_testdata('link_extractor', 'sgml_linkextractor.html') + resp = self.response_class('http://example.com/index', body=body) + return resp + class TextResponseTest(BaseResponseTest): @@ -354,16 +386,81 @@ class TextResponseTest(BaseResponseTest): absolute = 'http://www.example.com/elsewhere/test' self.assertEqual(joined, absolute) + def test_follow_selector(self): + resp = self._links_response() + urls = [ + 'http://example.com/sample2.html', + 'http://example.com/sample3.html', + 'http://example.com/sample3.html', + 'http://www.google.com/something', + 'http://example.com/innertag.html' + ] + + # select elements + for sellist in [resp.css('a'), resp.xpath('//a')]: + for sel, url in zip(sellist, urls): + self._assert_followed_url(sel, url, response=resp) + + # href attributes should work + for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]: + for sel, url in zip(sellist, urls): + self._assert_followed_url(sel, url, response=resp) + + # non-a elements are not supported + self.assertRaises(ValueError, resp.follow, resp.css('div')[0]) + + def test_follow_selector_list(self): + resp = self._links_response() + self.assertRaisesRegex(ValueError, 'SelectorList', + resp.follow, resp.css('a')) + + def test_follow_selector_attribute(self): + resp = self._links_response() + for src in resp.css('img::attr(src)'): + self._assert_followed_url(src, 'http://example.com/sample2.jpg') + + def test_follow_whitespace_selector(self): + resp = self.response_class( + 'http://example.com', + body=b'''click me''' + ) + self._assert_followed_url(resp.css('a')[0], + 'http://example.com/foo', + response=resp) + self._assert_followed_url(resp.css('a::attr(href)')[0], + 'http://example.com/foo', + response=resp) + + def test_follow_encoding(self): + resp1 = self.response_class( + 'http://example.com', + encoding='utf8', + body='click me'.encode('utf8') + ) + req = self._assert_followed_url( + resp1.css('a')[0], + 'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82', + response=resp1, + ) + self.assertEqual(req.encoding, 'utf8') + + resp2 = self.response_class( + 'http://example.com', + encoding='cp1251', + body='click me'.encode('cp1251') + ) + req = self._assert_followed_url( + resp2.css('a')[0], + 'http://example.com/foo?%EF%F0%E8%E2%E5%F2', + response=resp2, + ) + self.assertEqual(req.encoding, 'cp1251') + class HtmlResponseTest(TextResponseTest): response_class = HtmlResponse - def _links_response(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') - resp = self.response_class('http://example.com/index', body=body) - return resp - def test_html_encoding(self): body = b"""Some page @@ -396,103 +493,6 @@ class HtmlResponseTest(TextResponseTest): r1 = self.response_class("http://www.example.com", body=body) self._assert_response_values(r1, 'gb2312', body) - def assert_followed_url(self, follow_obj, target_url, response=None): - if response is None: - response = self._links_response() - req = response.follow(follow_obj) - self.assertEqual(req.url, target_url) - return req - - def test_follow_url_absolute(self): - self.assert_followed_url('http://foo.example.com', - 'http://foo.example.com') - - def test_follow_url_relative(self): - self.assert_followed_url('foo', - 'http://example.com/foo') - - def test_follow_link(self): - self.assert_followed_url(Link('http://example.com/foo'), - 'http://example.com/foo') - - def test_follow_selector(self): - resp = self._links_response() - urls = [ - 'http://example.com/sample2.html', - 'http://example.com/sample3.html', - 'http://example.com/sample3.html', - 'http://www.google.com/something', - 'http://example.com/innertag.html' - ] - - # select elements - for sellist in [resp.css('a'), resp.xpath('//a')]: - for sel, url in zip(sellist, urls): - self.assert_followed_url(sel, url, response=resp) - - # href attributes should work - for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]: - for sel, url in zip(sellist, urls): - self.assert_followed_url(sel, url, response=resp) - - # non-a elements are not supported - self.assertRaises(ValueError, resp.follow, resp.css('div')[0]) - - def test_follow_selector_list(self): - resp = self._links_response() - self.assertRaisesRegex(ValueError, 'SelectorList', - resp.follow, resp.css('a')) - - def test_follow_selector_attribute(self): - resp = self._links_response() - for src in resp.css('img::attr(src)'): - self.assert_followed_url(src, 'http://example.com/sample2.jpg') - - def test_follow_whitespace_url(self): - self.assert_followed_url('foo ', - 'http://example.com/foo%20') - - def test_follow_whitespace_link(self): - self.assert_followed_url(Link('http://example.com/foo '), - 'http://example.com/foo%20') - - def test_follow_whitespace_selector(self): - resp = self.response_class( - 'http://example.com', - body=b'''click me''' - ) - self.assert_followed_url(resp.css('a')[0], - 'http://example.com/foo', - response=resp) - self.assert_followed_url(resp.css('a::attr(href)')[0], - 'http://example.com/foo', - response=resp) - - def test_follow_encoding(self): - resp1 = self.response_class( - 'http://example.com', - encoding='utf8', - body='click me'.encode('utf8') - ) - req = self.assert_followed_url( - resp1.css('a')[0], - 'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82', - response=resp1, - ) - self.assertEqual(req.encoding, 'utf8') - - resp2 = self.response_class( - 'http://example.com', - encoding='cp1251', - body='click me'.encode('cp1251') - ) - req = self.assert_followed_url( - resp2.css('a')[0], - 'http://example.com/foo?%EF%F0%E8%E2%E5%F2', - response=resp2, - ) - self.assertEqual(req.encoding, 'cp1251') - class XmlResponseTest(TextResponseTest): From 160da6abab8954906181ce69593ff6e84d950ac1 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Feb 2017 04:41:53 +0500 Subject: [PATCH 64/91] fixed tests in Python 2 --- tests/test_http_response.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index e64d0eeba..fa74b468b 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -411,8 +411,8 @@ class TextResponseTest(BaseResponseTest): def test_follow_selector_list(self): resp = self._links_response() - self.assertRaisesRegex(ValueError, 'SelectorList', - resp.follow, resp.css('a')) + self.assertRaisesRegexp(ValueError, 'SelectorList', + resp.follow, resp.css('a')) def test_follow_selector_attribute(self): resp = self._links_response() @@ -435,7 +435,7 @@ class TextResponseTest(BaseResponseTest): resp1 = self.response_class( 'http://example.com', encoding='utf8', - body='click me'.encode('utf8') + body=u'click me'.encode('utf8') ) req = self._assert_followed_url( resp1.css('a')[0], @@ -447,7 +447,7 @@ class TextResponseTest(BaseResponseTest): resp2 = self.response_class( 'http://example.com', encoding='cp1251', - body='click me'.encode('cp1251') + body=u'click me'.encode('cp1251') ) req = self._assert_followed_url( resp2.css('a')[0], From 5b79c6a679b66868c89302a1693e5dedc62b6f61 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 16 Feb 2017 00:06:52 +0500 Subject: [PATCH 65/91] DOC document response.follow methods; expand the tutorial --- docs/intro/tutorial.rst | 41 +++++++++++++++++++++++++------- docs/topics/request-response.rst | 7 +++--- scrapy/http/response/__init__.py | 15 ++++++------ scrapy/http/response/text.py | 18 +++++++------- 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index d47bf69e5..3b3bd8d21 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -399,7 +399,7 @@ quotes elements and put them together into a Python dictionary:: >>> Extracting data in our spider ------------------------------- +----------------------------- Let's get back to our spider. Until now, it doesn't extract any data in particular, just saves the whole HTML page to a local file. Let's integrate the @@ -551,8 +551,14 @@ In our example, it creates a sort of loop, following all the links to the next p until it doesn't find one -- handy for crawling blogs, forums and other sites with pagination. + +.. _response-follow-example: + +A shortcut for creating Requests +-------------------------------- + As a shortcut for creating Request objects you can use -:meth:`response.follow ` method:: +:meth:`response.follow `:: import scrapy @@ -571,13 +577,32 @@ As a shortcut for creating Request objects you can use 'tags': quote.css('div.tags a.tag::text').extract(), } - for href in response.css('li.next a::attr(href)'): - yield response.follow(href, callback=self.parse) + next_page = response.css('li.next a::attr(href)').extract_first() + if next_page is not None: + yield response.follow(next_page, callback=self.parse) -Unlike scrapy.Request, ``response.follow`` supports -relative URLs directly; you can also pass a selector to it instead of -a string. Note that ``response.follow`` just returns a Request instance; -you still have to yield this Request. +Unlike scrapy.Request, ``response.follow`` supports relative URLs directly - no +need to call urljoin. Note that ``response.follow`` just returns a Request +instance; you still have to yield this Request. + +You can also pass a selector to ``response.follow`` instead of a string; +this selector should extract necessary attributes:: + + for href in response.css('li.next a::attr(href)'): + yield response.follow(href, callback=self.parse) + +For ```` elements there is a shortcut: ``response.follow`` uses their href +attribute automatically. So the code can be shortened further:: + + for a in response.css('li.next a'): + yield response.follow(a, callback=self.parse) + +.. note:: + + ``response.follow(response.css('li.next a'))`` is not valid because + ``response.css`` returns a list-like object with selectors for all results, + not a single selector. A ``for`` loop like in the example above, or + ``response.follow(response.css('li.next a')[0])`` is fine. More examples and patterns -------------------------- diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 71050fddd..3e80f18b5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -597,6 +597,9 @@ Response objects urlparse.urljoin(response.url, url) + .. automethod:: Response.follow + + .. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin .. _topics-request-response-ref-response-subclasses: @@ -683,9 +686,7 @@ TextResponse objects response.css('p') - .. method:: TextResponse.follow(url, ...) - - Return a scrapy.Request instance to follow a link ``url``. + .. automethod:: TextResponse.follow .. method:: TextResponse.body_as_unicode() diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index e5fb4eef8..434d87eab 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -109,13 +109,14 @@ class Response(object_ref): dont_filter=False, errback=None): # type: (...) -> Request """ - Return a scrapy.Request instance to follow a link ``url``. - - ``url`` can be: - - * absolute URL; - * relative URL; - * scrapy.link.Link object. + Return a :class:`~.Request` instance to follow a link ``url``. + It accepts the same arguments as ``Request.__init__`` method, + but ``url`` can be a relative URL or a ``scrapy.link.Link`` object, + not only an absolute URL. + + :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` + method which supports selectors in addition to absolute/relative URLs + and Link objects. """ if isinstance(url, Link): url = url.url diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 3c360bcf9..6415e191a 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -13,7 +13,6 @@ from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding from w3lib.html import strip_html5_whitespace -from scrapy.link import Link from scrapy.http.request import Request from scrapy.http.response import Response from scrapy.utils.response import get_base_url @@ -127,18 +126,19 @@ class TextResponse(Response): dont_filter=False, errback=None): # type: (...) -> Request """ - Return a scrapy.Request instance to follow a link ``url``. - - ``url`` can be: - - * absolute URL; - * relative URL; - * scrapy.link.Link object (e.g. a link extractor result); - * attribute Selector (not SelectorList) - e.g. + Return a :class:`~.Request` instance to follow a link ``url``. + It accepts the same arguments as ``Request.__init__`` method, + but ``url`` can be not only an absolute URL, but also + + * a relative URL; + * a scrapy.link.Link object (e.g. a link extractor result); + * an attribute Selector (not SelectorList) - e.g. ``response.css('a::attr(href)')[0]`` or ``response.xpath('//img/@src')[0]``. * a Selector for ```` element, e.g. ``response.css('a.my_link')[0]``. + + See :ref:`response-follow-example` for usage examples. """ if isinstance(url, parsel.Selector): url = _url_from_selector(url) From fade5763af3d03f076f3317589038201bbdeccaf Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 16 Feb 2017 02:02:50 +0500 Subject: [PATCH 66/91] TST more response.follow tests --- tests/test_http_response.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index fa74b468b..924bb7979 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -414,11 +414,24 @@ class TextResponseTest(BaseResponseTest): self.assertRaisesRegexp(ValueError, 'SelectorList', resp.follow, resp.css('a')) + def test_follow_selector_invalid(self): + resp = self._links_response() + self.assertRaisesRegexp(ValueError, 'Unsupported', + resp.follow, resp.xpath('count(//div)')[0]) + def test_follow_selector_attribute(self): resp = self._links_response() for src in resp.css('img::attr(src)'): self._assert_followed_url(src, 'http://example.com/sample2.jpg') + def test_follow_selector_no_href(self): + resp = self.response_class( + url='http://example.com', + body=b'click me', + ) + self.assertRaisesRegexp(ValueError, 'no href', + resp.follow, resp.css('a')[0]) + def test_follow_whitespace_selector(self): resp = self.response_class( 'http://example.com', From 074caf434e255bc96f106e57e3e288028f372485 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 9 Feb 2017 00:17:56 +0500 Subject: [PATCH 67/91] FormRequest: handle whitespaces in action attribute properly --- scrapy/http/request/form.py | 10 ++++++++-- tests/test_http_request.py | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 2862dc096..905d8412f 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,10 +5,13 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +import six from six.moves.urllib.parse import urljoin, urlencode + import lxml.html from parsel.selector import create_root_node -import six +from w3lib.html import strip_html5_whitespace + from scrapy.http.request import Request from scrapy.utils.python import to_bytes, is_listlike from scrapy.utils.response import get_base_url @@ -51,7 +54,10 @@ class FormRequest(Request): def _get_form_url(form, url): if url is None: - return urljoin(form.base_url, form.action) + action = form.get('action') + if action is None: + return form.base_url + return urljoin(form.base_url, strip_html5_whitespace(action)) return urljoin(form.base_url, url) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index d7216e1d2..7eadb874f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -556,7 +556,6 @@ class FormRequestTest(RequestTest): fs = _qs(req, to_unicode=True, encoding='latin1') self.assertTrue(fs[u'price in \u00a5']) - def test_from_response_multiple_forms_clickdata(self): response = _buildresponse( """ @@ -989,7 +988,7 @@ class FormRequestTest(RequestTest): """ - + @@ -1002,6 +1001,11 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response) self.assertEqual(req.url, 'http://b.com/test_form') + def test_spaces_in_action(self): + resp = _buildresponse('') + req = self.request_class.from_response(resp) + self.assertEqual(req.url, 'http://example.com/path') + def test_from_response_css(self): response = _buildresponse( """
@@ -1023,12 +1027,14 @@ class FormRequestTest(RequestTest): self.assertRaises(ValueError, self.request_class.from_response, response, formcss="input[name='abc']") + def _buildresponse(body, **kwargs): kwargs.setdefault('body', body) kwargs.setdefault('url', 'http://example.com') kwargs.setdefault('encoding', 'utf-8') return HtmlResponse(**kwargs) + def _qs(req, encoding='utf-8', to_unicode=False): if req.method == 'POST': qs = req.body From ad36a4a6ae8376a779f9feb08adfb2ca4a59dbb4 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 8 Feb 2017 16:58:38 +0500 Subject: [PATCH 68/91] RegexLinkExtractor: add \x0c to whitespace characters, as per html5 standard --- scrapy/linkextractors/regex.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py index 0fc7b079f..e689b4727 100644 --- a/scrapy/linkextractors/regex.py +++ b/scrapy/linkextractors/regex.py @@ -10,9 +10,10 @@ linkre = re.compile( "|\s.*?>)(.*?)<[/ ]?a>", re.DOTALL | re.IGNORECASE) + def clean_link(link_text): """Remove leading and trailing whitespace and punctuation""" - return link_text.strip("\t\r\n '\"") + return link_text.strip("\t\r\n '\"\x0c") class RegexLinkExtractor(SgmlLinkExtractor): From d079e15fe2fcf269d040ac435e2eb414d2b4c334 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 8 Feb 2017 17:03:11 +0500 Subject: [PATCH 69/91] Strip leading/trailing whitespaces in link extractors. Fixes GH-838. --- docs/topics/link-extractors.rst | 10 +++++++++- scrapy/linkextractors/htmlparser.py | 8 ++++++-- scrapy/linkextractors/lxmlhtml.py | 16 ++++++++++----- scrapy/linkextractors/sgml.py | 13 ++++++++---- scrapy/utils/url.py | 13 ++++++++++++ .../link_extractor/sgml_linkextractor.html | 1 + tests/test_linkextractors.py | 3 +++ tests/test_linkextractors_deprecated.py | 20 +++++++++++++------ 8 files changed, 66 insertions(+), 18 deletions(-) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 4636ddb18..2486e0982 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -51,7 +51,7 @@ LxmlLinkExtractor :synopsis: lxml's HTMLParser-based link extractors -.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None) +.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, strip=True) LxmlLinkExtractor is the recommended link extractor with handy filtering options. It is implemented using lxml's robust HTMLParser. @@ -132,4 +132,12 @@ LxmlLinkExtractor :type process_value: callable + :param strip: whether to strip whitespaces from extracted attributes. + According to HTML5 standard, leading and trailing whitespaces + must be stripped from ``href`` attributes of ```` and ```` + elements, so LinkExtractor strips them by default. Set ``strip=False`` + to turn it off (e.g. if you're extracting urls from elements or + attributes which allow leading/trailing whitespaces). + :type strip: boolean + .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py index 9867e1179..4841e4a54 100644 --- a/scrapy/linkextractors/htmlparser.py +++ b/scrapy/linkextractors/htmlparser.py @@ -1,7 +1,6 @@ """ HTMLParser-based link extractor """ - import warnings import six from six.moves.html_parser import HTMLParser @@ -11,12 +10,14 @@ from w3lib.url import safe_url_string from scrapy.link import Link from scrapy.utils.python import unique as unique_list +from scrapy.utils.url import trim_href_attribute from scrapy.exceptions import ScrapyDeprecationWarning class HtmlParserLinkExtractor(HTMLParser): - def __init__(self, tag="a", attr="href", process=None, unique=False): + def __init__(self, tag="a", attr="href", process=None, unique=False, + strip=True): HTMLParser.__init__(self) warnings.warn( @@ -29,6 +30,7 @@ class HtmlParserLinkExtractor(HTMLParser): self.scan_attr = attr if callable(attr) else lambda a: a == attr self.process_attr = process if callable(process) else lambda v: v self.unique = unique + self.strip = strip def _extract_links(self, response_text, response_url, response_encoding): self.reset() @@ -70,6 +72,8 @@ class HtmlParserLinkExtractor(HTMLParser): for attr, value in attrs: if self.scan_attr(attr): url = self.process_attr(value) + if self.strip: + url = trim_href_attribute(url) link = Link(url=url) self.links.append(link) self.current_link = link diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 71d57b392..f753033ab 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -9,8 +9,9 @@ import lxml.etree as etree from scrapy.link import Link from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list, to_native_str -from scrapy.linkextractors import FilteringLinkExtractor from scrapy.utils.response import get_base_url +from scrapy.utils.url import trim_href_attribute +from scrapy.linkextractors import FilteringLinkExtractor # from lxml/src/lxml/html/__init__.py @@ -27,11 +28,13 @@ def _nons(tag): class LxmlParserLinkExtractor(object): - def __init__(self, tag="a", attr="href", process=None, unique=False): + def __init__(self, tag="a", attr="href", process=None, unique=False, + strip=True): self.scan_tag = tag if callable(tag) else lambda t: t == tag self.scan_attr = attr if callable(attr) else lambda a: a == attr self.process_attr = process if callable(process) else lambda v: v self.unique = unique + self.strip = strip def _iter_links(self, document): for el in document.iter(etree.Element): @@ -49,9 +52,11 @@ class LxmlParserLinkExtractor(object): for el, attr, attr_val in self._iter_links(selector.root): # pseudo lxml.html.HtmlElement.make_links_absolute(base_url) try: + if self.strip: + attr_val = trim_href_attribute(attr_val) attr_val = urljoin(base_url, attr_val) except ValueError: - continue # skipping bogus links + continue # skipping bogus links else: url = self.process_attr(attr_val) if url is None: @@ -85,12 +90,13 @@ class LxmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, - unique=True, process_value=None, deny_extensions=None, restrict_css=()): + unique=True, process_value=None, deny_extensions=None, restrict_css=(), + strip=True): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) tag_func = lambda x: x in tags attr_func = lambda x: x in attrs lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process=process_value) + unique=unique, process=process_value, strip=strip) super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, allow_domains=allow_domains, deny_domains=deny_domains, diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index c68dae4c8..6ecfd52aa 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -7,18 +7,19 @@ import warnings from sgmllib import SGMLParser from w3lib.url import safe_url_string -from scrapy.selector import Selector from scrapy.link import Link from scrapy.linkextractors import FilteringLinkExtractor from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list, to_unicode from scrapy.utils.response import get_base_url +from scrapy.utils.url import trim_href_attribute from scrapy.exceptions import ScrapyDeprecationWarning class BaseSgmlLinkExtractor(SGMLParser): - def __init__(self, tag="a", attr="href", unique=False, process_value=None): + def __init__(self, tag="a", attr="href", unique=False, process_value=None, + strip=True): warnings.warn( "BaseSgmlLinkExtractor is deprecated and will be removed in future releases. " "Please use scrapy.linkextractors.LinkExtractor", @@ -30,6 +31,7 @@ class BaseSgmlLinkExtractor(SGMLParser): self.process_value = (lambda v: v) if process_value is None else process_value self.current_link = None self.unique = unique + self.strip = strip def _extract_links(self, response_text, response_url, response_encoding, base_url=None): """ Do the real extraction work """ @@ -81,6 +83,8 @@ class BaseSgmlLinkExtractor(SGMLParser): if self.scan_attr(attr): url = self.process_value(value) if url is not None: + if self.strip: + url = trim_href_attribute(url) link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel'))) self.links.append(link) self.current_link = link @@ -103,7 +107,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, - process_value=None, deny_extensions=None, restrict_css=()): + process_value=None, deny_extensions=None, restrict_css=(), + strip=True): warnings.warn( "SgmlLinkExtractor is deprecated and will be removed in future releases. " @@ -118,7 +123,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor): with warnings.catch_warnings(): warnings.simplefilter('ignore', ScrapyDeprecationWarning) lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func, - unique=unique, process_value=process_value) + unique=unique, process_value=process_value, strip=strip) super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, allow_domains=allow_domains, deny_domains=deny_domains, diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index dc1cce4ac..090f65f80 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -103,3 +103,16 @@ def guess_scheme(url): return any_to_uri(url) else: return add_http_if_no_scheme(url) + + +def trim_href_attribute(href): + """ + Process href attribute of ``a`` or ``area`` elements according to HTML5 + standards (strip all leading and trailing whitespaces). References: + + * https://www.w3.org/TR/html5/links.html#links-created-by-a-and-area-elements + * https://www.w3.org/TR/html5/infrastructure.html#valid-url-potentially-surrounded-by-spaces + * https://www.w3.org/TR/html5/infrastructure.html#strip-leading-and-trailing-whitespace + * https://www.w3.org/TR/html5/infrastructure.html#space-character + """ + return href.strip(' \t\n\r\x0c') diff --git a/tests/sample_data/link_extractor/sgml_linkextractor.html b/tests/sample_data/link_extractor/sgml_linkextractor.html index 35aa457ee..fbb803f2d 100644 --- a/tests/sample_data/link_extractor/sgml_linkextractor.html +++ b/tests/sample_data/link_extractor/sgml_linkextractor.html @@ -13,6 +13,7 @@ sample 3 repetition inner tag +href with whitespaces
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 129336d14..340c64f35 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -32,6 +32,7 @@ class Base: Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), + Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), ]) def test_extract_filter_allow(self): @@ -281,6 +282,7 @@ class Base: Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), + Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), ]) lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=()) @@ -291,6 +293,7 @@ class Base: Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://www.google.com/something', text=u''), Link(url='http://example.com/innertag.html', text=u'inner tag'), + Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), ]) lx = self.extractor_cls(attrs=None) diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py index 36dfe174f..fef227aa1 100644 --- a/tests/test_linkextractors_deprecated.py +++ b/tests/test_linkextractors_deprecated.py @@ -117,12 +117,14 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase): def test_extraction(self): # Default arguments lx = HtmlParserLinkExtractor() - self.assertEqual(lx.extract_links(self.response), - [Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'),]) + self.assertEqual(lx.extract_links(self.response), [ + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + Link(url='http://www.google.com/something', text=u''), + Link(url='http://example.com/innertag.html', text=u'inner tag'), + Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), + ]) def test_link_wrong_href(self): html = """ @@ -220,3 +222,9 @@ class RegexLinkExtractorTestCase(unittest.TestCase): self.assertEqual([link for link in lx.extract_links(response)], [ Link(url='http://b.com/test.html', text=u'', nofollow=False), ]) + + @unittest.expectedFailure + def test_extraction(self): + # RegexLinkExtractor doesn't parse URLs with leading/trailing + # whitespaces correctly. + super(RegexLinkExtractorTestCase, self).test_extraction() From d09eed7674b5df2a1883a7c7abad40fdfd062c74 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 8 Feb 2017 23:44:55 +0500 Subject: [PATCH 70/91] use w3lib.html.strip_html5_whitespace function; expand docs; strip consistently before calling process_value --- docs/topics/link-extractors.rst | 9 +++++---- scrapy/linkextractors/htmlparser.py | 6 +++--- scrapy/linkextractors/lxmlhtml.py | 6 +++--- scrapy/linkextractors/sgml.py | 7 ++++--- scrapy/utils/url.py | 13 ------------- 5 files changed, 15 insertions(+), 26 deletions(-) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 2486e0982..75bdb4142 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -134,10 +134,11 @@ LxmlLinkExtractor :param strip: whether to strip whitespaces from extracted attributes. According to HTML5 standard, leading and trailing whitespaces - must be stripped from ``href`` attributes of ```` and ```` - elements, so LinkExtractor strips them by default. Set ``strip=False`` - to turn it off (e.g. if you're extracting urls from elements or - attributes which allow leading/trailing whitespaces). + must be stripped from ``href`` attributes of ````, ```` + and many other elements, ``src`` attribute of ````, ``