From 7f4f98fd38d3fdf6b45a9d0289df0cbb48bcd22b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 30 Sep 2019 18:22:28 +0200 Subject: [PATCH 01/85] Provide complete API documentation coverage of scrapy.linkextractors --- docs/conf.py | 4 +++ docs/topics/link-extractors.rst | 45 ++++++++++++------------------- scrapy/linkextractors/__init__.py | 11 ++++++++ scrapy/linkextractors/lxmlhtml.py | 8 ++++++ tests/test_linkextractors.py | 30 +++++++++++++++++++++ 5 files changed, 70 insertions(+), 28 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 34dd5bcb7..5ba6b8d43 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -265,6 +265,10 @@ coverage_ignore_pyobjects = [ # Never documented before, and deprecated now. r'^scrapy\.item\.DictItem$', + r'^scrapy\.linkextractors\.FilteringLinkExtractor$', + + # Implementation detail of LxmlLinkExtractor + r'^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor', ] diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 713a94e10..f9936a498 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -4,46 +4,33 @@ Link Extractors =============== -Link extractors are objects whose only purpose is to extract links from web -pages (:class:`scrapy.http.Response` objects) which will be eventually -followed. +A link extractor is an object that extracts links from responses. -There is ``scrapy.linkextractors.LinkExtractor`` available -in Scrapy, but you can create your own custom Link Extractors to suit your -needs by implementing a simple interface. - -The only public method that every link extractor has is ``extract_links``, -which receives a :class:`~scrapy.http.Response` object and returns a list -of :class:`scrapy.link.Link` objects. Link extractors are meant to be -instantiated once and their ``extract_links`` method called several times -with different responses to extract links to follow. - -Link extractors are used in the :class:`~scrapy.spiders.CrawlSpider` -class (available in Scrapy), through a set of rules, but you can also use it in -your spiders, even if you don't subclass from -:class:`~scrapy.spiders.CrawlSpider`, as its purpose is very simple: to -extract links. +The constructor of :class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` +takes settings that determine which links may be extracted. +:class:`LxmlLinkExtractor.extract_links +` returns a +list of matching :class:`scrapy.link.Link` objects from a +:class:`~scrapy.http.Response` object. +Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders +through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link +extractors in regular spiders. .. _topics-link-extractors-ref: -Built-in link extractors reference -================================== +Link extractor reference +======================== .. module:: scrapy.linkextractors :synopsis: Link extractors classes -Link extractors classes bundled with Scrapy are provided in the -:mod:`scrapy.linkextractors` module. - -The default link extractor is ``LinkExtractor``, which is the same as -:class:`~.LxmlLinkExtractor`:: +The link extractor class is +:class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it +can also be imported as ``scrapy.linkextractors.LinkExtractor``:: from scrapy.linkextractors import LinkExtractor -There used to be other link extractor classes in previous Scrapy versions, -but they are deprecated now. - LxmlLinkExtractor ----------------- @@ -152,4 +139,6 @@ LxmlLinkExtractor from elements or attributes which allow leading/trailing whitespaces). :type strip: boolean + .. automethod:: extract_links + .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index ebf3cd7d8..ca80dc339 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -6,11 +6,13 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ import re +from warnings import warn from six.moves.urllib.parse import urlparse from parsel.csstranslator import HTMLTranslator from w3lib.url import canonicalize_url +from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.misc import arg_to_iter from scrapy.utils.url import ( url_is_from_any_domain, url_has_any_extension, @@ -49,6 +51,15 @@ class FilteringLinkExtractor(object): _csstranslator = HTMLTranslator() + def __new__(cls, *args, **kwargs): + from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor + if (issubclass(cls, FilteringLinkExtractor) and + not issubclass(cls, LxmlLinkExtractor)): + warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' + 'please use scrapy.linkextractors.LinkExtractor instead', + ScrapyDeprecationWarning, stacklevel=2) + return super(FilteringLinkExtractor, cls).__new__(cls) + def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text): diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 8f6f93a44..41091ba23 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -117,6 +117,14 @@ class LxmlLinkExtractor(FilteringLinkExtractor): restrict_text=restrict_text) def extract_links(self, response): + """Returns a list of :class:`~scrapy.link.Link` objects from the + specified :class:`response `. + + Only links that match the settings passed to the link extractor + constructor are returned. + + Duplicate links are omitted. + """ base_url = get_base_url(response) if self.restrict_xpaths: docs = [subdoc diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index d96e259f6..ea6db28c0 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,10 +1,13 @@ import re import unittest +from warnings import catch_warnings import pytest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link +from scrapy.linkextractors import FilteringLinkExtractor from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor from tests import get_testdata @@ -506,3 +509,30 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): @pytest.mark.xfail def test_restrict_xpaths_with_html_entities(self): super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() + + def test_filteringlinkextractor_deprecation_warning(self): + """Make sure the FilteringLinkExtractor deprecation warning is not + issued for LxmlLinkExtractor""" + with catch_warnings(record=True) as warnings: + extractor = LxmlLinkExtractor() + self.assertEqual(len(warnings), 0) + class SubclassedItem(LxmlLinkExtractor): + pass + subclassed_extractor = SubclassedItem() + self.assertEqual(len(warnings), 0) + + +class FilteringLinkExtractorTest(unittest.TestCase): + + def test_deprecation_warning(self): + args = [None]*10 + with catch_warnings(record=True) as warnings: + extractor = FilteringLinkExtractor(*args) + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + with catch_warnings(record=True) as warnings: + class SubclassedFilteringLinkExtractor(FilteringLinkExtractor): + pass + subclassed_extractor = SubclassedFilteringLinkExtractor(*args) + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) From 0fbd1ff4a91c68e552a3f919824c60437fc9a141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 21 Oct 2019 14:06:45 +0200 Subject: [PATCH 02/85] =?UTF-8?q?constructor=20=E2=86=92=20=5F=5Finit=5F?= =?UTF-8?q?=5F=20method?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/link-extractors.rst | 6 +++--- scrapy/linkextractors/lxmlhtml.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index f9936a498..2119cb8f8 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -6,9 +6,9 @@ Link Extractors A link extractor is an object that extracts links from responses. -The constructor of :class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` -takes settings that determine which links may be extracted. -:class:`LxmlLinkExtractor.extract_links +The ``__init__`` method of +:class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that +determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links ` returns a list of matching :class:`scrapy.link.Link` objects from a :class:`~scrapy.http.Response` object. diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 41091ba23..37003720f 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -120,8 +120,8 @@ class LxmlLinkExtractor(FilteringLinkExtractor): """Returns a list of :class:`~scrapy.link.Link` objects from the specified :class:`response `. - Only links that match the settings passed to the link extractor - constructor are returned. + Only links that match the settings passed to the ``__init__`` method of + the link extractor are returned. Duplicate links are omitted. """ From 84fe4011b0063dae1a8efbcb563e772d3b9fce09 Mon Sep 17 00:00:00 2001 From: Kevin Lloyd Bernal Date: Wed, 23 Oct 2019 20:39:53 +0800 Subject: [PATCH 03/85] update docs of scrapy.loader.ItemLoader.item --- docs/topics/loaders.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 1c2f1da4d..4bd564014 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -485,6 +485,8 @@ ItemLoader objects .. attribute:: item The :class:`~scrapy.item.Item` object being parsed by this Item Loader. + This is mostly used as a property so when attempting to override this + value, you may want to check out :attr:`default_item_class` first. .. attribute:: context From b44bd6f8250579dc9ddc25d5e0f10d8b3790f5ea Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 22 Jul 2019 20:51:03 +0500 Subject: [PATCH 04/85] Remove Python 2-only tests. --- conftest.py | 10 +- tests/py3-ignores.txt | 3 - tests/test_downloader_handlers.py | 22 +-- tests/test_linkextractors_deprecated.py | 233 ------------------------ tests/test_proxy_connect.py | 120 ------------ tests/test_utils_python.py | 27 --- tests/test_webclient.py | 20 -- 7 files changed, 14 insertions(+), 421 deletions(-) delete mode 100644 tests/test_linkextractors_deprecated.py delete mode 100644 tests/test_proxy_connect.py diff --git a/conftest.py b/conftest.py index 06d65ba1d..ede091e9f 100644 --- a/conftest.py +++ b/conftest.py @@ -1,4 +1,3 @@ -import six import pytest @@ -8,11 +7,10 @@ collect_ignore = [ ] -if six.PY3: - for line in open('tests/py3-ignores.txt'): - file_path = line.strip() - if file_path and file_path[0] != '#': - collect_ignore.append(file_path) +for line in open('tests/py3-ignores.txt'): + file_path = line.strip() + if file_path and file_path[0] != '#': + collect_ignore.append(file_path) @pytest.fixture() diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 313e74ec9..45cf6fb92 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -1,6 +1,3 @@ -tests/test_linkextractors_deprecated.py -tests/test_proxy_connect.py - scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py scrapy/linkextractors/htmlparser.py diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 109469503..4d3c4d4aa 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -633,18 +633,16 @@ class Http11MockServerTestCase(unittest.TestCase): # download_maxsize < 100, hence the CancelledError self.assertIsInstance(failure.value, defer.CancelledError) - if six.PY2: - request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') - request = request.replace(url=self.mockserver.url('/xpayload')) - yield crawler.crawl(seed=request) - # download_maxsize = 50 is enough for the gzipped response - failure = crawler.spider.meta.get('failure') - self.assertTrue(failure == None) - reason = crawler.spider.meta['close_reason'] - self.assertTrue(reason, 'finished') - else: - # See issue https://twistedmatrix.com/trac/ticket/8175 - raise unittest.SkipTest("xpayload only enabled for PY2") + # See issue https://twistedmatrix.com/trac/ticket/8175 + raise unittest.SkipTest("xpayload only enabled for PY2") + request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') + request = request.replace(url=self.mockserver.url('/xpayload')) + yield crawler.crawl(seed=request) + # download_maxsize = 50 is enough for the gzipped response + failure = crawler.spider.meta.get('failure') + self.assertTrue(failure == None) + reason = crawler.spider.meta['close_reason'] + self.assertTrue(reason, 'finished') class UriResource(resource.Resource): diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py deleted file mode 100644 index 1366971be..000000000 --- a/tests/test_linkextractors_deprecated.py +++ /dev/null @@ -1,233 +0,0 @@ -# -*- coding: utf-8 -*- -import unittest -from scrapy.linkextractors.regex import RegexLinkExtractor -from scrapy.http import HtmlResponse -from scrapy.link import Link -from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor -from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor -from tests import get_testdata - -from tests.test_linkextractors import Base - - -class BaseSgmlLinkExtractorTestCase(unittest.TestCase): - # XXX: should we move some of these tests to base link extractor tests? - - def test_basic(self): - html = """Page title<title> - <body><p><a href="item/12.html">Item 12</a></p> - <p><a href="/about.html">About us</a></p> - <img src="/logo.png" alt="Company logo (not a link)" /> - <p><a href="../othercat.html">Other category</a></p> - <p><a href="/">>></a></p> - <p><a href="/" /></p> - </body></html>""" - response = HtmlResponse("http://example.org/somepage/index.html", body=html) - - lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/somepage/item/12.html', text='Item 12'), - Link(url='http://example.org/about.html', text='About us'), - Link(url='http://example.org/othercat.html', text='Other category'), - Link(url='http://example.org/', text='>>'), - Link(url='http://example.org/', text='')]) - - def test_base_url(self): - html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" /> - <body><p><a href="item/12.html">Item 12</a></p> - </body></html>""" - response = HtmlResponse("http://example.org/somepage/index.html", body=html) - - lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href - self.assertEqual(lx.extract_links(response), - [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) - - # base url is an absolute path and relative to host - html = """<html><head><title>Page title<title><base href="/" /> - <body><p><a href="item/12.html">Item 12</a></p></body></html>""" - response = HtmlResponse("https://example.org/somepage/index.html", body=html) - self.assertEqual(lx.extract_links(response), - [Link(url='https://example.org/item/12.html', text='Item 12')]) - - # base url has no scheme - html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" /> - <body><p><a href="item/12.html">Item 12</a></p></body></html>""" - response = HtmlResponse("https://example.org/somepage/index.html", body=html) - self.assertEqual(lx.extract_links(response), - [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')]) - - def test_link_text_wrong_encoding(self): - html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>""" - response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8') - lx = BaseSgmlLinkExtractor() - self.assertEqual(lx.extract_links(response), [ - Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'), - ]) - - def test_extraction_encoding(self): - body = get_testdata('link_extractor', 'linkextractor_noenc.html') - response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']}) - response_noenc = HtmlResponse(url='http://example.com/noenc', body=body) - body = get_testdata('link_extractor', 'linkextractor_latin1.html') - response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body) - - lx = BaseSgmlLinkExtractor() - self.assertEqual(lx.extract_links(response_utf8), [ - Link(url='http://example.com/sample_%C3%B1.html', text=''), - Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')), - ]) - - self.assertEqual(lx.extract_links(response_noenc), [ - Link(url='http://example.com/sample_%C3%B1.html', text=''), - Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')), - ]) - - # document encoding does not affect URL path component, only query part - # >>> u'sample_ñ.html'.encode('utf8') - # b'sample_\xc3\xb1.html' - # >>> u"sample_á.html".encode('utf8') - # b'sample_\xc3\xa1.html' - # >>> u"sample_ö.html".encode('utf8') - # b'sample_\xc3\xb6.html' - # >>> u"£32".encode('latin1') - # b'\xa332' - # >>> u"µ".encode('latin1') - # b'\xb5' - self.assertEqual(lx.extract_links(response_latin1), [ - Link(url='http://example.com/sample_%C3%B1.html', text=''), - Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')), - Link(url='http://example.com/sample_%C3%B6.html?price=%A332&%B5=unit', text=''), - ]) - - def test_matches(self): - url1 = 'http://lotsofstuff.com/stuff1/index' - url2 = 'http://evenmorestuff.com/uglystuff/index' - - lx = BaseSgmlLinkExtractor() - self.assertEqual(lx.matches(url1), True) - self.assertEqual(lx.matches(url2), True) - - -class HtmlParserLinkExtractorTestCase(unittest.TestCase): - - def setUp(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') - self.response = HtmlResponse(url='http://example.com/index', body=body) - - def test_extraction(self): - # Default arguments - lx = HtmlParserLinkExtractor() - self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), - Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), - ]) - - def test_link_wrong_href(self): - html = """ - <a href="http://example.org/item1.html">Item 1</a> - <a href="http://[example.org/item2.html">Item 2</a> - <a href="http://example.org/item3.html">Item 3</a> - """ - response = HtmlResponse("http://example.org/index.html", body=html) - lx = HtmlParserLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), - ]) - - -class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase): - extractor_cls = SgmlLinkExtractor - escapes_whitespace = True - - def test_deny_extensions(self): - html = """<a href="page.html">asd</a> and <a href="photo.jpg">""" - response = HtmlResponse("http://example.org/", body=html) - lx = SgmlLinkExtractor(deny_extensions="jpg") - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/page.html', text=u'asd'), - ]) - - def test_attrs_sgml(self): - html = """<html><area href="sample1.html"></area> - <a ref="sample2.html">sample text 2</a></html>""" - response = HtmlResponse("http://example.com/index.html", body=html) - lx = SgmlLinkExtractor(attrs="href") - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), - ]) - - def test_link_nofollow(self): - html = """ - <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a> - <a href="about.html">About us</a> - <a href="http://google.com/something" rel="external nofollow">Something</a> - """ - response = HtmlResponse("http://example.org/page.html", body=html) - lx = SgmlLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True), - Link(url='http://example.org/about.html', text=u'About us', nofollow=False), - Link(url='http://google.com/something', text=u'Something', nofollow=True), - ]) - - -class RegexLinkExtractorTestCase(unittest.TestCase): - # XXX: RegexLinkExtractor is not deprecated yet, but it must be rewritten - # not to depend on SgmlLinkExractor. Its speed is also much worse - # than it should be. - - def setUp(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') - self.response = HtmlResponse(url='http://example.com/index', body=body) - - def test_extraction(self): - # Default arguments - lx = RegexLinkExtractor() - self.assertEqual(lx.extract_links(self.response), - [Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'),]) - - def test_link_wrong_href(self): - html = """ - <a href="http://example.org/item1.html">Item 1</a> - <a href="http://[example.org/item2.html">Item 2</a> - <a href="http://example.org/item3.html">Item 3</a> - """ - response = HtmlResponse("http://example.org/index.html", body=html) - lx = RegexLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), - ]) - - def test_html_base_href(self): - html = """ - <html> - <head> - <base href="http://b.com/"> - </head> - <body> - <a href="test.html"></a> - </body> - </html> - """ - response = HtmlResponse("http://a.com/", body=html) - lx = RegexLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://b.com/test.html', text=u'', nofollow=False), - ]) - - @unittest.expectedFailure - def test_extraction(self): - # RegexLinkExtractor doesn't parse URLs with leading/trailing - # whitespaces correctly. - super(RegexLinkExtractorTestCase, self).test_extraction() diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py deleted file mode 100644 index ae1236bcb..000000000 --- a/tests/test_proxy_connect.py +++ /dev/null @@ -1,120 +0,0 @@ -import json -import os -import time - -from six.moves.urllib.parse import urlsplit, urlunsplit -from threading import Thread -from libmproxy import controller, proxy -from netlib import http_auth -from testfixtures import LogCapture - -from twisted.internet import defer -from twisted.trial.unittest import TestCase -from scrapy.utils.test import get_crawler -from scrapy.http import Request -from tests.spiders import SimpleSpider, SingleRequestSpider -from tests.mockserver import MockServer - - -class HTTPSProxy(controller.Master, Thread): - - def __init__(self): - password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy') - authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") - cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'keys', 'mitmproxy-ca.pem') - server = proxy.ProxyServer(proxy.ProxyConfig( - authenticator = authenticator, - cacert = cert_path), - 0) - self.server = server - Thread.__init__(self) - controller.Master.__init__(self, server) - - def http_address(self): - return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname() - - -def _wrong_credentials(proxy_url): - bad_auth_proxy = list(urlsplit(proxy_url)) - bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@') - return urlunsplit(bad_auth_proxy) - -class ProxyConnectTestCase(TestCase): - - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() - self._oldenv = os.environ.copy() - - self._proxy = HTTPSProxy() - self._proxy.start() - - # Wait for the proxy to start. - time.sleep(1.0) - os.environ['https_proxy'] = self._proxy.http_address() - os.environ['http_proxy'] = self._proxy.http_address() - - def tearDown(self): - self.mockserver.__exit__(None, None, None) - self._proxy.shutdown() - os.environ = self._oldenv - - @defer.inlineCallbacks - def test_https_connect_tunnel(self): - crawler = get_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) - - @defer.inlineCallbacks - def test_https_noconnect(self): - proxy = os.environ['https_proxy'] - os.environ['https_proxy'] = proxy + '?noconnect' - crawler = get_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) - - @defer.inlineCallbacks - def test_https_connect_tunnel_error(self): - crawler = get_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl("https://localhost:99999/status?n=200") - self._assert_got_tunnel_error(l) - - @defer.inlineCallbacks - def test_https_tunnel_auth_error(self): - os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) - crawler = get_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - # The proxy returns a 407 error code but it does not reach the client; - # he just sees a TunnelError. - self._assert_got_tunnel_error(l) - - @defer.inlineCallbacks - def test_https_tunnel_without_leak_proxy_authorization_header(self): - request = Request(self.mockserver.url("/echo", is_secure=True)) - crawler = get_crawler(SingleRequestSpider) - with LogCapture() as l: - yield crawler.crawl(seed=request) - self._assert_got_response_code(200, l) - echo = json.loads(crawler.spider.meta['responses'][0].body) - self.assertTrue('Proxy-Authorization' not in echo['headers']) - - @defer.inlineCallbacks - def test_https_noconnect_auth_error(self): - os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' - crawler = get_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(407, l) - - def _assert_got_response_code(self, code, log): - print(log) - self.assertEqual(str(log).count('Crawled (%d)' % code), 1) - - def _assert_got_tunnel_error(self, log): - print(log) - self.assertIn('TunnelError', str(log)) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3e1148354..6cb32cbdd 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -163,33 +163,6 @@ class UtilsPythonTestCase(unittest.TestCase): gc.collect() self.assertFalse(len(wk._weakdict)) - @unittest.skipUnless(six.PY2, "deprecated function") - def test_stringify_dict(self): - d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'} - d2 = stringify_dict(d, keys_only=False) - self.assertEqual(d, d2) - self.assertIsNot(d, d2) # shouldn't modify in place - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) - - @unittest.skipUnless(six.PY2, "deprecated function") - def test_stringify_dict_tuples(self): - tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')] - d = dict(tuples) - d2 = stringify_dict(tuples, keys_only=False) - self.assertEqual(d, d2) - self.assertIsNot(d, d2) # shouldn't modify in place - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys()) - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) - - @unittest.skipUnless(six.PY2, "deprecated function") - def test_stringify_dict_keys_only(self): - d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'} - d2 = stringify_dict(d) - self.assertEqual(d, d2) - self.assertIsNot(d, d2) # shouldn't modify in place - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) - def test_get_func_args(self): def f1(a, b, c): pass diff --git a/tests/test_webclient.py b/tests/test_webclient.py index a81946490..7b015ff8d 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -78,26 +78,6 @@ class ParseUrlTestCase(unittest.TestCase): to_bytes(x) if not isinstance(x, int) else x for x in test) self.assertEqual(client._parse(url), test, url) - def test_externalUnicodeInterference(self): - """ - L{client._parse} should return C{str} for the scheme, host, and path - elements of its return tuple, even when passed an URL which has - previously been passed to L{urlparse} as a C{unicode} string. - """ - if not six.PY2: - raise unittest.SkipTest( - "Applies only to Py2, as urls can be ONLY unicode on Py3") - badInput = u'http://example.com/path' - goodInput = badInput.encode('ascii') - self._parse(badInput) # cache badInput in urlparse_cached - scheme, netloc, host, port, path = self._parse(goodInput) - self.assertTrue(isinstance(scheme, str)) - self.assertTrue(isinstance(netloc, str)) - self.assertTrue(isinstance(host, str)) - self.assertTrue(isinstance(path, str)) - self.assertTrue(isinstance(port, int)) - - class ScrapyHTTPPageGetterTests(unittest.TestCase): From b0d6f4917d782f2396a702e90c36ffbc8bb42e7e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 3 Sep 2019 15:17:03 +0500 Subject: [PATCH 05/85] Restore tests/test_proxy_connect.py and update it to modern mitmproxy. --- tests/requirements-py3.txt | 1 + tests/test_proxy_connect.py | 134 ++++++++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 tests/test_proxy_connect.py diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index dd5b23cc3..f27e45a54 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,5 +1,6 @@ # Tests requirements jmespath +mitmproxy pytest pytest-cov pytest-twisted diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py new file mode 100644 index 000000000..8142d9a41 --- /dev/null +++ b/tests/test_proxy_connect.py @@ -0,0 +1,134 @@ +import json +import os +import re +from subprocess import Popen, PIPE +import sys +import time + +from six.moves.urllib.parse import urlsplit, urlunsplit +from testfixtures import LogCapture + +from twisted.internet import defer +from twisted.trial.unittest import TestCase + +from scrapy.utils.test import get_crawler +from scrapy.http import Request +from tests.spiders import SimpleSpider, SingleRequestSpider +from tests.mockserver import MockServer + + +class MitmProxy: + auth_user = 'scrapy' + auth_pass = 'scrapy' + + def start(self): + from scrapy.utils.test import get_testenv + script = """ +import sys +from mitmproxy.tools.main import mitmdump +sys.argv[0] = "mitmdump" +sys.exit(mitmdump()) + """ + cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'keys', 'mitmproxy-ca.pem') + self.proc = Popen([sys.executable, + '-c', script, + '--listen-host', '127.0.0.1', + '--listen-port', '0', + '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass), + '--certs', cert_path, + '--ssl-insecure', + ], + stdout=PIPE, env=get_testenv()) + line = self.proc.stdout.readline().decode('utf-8') + host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1) + address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port) + return address + + def stop(self): + self.proc.kill() + self.proc.wait() + time.sleep(0.2) + + +def _wrong_credentials(proxy_url): + bad_auth_proxy = list(urlsplit(proxy_url)) + bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@') + return urlunsplit(bad_auth_proxy) + + +class ProxyConnectTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self._oldenv = os.environ.copy() + + self._proxy = MitmProxy() + proxy_url = self._proxy.start() + os.environ['https_proxy'] = proxy_url + os.environ['http_proxy'] = proxy_url + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + self._proxy.stop() + os.environ = self._oldenv + + @defer.inlineCallbacks + def test_https_connect_tunnel(self): + crawler = get_crawler(SimpleSpider) + with LogCapture() as l: + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) + self._assert_got_response_code(200, l) + + @defer.inlineCallbacks + def test_https_noconnect(self): + proxy = os.environ['https_proxy'] + os.environ['https_proxy'] = proxy + '?noconnect' + crawler = get_crawler(SimpleSpider) + with LogCapture() as l: + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) + self._assert_got_response_code(200, l) + + @defer.inlineCallbacks + def test_https_connect_tunnel_error(self): + crawler = get_crawler(SimpleSpider) + with LogCapture() as l: + yield crawler.crawl("https://localhost:99999/status?n=200") + self._assert_got_tunnel_error(l) + + @defer.inlineCallbacks + def test_https_tunnel_auth_error(self): + os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + crawler = get_crawler(SimpleSpider) + with LogCapture() as l: + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) + # The proxy returns a 407 error code but it does not reach the client; + # he just sees a TunnelError. + self._assert_got_tunnel_error(l) + + @defer.inlineCallbacks + def test_https_tunnel_without_leak_proxy_authorization_header(self): + request = Request(self.mockserver.url("/echo", is_secure=True)) + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as l: + yield crawler.crawl(seed=request) + self._assert_got_response_code(200, l) + echo = json.loads(crawler.spider.meta['responses'][0].body) + self.assertTrue('Proxy-Authorization' not in echo['headers']) + + @defer.inlineCallbacks + def test_https_noconnect_auth_error(self): + os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' + crawler = get_crawler(SimpleSpider) + with LogCapture() as l: + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) + self._assert_got_response_code(407, l) + + def _assert_got_response_code(self, code, log): + print(log) + self.assertEqual(str(log).count('Crawled (%d)' % code), 1) + + def _assert_got_tunnel_error(self, log): + print(log) + self.assertIn('TunnelError', str(log)) From 439e37fc7b2c192b4591d29dde74c9f589763cc8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 3 Sep 2019 15:23:24 +0500 Subject: [PATCH 06/85] Mark failing proxy tests. --- tests/test_proxy_connect.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 8142d9a41..5e9470e39 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -5,6 +5,7 @@ from subprocess import Popen, PIPE import sys import time +import pytest from six.moves.urllib.parse import urlsplit, urlunsplit from testfixtures import LogCapture @@ -81,6 +82,7 @@ class ProxyConnectTestCase(TestCase): yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) + @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') @defer.inlineCallbacks def test_https_noconnect(self): proxy = os.environ['https_proxy'] @@ -90,6 +92,7 @@ class ProxyConnectTestCase(TestCase): yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) + @pytest.mark.xfail(reason='Python 3 fails this earlier') @defer.inlineCallbacks def test_https_connect_tunnel_error(self): crawler = get_crawler(SimpleSpider) @@ -117,6 +120,7 @@ class ProxyConnectTestCase(TestCase): echo = json.loads(crawler.spider.meta['responses'][0].body) self.assertTrue('Proxy-Authorization' not in echo['headers']) + @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') @defer.inlineCallbacks def test_https_noconnect_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' From 186f9d88acd9043cf5d8f21358c758c42e433105 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 3 Sep 2019 15:24:06 +0500 Subject: [PATCH 07/85] Fix the skip message for test_download_gzip_response. --- tests/test_downloader_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 4d3c4d4aa..9f74577a5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -634,7 +634,7 @@ class Http11MockServerTestCase(unittest.TestCase): self.assertIsInstance(failure.value, defer.CancelledError) # See issue https://twistedmatrix.com/trac/ticket/8175 - raise unittest.SkipTest("xpayload only enabled for PY2") + raise unittest.SkipTest("xpayload fails on PY3") request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') request = request.replace(url=self.mockserver.url('/xpayload')) yield crawler.crawl(seed=request) From bbd9f4be90e734df5c290aeb399e8e512f834224 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Mon, 22 Jul 2019 22:27:29 +0500 Subject: [PATCH 08/85] Remove six.PY2 and six.PY3 conditionals. --- docs/topics/downloader-middleware.rst | 6 +++--- scrapy/_monkeypatches.py | 10 ---------- scrapy/commands/fetch.py | 5 ++--- scrapy/crawler.py | 11 ----------- scrapy/exporters.py | 2 +- scrapy/extensions/feedexport.py | 3 +-- scrapy/http/request/form.py | 3 +-- scrapy/http/response/text.py | 3 --- scrapy/item.py | 8 +------- scrapy/link.py | 15 ++------------- scrapy/mail.py | 9 ++------- scrapy/settings/__init__.py | 8 +------- scrapy/settings/default_settings.py | 4 +--- scrapy/utils/boto.py | 10 +--------- scrapy/utils/conf.py | 5 +---- scrapy/utils/datatypes.py | 20 +++++++------------- scrapy/utils/gz.py | 13 +++---------- scrapy/utils/iterators.py | 6 +----- scrapy/utils/python.py | 10 ++-------- tests/__init__.py | 15 --------------- tests/test_http_request.py | 11 +++-------- tests/test_http_response.py | 3 +-- tests/test_item.py | 8 ++------ tests/test_link.py | 13 ++----------- tests/test_middleware.py | 21 ++++++--------------- tests/test_request_cb_kwargs.py | 8 +------- tests/test_settings/__init__.py | 8 +------- tests/test_utils_datatypes.py | 7 +------ tests/test_utils_python.py | 7 +++---- 29 files changed, 50 insertions(+), 202 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 8048e1c86..366b95510 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -474,7 +474,7 @@ DBM storage backend A DBM_ storage backend is also available for the HTTP cache middleware. - By default, it uses the anydbm_ module, but you can change it with the + By default, it uses the dbm_ module, but you can change it with the :setting:`HTTPCACHE_DBM_MODULE` setting. .. _httpcache-storage-custom: @@ -626,7 +626,7 @@ HTTPCACHE_DBM_MODULE .. versionadded:: 0.13 -Default: ``'anydbm'`` +Default: ``'dbm'`` The database module to use in the :ref:`DBM storage backend <httpcache-storage-dbm>`. This setting is specific to the DBM backend. @@ -1202,4 +1202,4 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. .. _DBM: https://en.wikipedia.org/wiki/Dbm -.. _anydbm: https://docs.python.org/2/library/anydbm.html +.. _dbm: https://docs.python.org/3/library/dbm.html diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index b68099cad..1f8067b35 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -1,16 +1,6 @@ -import six from six.moves import copyreg -if six.PY2: - from urlparse import urlparse - - # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4 - if urlparse('s3://bucket/key?key=value').query != 'key=value': - from urlparse import uses_query - uses_query.append('s3') - - # Undo what Twisted's perspective broker adds to pickle register # to prevent bugs like Twisted#7989 while serializing requests import twisted.persisted.styles # NOQA diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 7d4840529..d45133e0e 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,5 +1,5 @@ from __future__ import print_function -import sys, six +import sys from w3lib.url import is_url from scrapy.commands import ScrapyCommand @@ -45,8 +45,7 @@ class Command(ScrapyCommand): self._print_bytes(response.body) def _print_bytes(self, bytes_): - bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer - bytes_writer.write(bytes_ + b'\n') + sys.stdout.buffer.write(bytes_ + b'\n') def run(self, args, opts): if len(args) != 1 or not is_url(args[0]): diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ded3c082b..19b998e0d 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -88,20 +88,9 @@ class Crawler(object): yield self.engine.open_spider(self.spider, start_requests) yield defer.maybeDeferred(self.engine.start) except Exception: - # In Python 2 reraising an exception after yield discards - # the original traceback (see https://bugs.python.org/issue7563), - # so sys.exc_info() workaround is used. - # This workaround also works in Python 3, but it is not needed, - # and it is slower, so in Python 3 we use native `raise`. - if six.PY2: - exc_info = sys.exc_info() - self.crawling = False if self.engine is not None: yield self.engine.close() - - if six.PY2: - six.reraise(*exc_info) raise def _create_spider(self, *args, **kwargs): diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 6fc87ed18..5d1f1ad8f 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -216,7 +216,7 @@ class CsvItemExporter(BaseItemExporter): write_through=True, encoding=self.encoding, newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 - ) if six.PY3 else file + ) self.csv_writer = csv.writer(self.stream, **kwargs) self._headers_not_written = True self._join_multivalued = join_multivalued diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 6fb6397b1..07ffd3476 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -10,7 +10,6 @@ import logging import posixpath from tempfile import NamedTemporaryFile from datetime import datetime -import six from six.moves.urllib.parse import urlparse, unquote from ftplib import FTP @@ -65,7 +64,7 @@ class StdoutFeedStorage(object): def __init__(self, uri, _stdout=None): if not _stdout: - _stdout = sys.stdout if six.PY2 else sys.stdout.buffer + _stdout = sys.stdout.buffer self._stdout = _stdout def open(self, spider): diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 3ce8fc48e..b6feede07 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -104,8 +104,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): el = el.getparent() if el is None: break - encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape') - raise ValueError('No <form> element found with %s' % encoded) + raise ValueError('No <form> element found with %s' % formxpath) # If we get here, it means that either formname was None # or invalid diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 339913d4e..a8010877c 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -32,9 +32,6 @@ class TextResponse(Response): def _set_url(self, url): if isinstance(url, six.text_type): - if six.PY2 and self.encoding is None: - raise TypeError("Cannot convert unicode url - %s " - "has no encoding" % type(self).__name__) self._url = to_native_str(url, self.encoding) else: super(TextResponse, self)._set_url(url) diff --git a/scrapy/item.py b/scrapy/item.py index 73b8f54b0..32f9b2ebb 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -4,8 +4,8 @@ Scrapy Item See documentation in docs/topics/item.rst """ -import collections from abc import ABCMeta +from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat from warnings import warn @@ -16,12 +16,6 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref -if six.PY2: - MutableMapping = collections.MutableMapping -else: - MutableMapping = collections.abc.MutableMapping - - class BaseItem(object_ref): """Base class for all scraped items. diff --git a/scrapy/link.py b/scrapy/link.py index 2c8301680..a175b8afd 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,12 +4,6 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ -import warnings -import six - -from scrapy.utils.python import to_bytes - - class Link(object): """Link objects represent an extracted link by the LinkExtractor.""" @@ -17,13 +11,8 @@ class Link(object): def __init__(self, url, text='', fragment='', nofollow=False): if not isinstance(url, str): - if six.PY2: - warnings.warn("Link urls must be str objects. " - "Assuming utf-8 encoding (which could be wrong)") - url = to_bytes(url, encoding='utf8') - else: - got = url.__class__.__name__ - raise TypeError("Link urls must be str objects, got %s" % got) + got = url.__class__.__name__ + raise TypeError("Link urls must be str objects, got %s" % got) self.url = url self.text = text self.fragment = fragment diff --git a/scrapy/mail.py b/scrapy/mail.py index 5b944e1c4..746468e25 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -9,18 +9,13 @@ try: from cStringIO import StringIO as BytesIO except ImportError: from io import BytesIO -import six from email.utils import COMMASPACE, formatdate from six.moves.email_mime_multipart import MIMEMultipart from six.moves.email_mime_text import MIMEText from six.moves.email_mime_base import MIMEBase -if six.PY2: - from email.MIMENonMultipart import MIMENonMultipart - from email import Encoders -else: - from email.mime.nonmultipart import MIMENonMultipart - from email import encoders as Encoders +from email.mime.nonmultipart import MIMENonMultipart +from email import encoders as Encoders from twisted.internet import defer, reactor, ssl diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index f28c7940d..c871e86e0 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,19 +1,13 @@ import six import json import copy -import collections +from collections.abc import MutableMapping from importlib import import_module from pprint import pformat from scrapy.settings import default_settings -if six.PY2: - MutableMapping = collections.MutableMapping -else: - MutableMapping = collections.abc.MutableMapping - - SETTINGS_PRIORITIES = { 'default': 0, 'command': 10, diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9c22999cb..5c9678c01 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -17,8 +17,6 @@ import sys from importlib import import_module from os.path import join, abspath, dirname -import six - AJAXCRAWL_ENABLED = False AUTOTHROTTLE_ENABLED = False @@ -179,7 +177,7 @@ HTTPCACHE_ALWAYS_STORE = False HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = [] -HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm' +HTTPCACHE_DBM_MODULE = 'dbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 421ab2f7e..c8fc911bb 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,7 +1,6 @@ """Boto/botocore helpers""" from __future__ import absolute_import -import six from scrapy.exceptions import NotConfigured @@ -11,11 +10,4 @@ def is_botocore(): import botocore return True except ImportError: - if six.PY2: - try: - import boto - return False - except ImportError: - raise NotConfigured('missing botocore or boto library') - else: - raise NotConfigured('missing botocore library') + raise NotConfigured('missing botocore library') diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index fb7ca3310..561bb72fc 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,13 +1,10 @@ +from configparser import ConfigParser import os import sys import numbers from operator import itemgetter import six -if six.PY2: - from ConfigParser import SafeConfigParser as ConfigParser -else: - from configparser import ConfigParser from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index df2b99c28..6e9de47f3 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -7,6 +7,7 @@ This module must not depend on any module outside the Standard Library. import copy import collections +from collections.abc import Mapping import warnings import six @@ -14,12 +15,6 @@ import six from scrapy.exceptions import ScrapyDeprecationWarning -if six.PY2: - Mapping = collections.Mapping -else: - Mapping = collections.abc.Mapping - - class MultiValueDictKeyError(KeyError): def __init__(self, *args, **kwargs): warnings.warn( @@ -252,13 +247,12 @@ class MergeDict(object): first occurrence will be used. """ def __init__(self, *dicts): - if not six.PY2: - warnings.warn( - "scrapy.utils.datatypes.MergeDict is deprecated in favor " - "of collections.ChainMap (introduced in Python 3.3)", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) + warnings.warn( + "scrapy.utils.datatypes.MergeDict is deprecated in favor " + "of collections.ChainMap (introduced in Python 3.3)", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) self.dicts = dicts def __getitem__(self, key): diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index b3fb16b1e..9984492f0 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -6,7 +6,6 @@ except ImportError: from io import BytesIO from gzip import GzipFile -import six import re from scrapy.utils.decorators import deprecated @@ -17,14 +16,8 @@ from scrapy.utils.decorators import deprecated # (regression or bug-fix compared to Python 3.4) # - read1(), which fetches data before raising EOFError on next call # works here but is only available from Python>=3.3 -# - scrapy does not support Python 3.2 -# - Python 2.7 GzipFile works fine with standard read() + extrabuf -if six.PY2: - def read1(gzf, size=-1): - return gzf.read(size) -else: - def read1(gzf, size=-1): - return gzf.read1(size) +def read1(gzf, size=-1): + return gzf.read1(size) def gunzip(data): @@ -37,7 +30,7 @@ def gunzip(data): chunk = b'.' while chunk: try: - chunk = read1(f, 8196) + chunk = f.read1(8196) output_list.append(chunk) except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index a12e14005..dbc1e0d20 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -102,11 +102,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def row_to_unicode(row_): return [to_unicode(field, encoding) for field in row_] - # Python 3 csv reader input object needs to return strings - if six.PY3: - lines = StringIO(_body_or_str(obj, unicode=True)) - else: - lines = BytesIO(_body_or_str(obj, unicode=False)) + lines = StringIO(_body_or_str(obj, unicode=True)) kwargs = {} if delimiter: kwargs["delimiter"] = delimiter diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index c6140f885..5009aab81 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -113,10 +113,7 @@ def to_bytes(text, encoding=None, errors='strict'): def to_native_str(text, encoding=None, errors='strict'): """ Return str representation of ``text`` (bytes in Python 2.x and unicode in Python 3.x). """ - if six.PY2: - return to_bytes(text, encoding, errors) - else: - return to_unicode(text, encoding, errors) + return to_unicode(text, encoding, errors) def re_rsearch(pattern, text, chunk_size=1024): @@ -189,7 +186,7 @@ def _getargspec_py23(func): """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords, defaults) - Identical to inspect.getargspec() in python2, but uses + Was identical to inspect.getargspec() in python2, but uses inspect.getfullargspec() for python3 behind the scenes to avoid DeprecationWarning. @@ -199,9 +196,6 @@ def _getargspec_py23(func): >>> _getargspec_py23(f) ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,)) """ - if six.PY2: - return inspect.getargspec(func) - return inspect.ArgSpec(*inspect.getfullargspec(func)[:4]) diff --git a/tests/__init__.py b/tests/__init__.py index 9c9e35c35..a54367f8c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -35,18 +35,3 @@ def get_testdata(*paths): path = os.path.join(tests_datadir, *paths) with open(path, 'rb') as f: return f.read() - - -# FIXME: delete after dropping py2 support -# Monkey patch the unittest module to prevent the -# DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex -import six -if six.PY2: - import unittest - import twisted.trial.unittest - if not getattr(unittest.TestCase, 'assertRegex', None): - unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches - if not getattr(unittest.TestCase, 'assertRaisesRegex', None): - unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp - if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None): - twisted.trial.unittest.TestCase.assertRaisesRegex = twisted.trial.unittest.TestCase.assertRaisesRegexp diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 16d7a1cb8..828902b99 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -3,13 +3,12 @@ import cgi import unittest import re import json +from urllib.parse import unquote_to_bytes import warnings import six from six.moves import xmlrpc_client as xmlrpclib from six.moves.urllib.parse import urlparse, parse_qs, unquote -if six.PY3: - from urllib.parse import unquote_to_bytes from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_native_str @@ -1064,8 +1063,7 @@ class FormRequestTest(RequestTest): self.assertEqual(fs, {}) xpath = u"//form[@name='\u03b1']" - encoded = xpath if six.PY3 else xpath.encode('unicode_escape') - self.assertRaisesRegex(ValueError, re.escape(encoded), + self.assertRaisesRegex(ValueError, re.escape(xpath), self.request_class.from_response, response, formxpath=xpath) @@ -1208,10 +1206,7 @@ def _qs(req, encoding='utf-8', to_unicode=False): qs = req.body else: qs = req.url.partition('?')[2] - if six.PY2: - uqs = unquote(to_native_str(qs, encoding)) - elif six.PY3: - uqs = unquote_to_bytes(qs) + uqs = unquote_to_bytes(qs) if to_unicode: uqs = uqs.decode(encoding) return parse_qs(uqs, True) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index cd5c3486e..d6e77d6b8 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -21,8 +21,7 @@ class BaseResponseTest(unittest.TestCase): # Response requires url in the consturctor self.assertRaises(Exception, self.response_class) self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class)) - if not six.PY2: - self.assertRaises(TypeError, self.response_class, b"http://example.com") + self.assertRaises(TypeError, self.response_class, b"http://example.com") # body can be str or None self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class)) self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class)) diff --git a/tests/test_item.py b/tests/test_item.py index 947566686..0ad278701 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -62,12 +62,8 @@ class ItemTest(unittest.TestCase): i['number'] = 123 itemrepr = repr(i) - if six.PY2: - self.assertEqual(itemrepr, - "{'name': u'John Doe', 'number': 123}") - else: - self.assertEqual(itemrepr, - "{'name': 'John Doe', 'number': 123}") + self.assertEqual(itemrepr, + "{'name': 'John Doe', 'number': 123}") i2 = eval(itemrepr) self.assertEqual(i2['name'], 'John Doe') diff --git a/tests/test_link.py b/tests/test_link.py index 955430b37..5e2ce5eeb 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -1,6 +1,4 @@ import unittest -import warnings -import six from scrapy.link import Link @@ -46,12 +44,5 @@ class LinkTest(unittest.TestCase): self._assert_same_links(l1, l2) def test_non_str_url_py2(self): - if six.PY2: - with warnings.catch_warnings(record=True) as w: - link = Link(u"http://www.example.com/\xa3") - self.assertIsInstance(link.url, str) - self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3') - assert len(w) == 1, "warning not issued" - else: - with self.assertRaises(TypeError): - Link(b"http://www.example.com/\xc2\xa3") + with self.assertRaises(TypeError): + Link(b"http://www.example.com/\xc2\xa3") diff --git a/tests/test_middleware.py b/tests/test_middleware.py index aea0be825..af9b43d61 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -3,7 +3,6 @@ from twisted.trial import unittest from scrapy.settings import Settings from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager -import six class M1(object): @@ -66,20 +65,12 @@ class MiddlewareManagerTest(unittest.TestCase): def test_methods(self): mwman = TestMiddlewareManager(M1(), M2(), M3()) - if six.PY2: - self.assertEqual([x.im_class for x in mwman.methods['open_spider']], - [M1, M2]) - self.assertEqual([x.im_class for x in mwman.methods['close_spider']], - [M2, M1]) - self.assertEqual([x.im_class for x in mwman.methods['process']], - [M1, M3]) - else: - self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']], - [M1, M2]) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']], - [M2, M1]) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']], - [M1, M3]) + self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']], + [M1, M2]) + self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']], + [M2, M1]) + self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']], + [M1, M3]) def test_enabled(self): m1, m2, m3 = M1(), M2(), M3() diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index c9943faa8..a5cdc0de0 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -1,7 +1,6 @@ from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase -import six from scrapy.http import Request from scrapy.crawler import CrawlerRunner @@ -161,9 +160,4 @@ class CallbackKeywordArgumentsTestCase(TestCase): self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError) self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'") self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) - # py2 and py3 messages are different - exc_message = str(exceptions['takes_more'].exc_info[1]) - if six.PY2: - self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)") - elif six.PY3: - self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'") + self.assertEqual(str(exceptions['takes_more'].exc_info[1]), "parse_takes_more() missing 1 required positional argument: 'other'") diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 1dbacbea3..08286ff02 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -60,9 +60,6 @@ class SettingsAttributeTest(unittest.TestCase): class BaseSettingsTest(unittest.TestCase): - if six.PY3: - assertItemsEqual = unittest.TestCase.assertCountEqual - def setUp(self): self.settings = BaseSettings() @@ -152,7 +149,7 @@ class BaseSettingsTest(unittest.TestCase): self.settings.setmodule( 'tests.test_settings.default_settings', 10) - self.assertItemsEqual(six.iterkeys(self.settings.attributes), + self.assertCountEqual(six.iterkeys(self.settings.attributes), six.iterkeys(ctrl_attributes)) for key in six.iterkeys(ctrl_attributes): @@ -343,9 +340,6 @@ class BaseSettingsTest(unittest.TestCase): class SettingsTest(unittest.TestCase): - if six.PY3: - assertItemsEqual = unittest.TestCase.assertCountEqual - def setUp(self): self.settings = Settings() diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 535095b8d..47877f555 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,12 +1,7 @@ +from collections.abc import Mapping, MutableMapping import copy import unittest -import six -if six.PY2: - from collections import Mapping, MutableMapping -else: - from collections.abc import Mapping, MutableMapping - from scrapy.utils.datatypes import CaselessDict, SequenceExclude diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3e1148354..096aa50b7 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -231,12 +231,11 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), []) else: - stripself = not six.PY2 # PyPy3 exposes them as methods self.assertEqual( - get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit']) - self.assertEqual(get_func_args(" ".join, stripself), ['list']) + get_func_args(six.text_type.split, True), ['sep', 'maxsplit']) + self.assertEqual(get_func_args(" ".join, True), ['list']) self.assertEqual( - get_func_args(operator.itemgetter(2), stripself), ['obj']) + get_func_args(operator.itemgetter(2), True), ['obj']) def test_without_none_values(self): From de7789e52df5d8dc8595570d36a60ae92b4fcb50 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 20 Aug 2019 20:56:26 +0500 Subject: [PATCH 09/85] Remove unneeded and unused code from XmlItemExporter. --- scrapy/exporters.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 5d1f1ad8f..40567f53b 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -143,11 +143,11 @@ class XmlItemExporter(BaseItemExporter): def _beautify_newline(self, new_item=False): if self.indent is not None and (self.indent > 0 or new_item): - self._xg_characters('\n') + self.xg.characters('\n') def _beautify_indent(self, depth=1): if self.indent: - self._xg_characters(' ' * self.indent * depth) + self.xg.characters(' ' * self.indent * depth) def start_exporting(self): self.xg.startDocument() @@ -182,26 +182,12 @@ class XmlItemExporter(BaseItemExporter): self._export_xml_field('value', value, depth=depth+1) self._beautify_indent(depth=depth) elif isinstance(serialized_value, six.text_type): - self._xg_characters(serialized_value) + self.xg.characters(serialized_value) else: - self._xg_characters(str(serialized_value)) + self.xg.characters(str(serialized_value)) self.xg.endElement(name) self._beautify_newline() - # Workaround for https://bugs.python.org/issue17606 - # Before Python 2.7.4 xml.sax.saxutils required bytes; - # since 2.7.4 it requires unicode. The bug is likely to be - # fixed in 2.7.6, but 2.7.6 will still support unicode, - # and Python 3.x will require unicode, so ">= 2.7.4" should be fine. - if sys.version_info[:3] >= (2, 7, 4): - def _xg_characters(self, serialized_value): - if not isinstance(serialized_value, six.text_type): - serialized_value = serialized_value.decode(self.encoding) - return self.xg.characters(serialized_value) - else: # pragma: no cover - def _xg_characters(self, serialized_value): - return self.xg.characters(serialized_value) - class CsvItemExporter(BaseItemExporter): From c2898fdcf91cbbfa9d801196924960d3a3764b93 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 20 Aug 2019 21:06:52 +0500 Subject: [PATCH 10/85] Deprecate scrapy.utils.gz.read1. --- scrapy/utils/gz.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 9984492f0..dc8316d8c 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -16,6 +16,7 @@ from scrapy.utils.decorators import deprecated # (regression or bug-fix compared to Python 3.4) # - read1(), which fetches data before raising EOFError on next call # works here but is only available from Python>=3.3 +@deprecated('GzipFile.read1') def read1(gzf, size=-1): return gzf.read1(size) From cea2f5e244f30591bae667abf2b2689e566ab343 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 20 Aug 2019 21:09:49 +0500 Subject: [PATCH 11/85] Remove cStringIO imports. --- scrapy/downloadermiddlewares/decompression.py | 6 +----- scrapy/mail.py | 6 +----- scrapy/pipelines/files.py | 7 +------ scrapy/pipelines/images.py | 6 +----- scrapy/utils/gz.py | 9 ++------- scrapy/utils/iterators.py | 6 +----- tests/test_cmdline/__init__.py | 5 +---- tests/test_pipeline_media.py | 6 ++---- 8 files changed, 10 insertions(+), 41 deletions(-) diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index 49313cc04..e2d73f347 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -4,6 +4,7 @@ and extract the potentially compressed responses that may arrive. import bz2 import gzip +from io import BytesIO import zipfile import tarfile import logging @@ -11,11 +12,6 @@ from tempfile import mktemp import six -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO - from scrapy.responsetypes import responsetypes logger = logging.getLogger(__name__) diff --git a/scrapy/mail.py b/scrapy/mail.py index 746468e25..d24de2212 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -3,13 +3,9 @@ Mail sending helpers See documentation in docs/topics/email.rst """ +from io import BytesIO import logging -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO - from email.utils import COMMASPACE, formatdate from six.moves.email_mime_multipart import MIMEMultipart from six.moves.email_mime_text import MIMEText diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index cc3d10b63..8d74c5011 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -5,6 +5,7 @@ See documentation in topics/media-pipeline.rst """ import functools import hashlib +from io import BytesIO import mimetypes import os import os.path @@ -15,12 +16,6 @@ from six.moves.urllib.parse import urlparse from collections import defaultdict import six - -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO - from twisted.internet import defer, threads from scrapy.pipelines.media import MediaPipeline diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index fa4d12ad1..e77cef4ff 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -5,13 +5,9 @@ See documentation in topics/media-pipeline.rst """ import functools import hashlib +from io import BytesIO import six -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO - from PIL import Image from scrapy.utils.misc import md5sum diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index dc8316d8c..f41e62fe3 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,12 +1,7 @@ -import struct - -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO from gzip import GzipFile - +from io import BytesIO import re +import struct from scrapy.utils.decorators import deprecated diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index dbc1e0d20..9693ba768 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,11 +1,7 @@ import re import csv -import logging -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO from io import StringIO +import logging import six from scrapy.http import TextResponse, Response diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 68dfb1cca..56cfe642a 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -1,3 +1,4 @@ +from io import StringIO import json import os import pstats @@ -7,10 +8,6 @@ from subprocess import Popen, PIPE import sys import tempfile import unittest -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO from scrapy.utils.test import get_testenv diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 28e39cefa..ad2618ec9 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -144,10 +144,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase): # The Failure should encapsulate a FileException ... self.assertEqual(failure.value, file_exc) - # ... and if we're running on Python 3 ... - if sys.version_info.major >= 3: - # ... it should have the returnValue exception set as its context - self.assertEqual(failure.value.__context__, def_gen_return_exc) + # ... and it should have the returnValue exception set as its context + self.assertEqual(failure.value.__context__, def_gen_return_exc) # Let's calculate the request fingerprint and fake some runtime data... fp = request_fingerprint(request) From 5b70b051a6cec699c3cb14c3ac9cf530671a8487 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 20 Aug 2019 21:13:01 +0500 Subject: [PATCH 12/85] Some text function messages cleanup, deprecate to_native_str. --- scrapy/http/response/__init__.py | 2 +- scrapy/utils/python.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index b0a526b72..a81404afb 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -88,7 +88,7 @@ class Response(object_ref): @property def text(self): """For subclasses of TextResponse, this will return the body - as text (unicode object in Python 2 and str in Python 3) + as str """ raise AttributeError("Response content isn't text") diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 5009aab81..974abaeb1 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -90,7 +90,7 @@ def to_unicode(text, encoding=None, errors='strict'): if isinstance(text, six.text_type): return text if not isinstance(text, (bytes, six.text_type)): - raise TypeError('to_unicode must receive a bytes, str or unicode ' + raise TypeError('to_unicode must receive a bytes or str ' 'object, got %s' % type(text).__name__) if encoding is None: encoding = 'utf-8' @@ -103,16 +103,16 @@ def to_bytes(text, encoding=None, errors='strict'): if isinstance(text, bytes): return text if not isinstance(text, six.string_types): - raise TypeError('to_bytes must receive a unicode, str or bytes ' + raise TypeError('to_bytes must receive a str or bytes ' 'object, got %s' % type(text).__name__) if encoding is None: encoding = 'utf-8' return text.encode(encoding, errors) +@deprecated('to_unicode') def to_native_str(text, encoding=None, errors='strict'): - """ Return str representation of ``text`` - (bytes in Python 2.x and unicode in Python 3.x). """ + """ Return str representation of ``text``. """ return to_unicode(text, encoding, errors) From 3ac4b430ae0165f25d8cd2fb1ac25ac9f307df8b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 31 Oct 2019 15:20:28 +0500 Subject: [PATCH 13/85] Remove an unused six import. --- tests/test_downloader_handlers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 9f74577a5..e6856945c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,5 +1,4 @@ import os -import six import shutil import tempfile import contextlib From 75b1d051d99cda17558a210326e77046eec851a7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 20 Aug 2019 21:27:52 +0500 Subject: [PATCH 14/85] Simplify some more imports. --- scrapy/downloadermiddlewares/httpproxy.py | 5 +---- scrapy/loader/processors.py | 5 +---- tests/__init__.py | 5 ----- tests/test_downloader_handlers.py | 5 +---- tests/test_downloadermiddleware.py | 3 ++- tests/test_downloadermiddleware_robotstxt.py | 4 +++- tests/test_extension_telnet.py | 5 ----- tests/test_feedexport.py | 2 +- tests/test_http_request.py | 3 +-- tests/test_item.py | 2 +- tests/test_pipeline_files.py | 6 +----- tests/test_settings/__init__.py | 3 +-- tests/test_spider.py | 3 +-- tests/test_spidermiddleware.py | 3 ++- tests/test_stats.py | 6 +----- tests/test_utils_deprecate.py | 3 +-- tests/test_utils_misc/__init__.py | 2 +- tests/test_utils_trackref.py | 2 +- 18 files changed, 20 insertions(+), 47 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 2c35d1b90..2212d9688 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,10 +1,7 @@ import base64 from six.moves.urllib.parse import unquote, urlunparse from six.moves.urllib.request import getproxies, proxy_bypass -try: - from urllib2 import _parse_proxy -except ImportError: - from urllib.request import _parse_proxy +from urllib.request import _parse_proxy from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py index 2acdc8093..02c625acc 100644 --- a/scrapy/loader/processors.py +++ b/scrapy/loader/processors.py @@ -3,10 +3,7 @@ This module provides some commonly used processors for Item Loaders. See documentation in docs/topics/loaders.rst """ -try: - from collections import ChainMap -except ImportError: - from scrapy.utils.datatypes import MergeDict as ChainMap +from collections import ChainMap from scrapy.utils.misc import arg_to_iter from scrapy.loader.common import wrap_loader_context diff --git a/tests/__init__.py b/tests/__init__.py index a54367f8c..12ce79fa9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -21,11 +21,6 @@ if 'COV_CORE_CONFIG' in os.environ: os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot, os.environ['COV_CORE_CONFIG']) -try: - import unittest.mock as mock -except ImportError: - import mock - tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data') diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 109469503..6090998d4 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -2,11 +2,8 @@ import os import six import shutil import tempfile +from unittest import mock import contextlib -try: - from unittest import mock -except ImportError: - import mock from testfixtures import LogCapture from twisted.trial import unittest diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 03564e748..6b9a5bee8 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,3 +1,5 @@ +from unittest import mock + from twisted.trial.unittest import TestCase from twisted.python.failure import Failure @@ -7,7 +9,6 @@ from scrapy.exceptions import _InvalidOutput from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.utils.test import get_crawler from scrapy.utils.python import to_bytes -from tests import mock class ManagerTestCase(TestCase): diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index fbc46cba4..8266bf35f 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import + +from unittest import mock + from twisted.internet import reactor, error from twisted.internet.defer import Deferred, DeferredList, maybeDeferred from twisted.python import failure @@ -9,7 +12,6 @@ from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware, from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.settings import Settings -from tests import mock from tests.test_robotstxt_interface import rerp_available, reppy_available diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 4f389e5cb..875ceb83c 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -1,8 +1,3 @@ -try: - import unittest.mock as mock -except ImportError: - import mock - from twisted.trial import unittest from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e1436fbe5..7431f921f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -7,6 +7,7 @@ from io import BytesIO import tempfile import shutil import string +from unittest import mock from six.moves.urllib.parse import urljoin, urlparse, quote from six.moves.urllib.request import pathname2url @@ -15,7 +16,6 @@ from twisted.trial import unittest from twisted.internet import defer from scrapy.crawler import CrawlerRunner from scrapy.settings import Settings -from tests import mock from tests.mockserver import MockServer from w3lib.url import path_to_file_uri diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 828902b99..effb9e53b 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -3,6 +3,7 @@ import cgi import unittest import re import json +from unittest import mock from urllib.parse import unquote_to_bytes import warnings @@ -13,8 +14,6 @@ from six.moves.urllib.parse import urlparse, parse_qs, unquote from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_native_str -from tests import mock - class RequestTest(unittest.TestCase): diff --git a/tests/test_item.py b/tests/test_item.py index 0ad278701..d98c63ddd 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,12 +1,12 @@ import sys import unittest +from unittest import mock from warnings import catch_warnings import six from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta -from tests import mock PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index cb8f8da18..bd40e4103 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,10 +1,9 @@ import os import random import time -import hashlib -import warnings from tempfile import mkdtemp from shutil import rmtree +from unittest import mock from six.moves.urllib.parse import urlparse from six import BytesIO @@ -15,13 +14,10 @@ from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GC from scrapy.item import Item, Field from scrapy.http import Request, Response from scrapy.settings import Settings -from scrapy.utils.python import to_bytes from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete from scrapy.utils.boto import is_botocore -from tests import mock - def _mocked_download_func(request, info): response = request.meta.get('response') diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 08286ff02..32e65bed5 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,10 +1,9 @@ import six import unittest -import warnings +from unittest import mock from scrapy.settings import (BaseSettings, Settings, SettingsAttribute, SETTINGS_PRIORITIES, get_settings_priority) -from tests import mock from . import default_settings diff --git a/tests/test_spider.py b/tests/test_spider.py index 2220b8ffc..b913a56b7 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,5 +1,6 @@ import gzip import inspect +from unittest import mock import warnings from io import BytesIO @@ -17,8 +18,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref from scrapy.utils.test import get_crawler -from tests import mock - class SpiderTest(unittest.TestCase): diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 832fd3330..55d665e79 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,3 +1,5 @@ +from unittest import mock + from twisted.trial.unittest import TestCase from twisted.python.failure import Failure @@ -6,7 +8,6 @@ from scrapy.http import Request, Response from scrapy.exceptions import _InvalidOutput from scrapy.utils.test import get_crawler from scrapy.core.spidermw import SpiderMiddlewareManager -from tests import mock class SpiderMiddlewareTestCase(TestCase): diff --git a/tests/test_stats.py b/tests/test_stats.py index 2033dbe07..2bbbb9e2c 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -1,10 +1,6 @@ from datetime import datetime import unittest - -try: - from unittest import mock -except ImportError: - import mock +from unittest import mock from scrapy.extensions.corestats import CoreStats from scrapy.spiders import Spider diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 3e7236fb1..ce04e7f29 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -2,12 +2,11 @@ from __future__ import absolute_import import inspect import unittest +from unittest import mock import warnings from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import create_deprecated_class, update_classpath -from tests import mock - class MyWarning(UserWarning): pass diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index e109d5343..de9da9104 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -1,11 +1,11 @@ import sys import os import unittest +from unittest import mock from scrapy.item import Item, Field from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules -from tests import mock __doctests__ = ['scrapy.utils.misc'] diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index c6072fc0d..480a717e7 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,7 +1,7 @@ import six import unittest +from unittest import mock from scrapy.utils import trackref -from tests import mock class Foo(trackref.object_ref): From 397e8835564614608647c855c62c36232020f078 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 20 Aug 2019 21:35:13 +0500 Subject: [PATCH 15/85] Replace to_native_str calls with to_unicode. --- scrapy/core/downloader/handlers/http11.py | 2 +- scrapy/downloadermiddlewares/cookies.py | 6 +++--- scrapy/downloadermiddlewares/robotstxt.py | 3 --- scrapy/exporters.py | 4 ++-- scrapy/http/cookies.py | 10 +++++----- scrapy/http/response/text.py | 8 ++++---- scrapy/linkextractors/lxmlhtml.py | 4 ++-- scrapy/responsetypes.py | 6 +++--- scrapy/robotstxt.py | 8 ++++---- scrapy/spidermiddlewares/referer.py | 5 ++--- scrapy/utils/reqser.py | 4 ++-- scrapy/utils/request.py | 4 ++-- scrapy/utils/response.py | 4 ++-- scrapy/utils/ssl.py | 4 ++-- tests/test_command_parse.py | 5 ++--- tests/test_commands.py | 7 ++----- tests/test_feedexport.py | 7 +++---- tests/test_http_request.py | 7 +++---- tests/test_http_response.py | 6 +++--- tests/test_robotstxt_interface.py | 1 - 20 files changed, 47 insertions(+), 58 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 91b45a8fc..7d917cb74 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -174,7 +174,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None): r""" Return binary content of a CONNECT request. - >>> from scrapy.utils.python import to_native_str as s + >>> from scrapy.utils.python import to_unicode as s >>> s(tunnel_request_data("example.com", 8080)) 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n' >>> s(tunnel_request_data("example.com", 8080, b"123")) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 321c0171b..0d2b9900c 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -6,7 +6,7 @@ from collections import defaultdict from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode logger = logging.getLogger(__name__) @@ -53,7 +53,7 @@ class CookiesMiddleware(object): def _debug_cookie(self, request, spider): if self.debug: - cl = [to_native_str(c, errors='replace') + cl = [to_unicode(c, errors='replace') for c in request.headers.getlist('Cookie')] if cl: cookies = "\n".join("Cookie: {}\n".format(c) for c in cl) @@ -62,7 +62,7 @@ class CookiesMiddleware(object): def _debug_set_cookie(self, response, spider): if self.debug: - cl = [to_native_str(c, errors='replace') + cl = [to_unicode(c, errors='replace') for c in response.headers.getlist('Set-Cookie')] if cl: cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 6a5dfb79c..251706c50 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -5,15 +5,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. """ import logging -import sys -import re from twisted.internet.defer import Deferred, maybeDeferred from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.python import to_native_str from scrapy.utils.misc import load_object logger = logging.getLogger(__name__) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 40567f53b..f276c28e8 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -12,7 +12,7 @@ from six.moves import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder -from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike +from scrapy.utils.python import to_bytes, to_unicode, is_listlike from scrapy.item import BaseItem from scrapy.exceptions import ScrapyDeprecationWarning import warnings @@ -232,7 +232,7 @@ class CsvItemExporter(BaseItemExporter): def _build_row(self, values): for s in values: try: - yield to_native_str(s, self.encoding) + yield to_unicode(s, self.encoding) except TypeError: yield s diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 4e8056750..4532c3ab7 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -3,7 +3,7 @@ from six.moves.http_cookiejar import ( CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE ) from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode class CookieJar(object): @@ -165,13 +165,13 @@ class WrappedRequest(object): return name in self.request.headers def get_header(self, name, default=None): - return to_native_str(self.request.headers.get(name, default), + return to_unicode(self.request.headers.get(name, default), errors='replace') def header_items(self): return [ - (to_native_str(k, errors='replace'), - [to_native_str(x, errors='replace') for x in v]) + (to_unicode(k, errors='replace'), + [to_unicode(x, errors='replace') for x in v]) for k, v in self.request.headers.items() ] @@ -189,7 +189,7 @@ class WrappedResponse(object): # python3 cookiejars calls get_all def get_all(self, name, default=None): - return [to_native_str(v, errors='replace') + return [to_unicode(v, errors='replace') for v in self.response.headers.getlist(name)] # python2 cookiejars calls getheaders getheaders = get_all diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index a8010877c..37f450e54 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -16,7 +16,7 @@ from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request from scrapy.http.response import Response from scrapy.utils.response import get_base_url -from scrapy.utils.python import memoizemethod_noargs, to_native_str +from scrapy.utils.python import memoizemethod_noargs, to_unicode class TextResponse(Response): @@ -32,7 +32,7 @@ class TextResponse(Response): def _set_url(self, url): if isinstance(url, six.text_type): - self._url = to_native_str(url, self.encoding) + self._url = to_unicode(url, self.encoding) else: super(TextResponse, self)._set_url(url) @@ -81,11 +81,11 @@ class TextResponse(Response): @memoizemethod_noargs def _headers_encoding(self): content_type = self.headers.get(b'Content-Type', b'') - return http_content_type_encoding(to_native_str(content_type)) + return http_content_type_encoding(to_unicode(content_type)) def _body_inferred_encoding(self): if self._cached_benc is None: - content_type = to_native_str(self.headers.get(b'Content-Type', b'')) + content_type = to_unicode(self.headers.get(b'Content-Type', b'')) benc, ubody = html_to_unicode(content_type, self.body, auto_detect_fun=self._auto_detect_fun, default_encoding=self._DEFAULT_ENCODING) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 8f6f93a44..890c019c8 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -10,7 +10,7 @@ from w3lib.url import canonicalize_url from scrapy.link import Link from scrapy.utils.misc import arg_to_iter, rel_has_nofollow -from scrapy.utils.python import unique as unique_list, to_native_str +from scrapy.utils.python import unique as unique_list, to_unicode from scrapy.utils.response import get_base_url from scrapy.linkextractors import FilteringLinkExtractor @@ -67,7 +67,7 @@ class LxmlParserLinkExtractor(object): url = self.process_attr(attr_val) if url is None: continue - url = to_native_str(url, encoding=response_encoding) + url = to_unicode(url, encoding=response_encoding) # to fix relative links after process_value url = urljoin(response_url, url) link = Link(url, _collect_string_content(el) or u'', diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 4a2d5bf52..de62276c8 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -10,7 +10,7 @@ import six from scrapy.http import Response from scrapy.utils.misc import load_object -from scrapy.utils.python import binary_is_text, to_bytes, to_native_str +from scrapy.utils.python import binary_is_text, to_bytes, to_unicode class ResponseTypes(object): @@ -55,12 +55,12 @@ class ResponseTypes(object): header """ if content_encoding: return Response - mimetype = to_native_str(content_type).split(';')[0].strip().lower() + mimetype = to_unicode(content_type).split(';')[0].strip().lower() return self.from_mimetype(mimetype) def from_content_disposition(self, content_disposition): try: - filename = to_native_str(content_disposition, + filename = to_unicode(content_disposition, encoding='latin-1', errors='replace').split(';')[1].split('=')[1] filename = filename.strip('"\'') return self.from_filename(filename) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 189f165d1..95a8c09b8 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -3,14 +3,14 @@ import logging from abc import ABCMeta, abstractmethod from six import with_metaclass -from scrapy.utils.python import to_native_str, to_unicode +from scrapy.utils.python import to_unicode logger = logging.getLogger(__name__) def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): try: if to_native_str_type: - robotstxt_body = to_native_str(robotstxt_body) + robotstxt_body = to_unicode(robotstxt_body) else: robotstxt_body = robotstxt_body.decode('utf-8') except UnicodeDecodeError: @@ -66,8 +66,8 @@ class PythonRobotParser(RobotParser): return o def allowed(self, url, user_agent): - user_agent = to_native_str(user_agent) - url = to_native_str(url) + user_agent = to_unicode(user_agent) + url = to_unicode(url) return self.rp.can_fetch(user_agent, url) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 1ddfb37f4..c76e4d5a2 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -10,8 +10,7 @@ from w3lib.url import safe_url_string from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals -from scrapy.utils.python import to_native_str -from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object from scrapy.utils.url import strip_url @@ -322,7 +321,7 @@ class RefererMiddleware(object): if isinstance(resp_or_url, Response): policy_header = resp_or_url.headers.get('Referrer-Policy') if policy_header is not None: - policy_name = to_native_str(policy_header.decode('latin1')) + policy_name = to_unicode(policy_header.decode('latin1')) if policy_name is None: return self.default_policy() diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index c7ea7b425..495564ac0 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -4,7 +4,7 @@ Helper functions for serializing (and deserializing) requests. import six from scrapy.http import Request -from scrapy.utils.python import to_unicode, to_native_str +from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object @@ -54,7 +54,7 @@ def request_from_dict(d, spider=None): eb = _get_method(spider, eb) request_cls = load_object(d['_class']) if '_class' in d else Request return request_cls( - url=to_native_str(d['url']), + url=to_unicode(d['url']), callback=cb, errback=eb, method=d['method'], diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index fb5af66a2..63d0ae772 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -9,7 +9,7 @@ import weakref from six.moves.urllib.parse import urlunparse from w3lib.http import basic_auth_header -from scrapy.utils.python import to_bytes, to_native_str +from scrapy.utils.python import to_bytes, to_unicode from w3lib.url import canonicalize_url from scrapy.utils.httpobj import urlparse_cached @@ -97,4 +97,4 @@ def referer_str(request): referrer = request.headers.get('Referer') if referrer is None: return referrer - return to_native_str(referrer, errors='replace') + return to_unicode(referrer, errors='replace') diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c3236afd4..feab07431 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -8,7 +8,7 @@ import webbrowser import tempfile from twisted.web import http -from scrapy.utils.python import to_bytes, to_native_str +from scrapy.utils.python import to_bytes, to_unicode from w3lib import html @@ -36,7 +36,7 @@ def response_status_message(status): """Return status code plus status text descriptive message """ message = http.RESPONSES.get(int(status), "Unknown Status") - return '%s %s' % (status, to_native_str(message)) + return '%s %s' % (status, to_unicode(message)) def response_httprepr(response): diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 02aed60ee..6e81b33ff 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -3,7 +3,7 @@ import OpenSSL import OpenSSL._util as pyOpenSSLutil -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode # The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL. @@ -12,7 +12,7 @@ SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0) def ffi_buf_to_string(buf): - return to_native_str(pyOpenSSLutil.ffi.string(buf)) + return to_unicode(pyOpenSSLutil.ffi.string(buf)) def x509name_to_string(x509name): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 62d5d76b4..b134beb88 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,17 +1,16 @@ import os from os.path import join, abspath -from twisted.trial import unittest from twisted.internet import defer from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode from tests.test_commands import CommandTest def _textmode(bstr): """Normalize input the same as writing to a file and reading from it in text mode""" - return to_native_str(bstr).replace(os.linesep, '\n') + return to_unicode(bstr).replace(os.linesep, '\n') class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' diff --git a/tests/test_commands.py b/tests/test_commands.py index b8445ae6c..536379170 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -10,13 +10,10 @@ from contextlib import contextmanager from threading import Timer from twisted.trial import unittest -from twisted.internet import defer import scrapy -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv -from scrapy.utils.testsite import SiteTest -from scrapy.utils.testproc import ProcessTest from tests.test_crawler import ExceptionSpider, NoRequestsSpider @@ -56,7 +53,7 @@ class ProjectTest(unittest.TestCase): finally: timer.cancel() - return p, to_native_str(stdout), to_native_str(stderr) + return p, to_unicode(stdout), to_unicode(stderr) class StartprojectTest(ProjectTest): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 7431f921f..abe2ab557 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -26,8 +26,7 @@ from scrapy.extensions.feedexport import ( S3FeedStorage, StdoutFeedStorage, BlockingFeedStorage) from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler -from scrapy.utils.python import to_native_str -from scrapy.utils.project import get_project_settings +from scrapy.utils.python import to_unicode class FileFeedStorageTest(unittest.TestCase): @@ -456,7 +455,7 @@ class FeedExportTest(unittest.TestCase): settings.update({'FEED_FORMAT': 'csv'}) data = yield self.exported_data(items, settings) - reader = csv.DictReader(to_native_str(data).splitlines()) + reader = csv.DictReader(to_unicode(data).splitlines()) got_rows = list(reader) if ordered: self.assertEqual(reader.fieldnames, header) @@ -470,7 +469,7 @@ class FeedExportTest(unittest.TestCase): settings = settings or {} settings.update({'FEED_FORMAT': 'jl'}) data = yield self.exported_data(items, settings) - parsed = [json.loads(to_native_str(line)) for line in data.splitlines()] + parsed = [json.loads(to_unicode(line)) for line in data.splitlines()] rows = [{k: v for k, v in row.items() if v} for row in rows] self.assertEqual(rows, parsed) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index effb9e53b..807265981 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -7,12 +7,11 @@ from unittest import mock from urllib.parse import unquote_to_bytes import warnings -import six from six.moves import xmlrpc_client as xmlrpclib from six.moves.urllib.parse import urlparse, parse_qs, unquote from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse -from scrapy.utils.python import to_bytes, to_native_str +from scrapy.utils.python import to_bytes, to_unicode class RequestTest(unittest.TestCase): @@ -349,8 +348,8 @@ class FormRequestTest(RequestTest): request_class = FormRequest def assertQueryEqual(self, first, second, msg=None): - first = to_native_str(first).split("&") - second = to_native_str(second).split("&") + first = to_unicode(first).split("&") + second = to_unicode(second).split("&") return self.assertEqual(sorted(first), sorted(second), msg) def test_empty_formdata(self): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index d6e77d6b8..80bf51647 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -7,7 +7,7 @@ from w3lib.encoding import resolve_encoding from scrapy.http import (Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers) from scrapy.selector import Selector -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode from scrapy.exceptions import NotSupported from scrapy.link import Link from tests import get_testdata @@ -204,11 +204,11 @@ class TextResponseTest(BaseResponseTest): assert isinstance(resp.url, str) resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8') - self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3')) + self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1') self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}) - self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3')) + self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}) self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 9aaab560a..cd7480e33 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,6 +1,5 @@ # coding=utf-8 from twisted.trial import unittest -from scrapy.utils.python import to_native_str def reppy_available(): From 7299e91b1f1b231095e2a7ddbce5ad980ae7da8a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 28 Aug 2019 16:29:53 +0500 Subject: [PATCH 16/85] Remove Py2-only code that checks sys.version_info. --- tests/test_utils_reqser.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 11ac56897..92cd16de7 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -80,8 +80,6 @@ class RequestSerializationTest(unittest.TestCase): self._assert_serializes_ok(r, spider=self.spider) def test_mixin_private_callback_serialization(self): - if sys.version_info[0] < 3: - return r = Request("http://www.example.com", callback=self.spider._TestSpiderMixin__mixin_callback, errback=self.spider.handle_error) @@ -119,9 +117,8 @@ class RequestSerializationTest(unittest.TestCase): def test_private_name_mangling(self): self._assert_mangles_to( self.spider, '_TestSpider__parse_item_private') - if sys.version_info[0] >= 3: - self._assert_mangles_to( - self.spider, '_TestSpiderMixin__mixin_callback') + self._assert_mangles_to( + self.spider, '_TestSpiderMixin__mixin_callback') def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) From 864123132a99dda61feb407a59367c6c06133517 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 31 Oct 2019 22:55:58 +0500 Subject: [PATCH 17/85] Fix a duplicate ref name in docs. --- docs/topics/downloader-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 366b95510..e93645077 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -474,7 +474,7 @@ DBM storage backend A DBM_ storage backend is also available for the HTTP cache middleware. - By default, it uses the dbm_ module, but you can change it with the + By default, it uses the `dbm module`_, but you can change it with the :setting:`HTTPCACHE_DBM_MODULE` setting. .. _httpcache-storage-custom: @@ -1202,4 +1202,4 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. .. _DBM: https://en.wikipedia.org/wiki/Dbm -.. _dbm: https://docs.python.org/3/library/dbm.html +.. _dbm module: https://docs.python.org/3/library/dbm.html From a5eb59b92d3311f702933753d85b15e6d698faa1 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 31 Oct 2019 23:21:14 +0500 Subject: [PATCH 18/85] Fix test_proxy_connect.py for py3.5. --- tests/test_proxy_connect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 5e9470e39..f6381b5b1 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -117,7 +117,7 @@ class ProxyConnectTestCase(TestCase): with LogCapture() as l: yield crawler.crawl(seed=request) self._assert_got_response_code(200, l) - echo = json.loads(crawler.spider.meta['responses'][0].body) + echo = json.loads(crawler.spider.meta['responses'][0].body.decode('utf-8')) self.assertTrue('Proxy-Authorization' not in echo['headers']) @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') From 5eb01b617d207aabc705038d8a0e3b46e358bdbb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Thu, 31 Oct 2019 23:21:30 +0500 Subject: [PATCH 19/85] Use an older mitmproxy for py3.5. --- tests/requirements-py3.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index f27e45a54..c2b16bec6 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,6 +1,7 @@ # Tests requirements jmespath -mitmproxy +mitmproxy; python_version >= '3.6' +mitmproxy==3.0.4; python_version < '3.6' pytest pytest-cov pytest-twisted From e0c5c724969ebc35970daaf0203969dc2eb00d56 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 1 Nov 2019 19:46:19 +0500 Subject: [PATCH 20/85] Improve the test_https_tunnel_without_leak_proxy_authorization_header change. --- tests/test_proxy_connect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index f6381b5b1..651576c2c 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -117,7 +117,7 @@ class ProxyConnectTestCase(TestCase): with LogCapture() as l: yield crawler.crawl(seed=request) self._assert_got_response_code(200, l) - echo = json.loads(crawler.spider.meta['responses'][0].body.decode('utf-8')) + echo = json.loads(crawler.spider.meta['responses'][0].text) self.assertTrue('Proxy-Authorization' not in echo['headers']) @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') From 3c9963ab049e49a1d490d9505ad22ae9c1415421 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 1 Nov 2019 19:46:38 +0500 Subject: [PATCH 21/85] Only xfail test_https_connect_tunnel_error on 3.6+. --- tests/test_proxy_connect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 651576c2c..ec3f0716c 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -92,7 +92,7 @@ class ProxyConnectTestCase(TestCase): yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) - @pytest.mark.xfail(reason='Python 3 fails this earlier') + @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6) @defer.inlineCallbacks def test_https_connect_tunnel_error(self): crawler = get_crawler(SimpleSpider) From 4b0cdf7f3ed94e81612cceacfab6d81d54117865 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 1 Nov 2019 19:50:56 +0500 Subject: [PATCH 22/85] Use self.proc.communicate() after killing mitmdump. --- tests/test_proxy_connect.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index ec3f0716c..69925f80c 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -48,8 +48,7 @@ sys.exit(mitmdump()) def stop(self): self.proc.kill() - self.proc.wait() - time.sleep(0.2) + self.proc.communicate() def _wrong_credentials(proxy_url): From 350aa67c3dc8997ef9d3aac9ef3f596c83758e1c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 1 Nov 2019 19:52:57 +0500 Subject: [PATCH 23/85] Rename tests/py3-ignores.txt to tests/ignores.txt. --- conftest.py | 2 +- tests/{py3-ignores.txt => ignores.txt} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename tests/{py3-ignores.txt => ignores.txt} (100%) diff --git a/conftest.py b/conftest.py index ede091e9f..7da4c4976 100644 --- a/conftest.py +++ b/conftest.py @@ -7,7 +7,7 @@ collect_ignore = [ ] -for line in open('tests/py3-ignores.txt'): +for line in open('tests/ignores.txt'): file_path = line.strip() if file_path and file_path[0] != '#': collect_ignore.append(file_path) diff --git a/tests/py3-ignores.txt b/tests/ignores.txt similarity index 100% rename from tests/py3-ignores.txt rename to tests/ignores.txt From 48b8ac60099c3751ad1f595a9a629be26ffb34cd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Fri, 1 Nov 2019 20:05:37 +0500 Subject: [PATCH 24/85] Improve the dbm module ref. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves <adrian@chaves.io> --- docs/topics/downloader-middleware.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index e93645077..ae6d41809 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -474,7 +474,7 @@ DBM storage backend A DBM_ storage backend is also available for the HTTP cache middleware. - By default, it uses the `dbm module`_, but you can change it with the + By default, it uses the :mod:`dbm`, but you can change it with the :setting:`HTTPCACHE_DBM_MODULE` setting. .. _httpcache-storage-custom: @@ -1202,4 +1202,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. .. _DBM: https://en.wikipedia.org/wiki/Dbm -.. _dbm module: https://docs.python.org/3/library/dbm.html From 415526d922b6c70bfd5872c8d67d69fcce0ee1fb Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sat, 2 Nov 2019 23:25:15 -0300 Subject: [PATCH 25/85] Remove __future__ imports --- extras/qps-bench-server.py | 1 - scrapy/cmdline.py | 1 - scrapy/commands/check.py | 1 - scrapy/commands/fetch.py | 1 - scrapy/commands/genspider.py | 1 - scrapy/commands/list.py | 1 - scrapy/commands/parse.py | 1 - scrapy/commands/settings.py | 1 - scrapy/commands/startproject.py | 1 - scrapy/commands/version.py | 2 -- scrapy/core/downloader/__init__.py | 1 - scrapy/core/downloader/handlers/http.py | 1 - scrapy/downloadermiddlewares/ajaxcrawl.py | 1 - scrapy/dupefilters.py | 1 - scrapy/extensions/httpcache.py | 2 -- scrapy/pipelines/media.py | 2 -- scrapy/responsetypes.py | 1 - scrapy/shell.py | 2 -- scrapy/signalmanager.py | 1 - scrapy/spiderloader.py | 1 - scrapy/utils/boto.py | 2 -- scrapy/utils/display.py | 1 - scrapy/utils/engine.py | 1 - scrapy/utils/ossignal.py | 2 -- scrapy/utils/request.py | 1 - scrapy/utils/test.py | 1 - scrapy/utils/testproc.py | 1 - scrapy/utils/testsite.py | 1 - scrapy/utils/trackref.py | 1 - 29 files changed, 35 deletions(-) diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index 3bef20bf3..da7a0022b 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -from __future__ import print_function from time import time from collections import deque from twisted.web.server import Site, NOT_DONE_YET diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 418dc1ac9..69e917004 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,4 +1,3 @@ -from __future__ import print_function import sys import os import optparse diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index ab73e85e7..ac2a95c9b 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,4 +1,3 @@ -from __future__ import print_function import time import sys from collections import defaultdict diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index d45133e0e..95f6f7b9a 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,4 +1,3 @@ -from __future__ import print_function import sys from w3lib.url import is_url diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index d5498bb5c..adb01fa70 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,4 +1,3 @@ -from __future__ import print_function import os import shutil import string diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index a255b3b94..60686f109 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,4 +1,3 @@ -from __future__ import print_function from scrapy.commands import ScrapyCommand class Command(ScrapyCommand): diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ef8acd29c..ff6f1d8cd 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,4 +1,3 @@ -from __future__ import print_function import json import logging diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index bee52f06a..a34338715 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,4 +1,3 @@ -from __future__ import print_function import json from scrapy.commands import ScrapyCommand diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 67337c26e..34df25cef 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,4 +1,3 @@ -from __future__ import print_function import re import os import string diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 577365c3b..494855500 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import scrapy from scrapy.commands import ScrapyCommand from scrapy.utils.versions import scrapy_components_versions diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 949dacbc8..f5f8be6e8 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import import random import warnings from time import time diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index ac4b867c3..6111e132a 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,3 +1,2 @@ -from __future__ import absolute_import from .http10 import HTTP10DownloadHandler from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 72715dba7..78b802673 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import re import logging diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 0bcdd3495..f8802eb7d 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,4 +1,3 @@ -from __future__ import print_function import os import logging diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index f3fabf710..b1ed0b9f8 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import gzip import logging import os diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 95dca9a3f..c174addf9 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import functools import logging from collections import defaultdict diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index de62276c8..b64fbbd42 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -2,7 +2,6 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ -from __future__ import absolute_import from mimetypes import MimeTypes from pkgutil import get_data from io import StringIO diff --git a/scrapy/shell.py b/scrapy/shell.py index 80b625633..a649d555f 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -3,8 +3,6 @@ See documentation in docs/topics/shell.rst """ -from __future__ import print_function - import os import signal import warnings diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 296d27ed8..d474f1806 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import from pydispatch import dispatcher from scrapy.utils import signal as _signal diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 7478faa78..3beca4060 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import from collections import defaultdict import traceback import warnings diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index c8fc911bb..46816b54d 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,7 +1,5 @@ """Boto/botocore helpers""" -from __future__ import absolute_import - from scrapy.exceptions import NotConfigured diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index f6a6c4645..536de6b88 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -2,7 +2,6 @@ pprint and pformat wrappers with colorization support """ -from __future__ import print_function import sys from pprint import pformat as pformat_ diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 11dd36d91..36ef8626a 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -1,6 +1,5 @@ """Some debugging functions for working with the Scrapy engine""" -from __future__ import print_function from time import time # used in global tests code def get_engine_status(engine): diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index f87d5a803..7a7aec9be 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,5 +1,3 @@ - -from __future__ import absolute_import import signal from twisted.internet import reactor diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 63d0ae772..45f1ef17e 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -3,7 +3,6 @@ This module provides some useful functions for working with scrapy.http.Request objects """ -from __future__ import print_function import hashlib import weakref from six.moves.urllib.parse import urlunparse diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4b935c51b..febaa4dcc 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -2,7 +2,6 @@ This module contains some assorted functions used in tests """ -from __future__ import absolute_import import os from importlib import import_module diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index f268e91ff..0f15cf60a 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import import sys import os diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index e50a989b3..05d06d53b 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -1,4 +1,3 @@ -from __future__ import print_function from six.moves.urllib.parse import urljoin from twisted.internet import reactor diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index eed14c5a1..78389e464 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -9,7 +9,6 @@ and no performance penalty at all when disabled (as object_ref becomes just an alias to object in that case). """ -from __future__ import print_function import weakref from time import time from operator import itemgetter From c0bfaef37abe029c98072e7b1a1bf346ab79016c Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sat, 2 Nov 2019 23:27:04 -0300 Subject: [PATCH 26/85] Remove __future__ imports from tests --- tests/mockserver.py | 1 - tests/test_downloadermiddleware_httpcache.py | 1 - tests/test_downloadermiddleware_robotstxt.py | 2 -- tests/test_engine.py | 1 - tests/test_exporters.py | 1 - tests/test_feedexport.py | 1 - tests/test_pipeline_media.py | 2 -- tests/test_utils_deprecate.py | 1 - tests/test_utils_log.py | 1 - tests/test_utils_request.py | 1 - 10 files changed, 12 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 77908284b..8aff7b525 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,4 +1,3 @@ -from __future__ import print_function import sys, time, random, os, json from six.moves.urllib.parse import urlencode from subprocess import Popen, PIPE diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 950664ffe..db0843b57 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -1,4 +1,3 @@ -from __future__ import print_function import time import tempfile import shutil diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 8266bf35f..a1645ed96 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - from unittest import mock from twisted.internet import reactor, error diff --git a/tests/test_engine.py b/tests/test_engine.py index 30150391a..d5b911a40 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -10,7 +10,6 @@ module with the ``runserver`` argument:: python test_engine.py runserver """ -from __future__ import print_function import sys, os, re from six.moves.urllib.parse import urlparse diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 0046c5666..7880c5bf8 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import import re import json import marshal diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index abe2ab557..b13b16b41 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import import os import csv import json diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index ad2618ec9..70f11466b 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import sys from testfixtures import LogCapture diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index ce04e7f29..159ef8f25 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import inspect import unittest from unittest import mock diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 742e04803..2c23f3616 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function import sys import logging import unittest diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 625a32048..446497086 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,4 +1,3 @@ -from __future__ import print_function import unittest from scrapy.http import Request from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \ From df00389c16fc8e74c5ab7737e5bffcbde6cc011b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sat, 2 Nov 2019 23:48:11 -0300 Subject: [PATCH 27/85] Remove six.moves occurrences --- scrapy/_monkeypatches.py | 2 +- scrapy/commands/bench.py | 3 +-- scrapy/core/downloader/handlers/ftp.py | 2 +- scrapy/core/downloader/handlers/http11.py | 4 ++-- scrapy/core/downloader/handlers/s3.py | 2 +- scrapy/core/downloader/webclient.py | 2 +- scrapy/downloadermiddlewares/httpproxy.py | 5 ++--- scrapy/downloadermiddlewares/redirect.py | 3 ++- scrapy/exporters.py | 7 ++++--- scrapy/extensions/feedexport.py | 2 +- scrapy/extensions/httpcache.py | 2 +- scrapy/extensions/spiderstate.py | 3 ++- scrapy/http/cookies.py | 5 ++--- scrapy/http/request/form.py | 4 ++-- scrapy/http/request/rpc.py | 2 +- scrapy/http/response/__init__.py | 2 +- scrapy/http/response/text.py | 4 ++-- scrapy/linkextractors/__init__.py | 2 +- scrapy/linkextractors/htmlparser.py | 6 +++--- scrapy/linkextractors/lxmlhtml.py | 4 ++-- scrapy/linkextractors/regex.py | 3 ++- scrapy/linkextractors/sgml.py | 4 ++-- scrapy/mail.py | 14 +++++++------- scrapy/pipelines/files.py | 10 +++++----- scrapy/robotstxt.py | 2 +- scrapy/spidermiddlewares/referer.py | 2 +- scrapy/squeues.py | 2 +- scrapy/utils/benchserver.py | 3 ++- scrapy/utils/curl.py | 4 ++-- scrapy/utils/httpobj.py | 2 +- scrapy/utils/project.py | 6 +++--- scrapy/utils/python.py | 2 +- scrapy/utils/request.py | 6 +++--- scrapy/utils/sitemap.py | 3 ++- scrapy/utils/testsite.py | 2 +- scrapy/utils/url.py | 2 +- 36 files changed, 68 insertions(+), 65 deletions(-) diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index 1f8067b35..f74f89bda 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -1,4 +1,4 @@ -from six.moves import copyreg +import copyreg # Undo what Twisted's perspective broker adds to pickle register diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 90c8d56a2..7bbe362e7 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,8 +1,7 @@ import sys import time import subprocess - -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode import scrapy from scrapy.commands import ScrapyCommand diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 806a537d4..2116a5a44 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys: import re from io import BytesIO -from six.moves.urllib.parse import unquote +from urllib.parse import unquote from twisted.internet import reactor from twisted.protocols.ftp import FTPClient, CommandFailed diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 7d917cb74..63dedc19b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -2,10 +2,10 @@ import re import logging +import warnings from io import BytesIO from time import time -import warnings -from six.moves.urllib.parse import urldefrag +from urllib.parse import urldefrag from zope.interface import implementer from twisted.internet import defer, reactor, protocol diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index d8bbdd326..85b733228 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,4 +1,4 @@ -from six.moves.urllib.parse import unquote +from urllib.parse import unquote from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 3a5890ed0..9699da109 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,5 +1,5 @@ from time import time -from six.moves.urllib.parse import urlparse, urlunparse, urldefrag +from urllib.parse import urlparse, urlunparse, urldefrag from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 2212d9688..814ce78fe 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,7 +1,6 @@ import base64 -from six.moves.urllib.parse import unquote, urlunparse -from six.moves.urllib.request import getproxies, proxy_bypass -from urllib.request import _parse_proxy +from urllib.parse import unquote, urlunparse +from urllib.request import getproxies, proxy_bypass, _parse_proxy from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index b73f864dd..77cb5aa94 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,5 +1,5 @@ import logging -from six.moves.urllib.parse import urljoin, urlparse +from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string @@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse from scrapy.utils.response import get_meta_refresh from scrapy.exceptions import IgnoreRequest, NotConfigured + logger = logging.getLogger(__name__) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index f276c28e8..d9531e67c 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -7,15 +7,16 @@ import io import sys import pprint import marshal -import six -from six.moves import cPickle as pickle +import warnings +from import pickle from xml.sax.saxutils import XMLGenerator +import six + from scrapy.utils.serialize import ScrapyJSONEncoder from scrapy.utils.python import to_bytes, to_unicode, is_listlike from scrapy.item import BaseItem from scrapy.exceptions import ScrapyDeprecationWarning -import warnings __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 07ffd3476..0b854e6f3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -10,7 +10,7 @@ import logging import posixpath from tempfile import NamedTemporaryFile from datetime import datetime -from six.moves.urllib.parse import urlparse, unquote +from urllib.parse import urlparse, unquote from ftplib import FTP from zope.interface import Interface, implementer diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index b1ed0b9f8..b98ec218f 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,13 +1,13 @@ import gzip import logging import os +import pickle from email.utils import mktime_tz, parsedate_tz from importlib import import_module from time import time from warnings import warn from weakref import WeakKeyDictionary -from six.moves import cPickle as pickle from w3lib.http import headers_raw_to_dict, headers_dict_to_raw from scrapy.exceptions import ScrapyDeprecationWarning diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 2220cbd8f..2c8e46914 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,10 +1,11 @@ import os -from six.moves import cPickle as pickle +import pickle from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir + class SpiderState(object): """Store and load spider state during a scraping job""" diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 4532c3ab7..c39de0b52 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,7 +1,6 @@ import time -from six.moves.http_cookiejar import ( - CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE -) +from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE + from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index b6feede07..d2bcf77b7 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,10 +5,10 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -import six -from six.moves.urllib.parse import urljoin, urlencode +from urllib.parse import urljoin, urlencode import lxml.html +import six from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index bd09f7534..811d3ad6b 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from six.moves import xmlrpc_client as xmlrpclib +import xmlrpc.client as xmlrpclib from scrapy.http.request import Request from scrapy.utils.python import get_func_args diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index a81404afb..64e9c6c20 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,7 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from scrapy.http.request import Request from scrapy.http.headers import Headers diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 37f450e54..69bcba577 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -5,10 +5,10 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ -import six -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin import parsel +import six from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding from w3lib.html import strip_html5_whitespace diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index ebf3cd7d8..594c7264d 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -6,8 +6,8 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ import re +from urllib.parse import urlparse -from six.moves.urllib.parse import urlparse from parsel.csstranslator import HTMLTranslator from w3lib.url import canonicalize_url diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py index 27978a8a1..623732cee 100644 --- a/scrapy/linkextractors/htmlparser.py +++ b/scrapy/linkextractors/htmlparser.py @@ -2,10 +2,10 @@ HTMLParser-based link extractor """ import warnings -import six -from six.moves.html_parser import HTMLParser -from six.moves.urllib.parse import urljoin +from html.parser import HTMLParser +from urllib.parse import urljoin +import six from w3lib.url import safe_url_string from w3lib.html import strip_html5_whitespace diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 890c019c8..5b7e709de 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,9 +1,9 @@ """ Link extractor based on lxml.html """ -import six -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin +import six import lxml.etree as etree from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py index e689b4727..f96db256b 100644 --- a/scrapy/linkextractors/regex.py +++ b/scrapy/linkextractors/regex.py @@ -1,11 +1,12 @@ import re -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url from scrapy.link import Link from .sgml import SgmlLinkExtractor + linkre = re.compile( "<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>", re.DOTALL | re.IGNORECASE) diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 8940a4d77..98bed15e9 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -1,11 +1,11 @@ """ SGMLParser-based Link extractors """ -import six -from six.moves.urllib.parse import urljoin import warnings +from urllib.parse import urljoin from sgmllib import SGMLParser +import six from w3lib.url import safe_url_string, canonicalize_url from w3lib.html import strip_html5_whitespace diff --git a/scrapy/mail.py b/scrapy/mail.py index d24de2212..891bb5e09 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -3,21 +3,21 @@ Mail sending helpers See documentation in docs/topics/email.rst """ -from io import BytesIO import logging - -from email.utils import COMMASPACE, formatdate -from six.moves.email_mime_multipart import MIMEMultipart -from six.moves.email_mime_text import MIMEText -from six.moves.email_mime_base import MIMEBase -from email.mime.nonmultipart import MIMENonMultipart from email import encoders as Encoders +from email.mime.base import MIMEBase +from email.mime.multipart import MIMEMultipart +from email.mime.nonmultipart import MIMENonMultipart +from email.mime.text import MIMEText +from email.utils import COMMASPACE, formatdate +from io import BytesIO from twisted.internet import defer, reactor, ssl from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes + logger = logging.getLogger(__name__) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8d74c5011..432d4c182 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -5,17 +5,17 @@ See documentation in topics/media-pipeline.rst """ import functools import hashlib -from io import BytesIO +import logging import mimetypes import os import os.path import time -import logging -from email.utils import parsedate_tz, mktime_tz -from six.moves.urllib.parse import urlparse from collections import defaultdict -import six +from email.utils import parsedate_tz, mktime_tz +from io import BytesIO +from urllib.parse import urlparse +import six from twisted.internet import defer, threads from scrapy.pipelines.media import MediaPipeline diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 95a8c09b8..397924110 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -53,7 +53,7 @@ class RobotParser(with_metaclass(ABCMeta)): class PythonRobotParser(RobotParser): def __init__(self, robotstxt_body, spider): - from six.moves.urllib_robotparser import RobotFileParser + from urllib.robotparser import RobotFileParser self.spider = spider robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True) self.rp = RobotFileParser() diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index c76e4d5a2..dce2b3598 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,8 +2,8 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ -from six.moves.urllib.parse import urlparse import warnings +from urllib.parse import urlparse from w3lib.url import safe_url_string diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 30cc926e5..d5d3be67e 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -3,7 +3,7 @@ Scheduler queues """ import marshal -from six.moves import cPickle as pickle +import pickle from queuelib import queue diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 5bbda6e27..cdbe21942 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -1,5 +1,6 @@ import random -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode + from twisted.web.server import Site from twisted.web.resource import Resource from twisted.internet import reactor diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index b3fd0a497..a0a47e473 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -1,9 +1,9 @@ import argparse import warnings from shlex import split +from http.cookies import SimpleCookie +from urllib.parse import urlparse -from six.moves.http_cookies import SimpleCookie -from six.moves.urllib.parse import urlparse from six import string_types, iteritems from w3lib.http import basic_auth_header diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index b4c929b0e..54ffe086d 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,8 +1,8 @@ """Helper functions for scrapy.http objects (Request, Response)""" import weakref +from urllib.parse import urlparse -from six.moves.urllib.parse import urlparse _urlparse_cache = weakref.WeakKeyDictionary() def urlparse_cached(request_or_response): diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 1cbda141a..f28c2eaa1 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,5 +1,5 @@ import os -from six.moves import cPickle as pickle +import pickle import warnings from importlib import import_module @@ -7,8 +7,8 @@ from os.path import join, dirname, abspath, isabs, exists from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env from scrapy.settings import Settings -from scrapy.exceptions import NotConfigured -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning + ENVVAR = 'SCRAPY_SETTINGS_MODULE' DATADIR_CFG_SECTION = 'datadir' diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 974abaeb1..cb8ac3c8e 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -65,7 +65,7 @@ def is_listlike(x): True >>> is_listlike((x for x in range(3))) True - >>> is_listlike(six.moves.xrange(5)) + >>> is_listlike(range(5)) True """ return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes)) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 45f1ef17e..c5e877acd 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -5,13 +5,13 @@ scrapy.http.Request objects import hashlib import weakref -from six.moves.urllib.parse import urlunparse +from urllib.parse import urlunparse from w3lib.http import basic_auth_header -from scrapy.utils.python import to_bytes, to_unicode - from w3lib.url import canonicalize_url + from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_bytes, to_unicode _fingerprint_cache = weakref.WeakKeyDictionary() diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 4742b3e13..2f10cf4de 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -5,8 +5,9 @@ Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ +from urllib.parse import urljoin + import lxml.etree -from six.moves.urllib.parse import urljoin class Sitemap(object): diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 05d06d53b..6f5c21624 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -1,4 +1,4 @@ -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from twisted.internet import reactor from twisted.web import server, resource, static, util diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index b3a4be007..4b48868fe 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead. """ import posixpath import re -from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse) +from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this # move doesn't break old code From 5ab0f189ce33f235b847f96b66a243d808b5ce1f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 3 Nov 2019 00:01:09 -0300 Subject: [PATCH 28/85] Remove six.moves occurrences from tests --- tests/mockserver.py | 8 ++++++-- tests/spiders.py | 2 +- tests/test_crawl.py | 2 +- tests/test_engine.py | 6 ++++-- tests/test_exporters.py | 2 +- tests/test_feedexport.py | 6 +++--- tests/test_http_cookies.py | 2 +- tests/test_http_request.py | 8 +++----- tests/test_pipeline_files.py | 4 ++-- tests/test_proxy_connect.py | 6 +++--- tests/test_spidermiddleware_offsite.py | 9 ++++----- tests/test_spidermiddleware_referer.py | 2 +- tests/test_urlparse_monkeypatches.py | 2 +- tests/test_utils_datatypes.py | 8 -------- tests/test_utils_defer.py | 6 ++---- tests/test_utils_httpobj.py | 3 ++- tests/test_utils_response.py | 3 ++- tests/test_utils_url.py | 3 ++- 18 files changed, 39 insertions(+), 43 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 8aff7b525..15b1b24f5 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,6 +1,10 @@ -import sys, time, random, os, json -from six.moves.urllib.parse import urlencode +import json +import os +import random +import sys +import time from subprocess import Popen, PIPE +from urllib.parse import urlencode from OpenSSL import SSL from twisted.web.server import Site, NOT_DONE_YET diff --git a/tests/spiders.py b/tests/spiders.py index 7816bf7c7..72a428c50 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -3,7 +3,7 @@ Some spiders used for testing and benchmarking """ import time -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode from scrapy.spiders import Spider from scrapy.http import Request diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3fc13eeb7..3307899b7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -140,7 +140,7 @@ class CrawlTestCase(TestCase): def test_unbounded_response(self): # Completeness of responses without Content-Length or Transfer-Encoding # can not be determined, we treat them as valid but flagged as "partial" - from six.moves.urllib.parse import urlencode + from urllib.parse import urlencode query = urlencode({'raw': '''\ HTTP/1.1 200 OK Server: Apache-Coyote/1.1 diff --git a/tests/test_engine.py b/tests/test_engine.py index d5b911a40..002c4e6bb 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -10,8 +10,10 @@ module with the ``runserver`` argument:: python test_engine.py runserver """ -import sys, os, re -from six.moves.urllib.parse import urlparse +import os +import re +import sys +from urllib.parse import urlparse from twisted.internet import reactor, defer from twisted.web import server, static, util diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 7880c5bf8..f151a1285 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1,11 +1,11 @@ import re import json import marshal +import pickle import tempfile import unittest from io import BytesIO from datetime import datetime -from six.moves import cPickle as pickle import lxml.etree import six diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index b13b16b41..7a71669c3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -2,13 +2,13 @@ import os import csv import json import warnings -from io import BytesIO import tempfile import shutil import string +from io import BytesIO from unittest import mock -from six.moves.urllib.parse import urljoin, urlparse, quote -from six.moves.urllib.request import pathname2url +from urllib.parse import urljoin, urlparse, quote +from urllib.request import pathname2url from zope.interface.verify import verifyObject from twisted.trial import unittest diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 0a9ed500a..45ddb42ba 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,4 +1,4 @@ -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from unittest import TestCase from scrapy.http import Request, Response diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 807265981..62d2847d7 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -3,12 +3,10 @@ import cgi import unittest import re import json -from unittest import mock -from urllib.parse import unquote_to_bytes +import xmlrpc.client as xmlrpclib import warnings - -from six.moves import xmlrpc_client as xmlrpclib -from six.moves.urllib.parse import urlparse, parse_qs, unquote +from unittest import mock +from urllib.parse import parse_qs, unquote, unquote_to_bytes, urlparse from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_unicode diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index bd40e4103..dede4bf12 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -4,9 +4,9 @@ import time from tempfile import mkdtemp from shutil import rmtree from unittest import mock -from six.moves.urllib.parse import urlparse -from six import BytesIO +from urllib.parse import urlparse +from six import BytesIO from twisted.trial import unittest from twisted.internet import defer diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index ae1236bcb..2b62a378d 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -1,15 +1,15 @@ import json import os import time - -from six.moves.urllib.parse import urlsplit, urlunsplit +from urllib.parse import urlsplit, urlunsplit from threading import Thread + from libmproxy import controller, proxy from netlib import http_auth from testfixtures import LogCapture - from twisted.internet import defer from twisted.trial.unittest import TestCase + from scrapy.utils.test import get_crawler from scrapy.http import Request from tests.spiders import SimpleSpider, SingleRequestSpider diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 7e4af0d4c..5833591af 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -1,13 +1,12 @@ from unittest import TestCase - -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse +import warnings from scrapy.http import Response, Request from scrapy.spiders import Spider -from scrapy.spidermiddlewares.offsite import OffsiteMiddleware -from scrapy.spidermiddlewares.offsite import URLWarning +from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning from scrapy.utils.test import get_crawler -import warnings + class TestOffsiteMiddleware(TestCase): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 21439c20e..23c38d9b1 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,4 +1,4 @@ -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from unittest import TestCase import warnings diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py index 22e39821c..bea0cf3e5 100644 --- a/tests/test_urlparse_monkeypatches.py +++ b/tests/test_urlparse_monkeypatches.py @@ -1,4 +1,4 @@ -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse import unittest diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 47877f555..8782e4c53 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -192,14 +192,6 @@ class SequenceExcludeTest(unittest.TestCase): self.assertIn(20, d) self.assertNotIn(15, d) - def test_six_range(self): - import six.moves - seq = six.moves.range(10**3, 10**6) - d = SequenceExclude(seq) - self.assertIn(10**2, d) - self.assertIn(10**7, d) - self.assertNotIn(10**4, d) - def test_range_step(self): seq = range(10, 20, 3) d = SequenceExclude(seq) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 003bb9b02..0d8c46657 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -5,8 +5,6 @@ from twisted.python.failure import Failure from scrapy.utils.defer import mustbe_deferred, process_chain, \ process_chain_both, process_parallel, iter_errback -from six.moves import xrange - class MustbeDeferredTest(unittest.TestCase): def test_success_function(self): @@ -83,7 +81,7 @@ class IterErrbackTest(unittest.TestCase): def test_iter_errback_good(self): def itergood(): - for x in xrange(10): + for x in range(10): yield x errors = [] @@ -93,7 +91,7 @@ class IterErrbackTest(unittest.TestCase): def test_iter_errback_bad(self): def iterbad(): - for x in xrange(10): + for x in range(10): if x == 5: a = 1/0 yield x diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 4f9f7a370..cf8ad1f23 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -1,9 +1,10 @@ import unittest -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached + class HttpobjUtilsTest(unittest.TestCase): def test_urlparse_cached(self): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index bea4dade3..6ebf290c0 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,12 +1,13 @@ import os import unittest -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from scrapy.http import Response, TextResponse, HtmlResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import (response_httprepr, open_in_browser, get_meta_refresh, get_base_url, response_status_message) + __doctests__ = ['scrapy.utils.response'] diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index e6588055c..a8e37d7b8 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,14 +1,15 @@ # -*- coding: utf-8 -*- import unittest +from urllib.parse import urlparse import six -from six.moves.urllib.parse import urlparse from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, add_http_if_no_scheme, guess_scheme, parse_url, strip_url) + __doctests__ = ['scrapy.utils.url'] From 1aba5136939ff26503a59ca9fb9320a030aa77de Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 3 Nov 2019 00:26:44 -0300 Subject: [PATCH 29/85] Remove six.iter* occurrences --- scrapy/core/downloader/__init__.py | 3 +-- scrapy/core/downloader/handlers/__init__.py | 5 +++-- scrapy/downloadermiddlewares/cookies.py | 8 +++++--- scrapy/downloadermiddlewares/decompression.py | 7 +++---- scrapy/exporters.py | 6 +++--- scrapy/extensions/memdebug.py | 3 +-- scrapy/http/request/form.py | 3 +-- scrapy/item.py | 2 +- scrapy/loader/__init__.py | 6 ++---- scrapy/pipelines/files.py | 11 +++++------ scrapy/pipelines/images.py | 3 +-- scrapy/responsetypes.py | 3 +-- scrapy/settings/__init__.py | 8 ++++---- scrapy/utils/conf.py | 12 +++++------- scrapy/utils/datatypes.py | 7 ++----- scrapy/utils/python.py | 4 ++-- scrapy/utils/spider.py | 5 ++--- scrapy/utils/trackref.py | 13 ++++++------- tests/test_settings/__init__.py | 10 +++++----- tests/test_webclient.py | 2 +- 20 files changed, 54 insertions(+), 67 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index f5f8be6e8..73d84664f 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -4,7 +4,6 @@ from time import time from datetime import datetime from collections import deque -import six from twisted.internet import reactor, defer, task from scrapy.utils.defer import mustbe_deferred @@ -189,7 +188,7 @@ class Downloader(object): def close(self): self._slot_gc_loop.stop() - for slot in six.itervalues(self.slots): + for slot in self.slots.values(): slot.close() def _slot_gc(self, age=60): diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 0b55d32fa..39a0b1f51 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,8 +1,9 @@ """Download handlers for different schemes""" import logging + from twisted.internet import defer -import six + from scrapy.exceptions import NotSupported, NotConfigured from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object @@ -22,7 +23,7 @@ class DownloadHandlers(object): self._notconfigured = {} # remembers failed handlers handlers = without_none_values( crawler.settings.getwithbase('DOWNLOAD_HANDLERS')) - for scheme, clspath in six.iteritems(handlers): + for scheme, clspath in handlers.items(): self._schemes[scheme] = clspath self._load_handler(scheme, skip_lazy=True) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 0d2b9900c..9deba40d6 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -1,5 +1,4 @@ import os -import six import logging from collections import defaultdict @@ -8,6 +7,7 @@ from scrapy.http import Response from scrapy.http.cookies import CookieJar from scrapy.utils.python import to_unicode + logger = logging.getLogger(__name__) @@ -82,8 +82,10 @@ class CookiesMiddleware(object): def _get_request_cookies(self, jar, request): if isinstance(request.cookies, dict): - cookie_list = [{'name': k, 'value': v} for k, v in \ - six.iteritems(request.cookies)] + cookie_list = [ + {'name': k, 'value': v} + for k, v in request.cookies.items() + ] else: cookie_list = request.cookies diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index e2d73f347..dfd64c35f 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -4,16 +4,15 @@ and extract the potentially compressed responses that may arrive. import bz2 import gzip -from io import BytesIO import zipfile import tarfile import logging +from io import BytesIO from tempfile import mktemp -import six - from scrapy.responsetypes import responsetypes + logger = logging.getLogger(__name__) @@ -75,7 +74,7 @@ class DecompressionMiddleware(object): if not response.body: return response - for fmt, func in six.iteritems(self._formats): + for fmt, func in self._formats.items(): new_response = func(response) if new_response: logger.debug('Decompressed response with format: %(responsefmt)s', diff --git a/scrapy/exporters.py b/scrapy/exporters.py index d9531e67c..19af2d6e4 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -62,9 +62,9 @@ class BaseItemExporter(object): include_empty = self.export_empty_fields if self.fields_to_export is None: if include_empty and not isinstance(item, dict): - field_iter = six.iterkeys(item.fields) + field_iter = item.fields.keys() else: - field_iter = six.iterkeys(item) + field_iter = item.keys() else: if include_empty: field_iter = self.fields_to_export @@ -326,7 +326,7 @@ class PythonItemExporter(BaseItemExporter): return value def _serialize_dict(self, value): - for key, val in six.iteritems(value): + for key, val in value.items(): key = to_bytes(key) if self.binary else key yield key, self._serialize_value(val) diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index 263d8ce4c..892aa8a86 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst """ import gc -import six from scrapy import signals from scrapy.exceptions import NotConfigured @@ -28,7 +27,7 @@ class MemoryDebugger(object): def spider_closed(self, spider, reason): gc.collect() self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider) - for cls, wdict in six.iteritems(live_refs): + for cls, wdict in live_refs.items(): if not wdict: continue self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d2bcf77b7..af02c8484 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -8,7 +8,6 @@ See documentation in docs/topics/request-response.rst from urllib.parse import urljoin, urlencode import lxml.html -import six from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace @@ -208,7 +207,7 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such xpath = u'.//*' + \ - u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata)) + u''.join(u'[@%s="%s"]' % c for c in clickdata.items()) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') diff --git a/scrapy/item.py b/scrapy/item.py index 32f9b2ebb..4e0f0ac44 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -78,7 +78,7 @@ class DictItem(MutableMapping, BaseItem): def __init__(self, *args, **kwargs): self._values = {} if args or kwargs: # avoid creating dict for most common case - for k, v in six.iteritems(dict(*args, **kwargs)): + for k, v in dict(*args, **kwargs).items(): self[k] = v def __getitem__(self, key): diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 60fd6d222..fe01a856f 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -5,8 +5,6 @@ See documentation in docs/topics/loaders.rst """ from collections import defaultdict -import six - from scrapy.item import Item from scrapy.loader.common import wrap_loader_context from scrapy.loader.processors import Identity @@ -72,7 +70,7 @@ class ItemLoader(object): if value is None: return if not field_name: - for k, v in six.iteritems(value): + for k, v in value.items(): self._add_value(k, v) else: self._add_value(field_name, value) @@ -82,7 +80,7 @@ class ItemLoader(object): if value is None: return if not field_name: - for k, v in six.iteritems(value): + for k, v in value.items(): self._replace_value(k, v) else: self._replace_value(field_name, value) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 432d4c182..6d55c8980 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -8,14 +8,12 @@ import hashlib import logging import mimetypes import os -import os.path import time from collections import defaultdict from email.utils import parsedate_tz, mktime_tz from io import BytesIO from urllib.parse import urlparse -import six from twisted.internet import defer, threads from scrapy.pipelines.media import MediaPipeline @@ -29,6 +27,7 @@ from scrapy.utils.request import referer_str from scrapy.utils.boto import is_botocore from scrapy.utils.datatypes import CaselessDict + logger = logging.getLogger(__name__) @@ -153,14 +152,14 @@ class S3FilesStore(object): Bucket=self.bucket, Key=key_name, Body=buf, - Metadata={k: str(v) for k, v in six.iteritems(meta or {})}, + Metadata={k: str(v) for k, v in (meta or {}).items()}, ACL=self.POLICY, **extra) else: b = self._get_boto_bucket() k = b.new_key(key_name) if meta: - for metakey, metavalue in six.iteritems(meta): + for metakey, metavalue in meta.items(): k.set_metadata(metakey, str(metavalue)) h = self.HEADERS.copy() if headers: @@ -201,7 +200,7 @@ class S3FilesStore(object): 'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation', }) extra = {} - for key, value in six.iteritems(headers): + for key, value in headers.items(): try: kwarg = mapping[key] except KeyError: @@ -249,7 +248,7 @@ class GCSFilesStore(object): def persist_file(self, path, buf, info, meta=None, headers=None): blob = self.bucket.blob(self.prefix + path) blob.cache_control = self.CACHE_CONTROL - blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})} + blob.metadata = {k: str(v) for k, v in (meta or {}).items()} return threads.deferToThread( blob.upload_from_string, data=buf.getvalue(), diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e77cef4ff..e9c6b759c 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -6,7 +6,6 @@ See documentation in topics/media-pipeline.rst import functools import hashlib from io import BytesIO -import six from PIL import Image @@ -126,7 +125,7 @@ class ImagesPipeline(FilesPipeline): image, buf = self.convert_image(orig_image) yield path, image, buf - for thumb_id, size in six.iteritems(self.thumbs): + for thumb_id, size in self.thumbs.items(): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_path, thumb_image, thumb_buf diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index b64fbbd42..91d309147 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -5,7 +5,6 @@ based on different criteria. from mimetypes import MimeTypes from pkgutil import get_data from io import StringIO -import six from scrapy.http import Response from scrapy.utils.misc import load_object @@ -36,7 +35,7 @@ class ResponseTypes(object): self.mimetypes = MimeTypes() mimedata = get_data('scrapy', 'mime.types').decode('utf8') self.mimetypes.readfp(StringIO(mimedata)) - for mimetype, cls in six.iteritems(self.CLASSES): + for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) def from_mimetype(self, mimetype): diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index c871e86e0..d53b28895 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -317,10 +317,10 @@ class BaseSettings(MutableMapping): values = json.loads(values) if values is not None: if isinstance(values, BaseSettings): - for name, value in six.iteritems(values): + for name, value in values.items(): self.set(name, value, values.getpriority(name)) else: - for name, value in six.iteritems(values): + for name, value in values.items(): self.set(name, value, priority) def delete(self, name, priority='project'): @@ -377,7 +377,7 @@ class BaseSettings(MutableMapping): def _to_dict(self): return {k: (v._to_dict() if isinstance(v, BaseSettings) else v) - for k, v in six.iteritems(self)} + for k, v in self.items()} def copy_to_dict(self): """ @@ -445,7 +445,7 @@ class Settings(BaseSettings): self.setmodule(default_settings, 'default') # Promote default dictionaries to BaseSettings instances for per-key # priorities - for name, val in six.iteritems(self): + for name, val in self.items(): if isinstance(val, dict): self.set(name, BaseSettings(val, 'default'), 'default') self.update(values, priority) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 561bb72fc..7a15e77ff 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,11 +1,9 @@ -from configparser import ConfigParser import os import sys import numbers +from configparser import ConfigParser from operator import itemgetter -import six - from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values @@ -22,7 +20,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): def _map_keys(compdict): if isinstance(compdict, BaseSettings): compbs = BaseSettings() - for k, v in six.iteritems(compdict): + for k, v in compdict.items(): prio = compdict.getpriority(k) if compbs.getpriority(convert(k)) == prio: raise ValueError('Some paths in {!r} convert to the same ' @@ -33,11 +31,11 @@ def build_component_list(compdict, custom=None, convert=update_classpath): return compbs else: _check_components(compdict) - return {convert(k): v for k, v in six.iteritems(compdict)} + return {convert(k): v for k, v in compdict.items()} def _validate_values(compdict): """Fail if a value in the components dict is not a real number or None.""" - for name, value in six.iteritems(compdict): + for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): raise ValueError('Invalid value {} for component {}, please provide ' \ 'a real number or None instead'.format(value, name)) @@ -53,7 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): _validate_values(compdict) compdict = without_none_values(_map_keys(compdict)) - return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))] + return [k for k, v in sorted(compdict.items(), key=itemgetter(1))] def arglist_to_dict(arglist): diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 6e9de47f3..56d4d1b8e 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -7,11 +7,8 @@ This module must not depend on any module outside the Standard Library. import copy import collections -from collections.abc import Mapping import warnings -import six - from scrapy.exceptions import ScrapyDeprecationWarning @@ -151,7 +148,7 @@ class MultiValueDict(dict): self.setlistdefault(key, []).append(value) except TypeError: raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary") - for key, value in six.iteritems(kwargs): + for key, value in kwargs.items(): self.setlistdefault(key, []).append(value) @@ -226,7 +223,7 @@ class CaselessDict(dict): return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) def update(self, seq): - seq = seq.items() if isinstance(seq, Mapping) else seq + seq = seq.items() if isinstance(seq, collections.abc.Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super(CaselessDict, self).update(iseq) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index cb8ac3c8e..845c19fb9 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -300,7 +300,7 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): dict or a list of tuples, like any dict constructor supports. """ d = {} - for k, v in six.iteritems(dict(dct_or_tuples)): + for k, v in dict(dct_or_tuples).items(): k = k.encode(encoding) if isinstance(k, six.text_type) else k if not keys_only: v = v.encode(encoding) if isinstance(v, six.text_type) else v @@ -345,7 +345,7 @@ def without_none_values(iterable): value ``None`` have been removed. """ try: - return {k: v for k, v in six.iteritems(iterable) if v is not None} + return {k: v for k, v in iterable.items() if v is not None} except AttributeError: return type(iterable)((v for v in iterable if v is not None)) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 94b24f67e..48ad5041e 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -1,11 +1,10 @@ import logging import inspect -import six - from scrapy.spiders import Spider from scrapy.utils.misc import arg_to_iter + logger = logging.getLogger(__name__) @@ -21,7 +20,7 @@ def iter_spider_classes(module): # singleton in scrapy.spider.spiders from scrapy.spiders import Spider - for obj in six.itervalues(vars(module)): + for obj in vars(module).values(): if inspect.isclass(obj) and \ issubclass(obj, Spider) and \ obj.__module__ == module.__name__ and \ diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 78389e464..4842b95df 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -13,7 +13,6 @@ import weakref from time import time from operator import itemgetter from collections import defaultdict -import six NoneType = type(None) @@ -36,13 +35,13 @@ def format_live_refs(ignore=NoneType): """Return a tabular representation of tracked objects""" s = "Live References\n\n" now = time() - for cls, wdict in sorted(six.iteritems(live_refs), + for cls, wdict in sorted(live_refs.items(), key=lambda x: x[0].__name__): if not wdict: continue if issubclass(cls, ignore): continue - oldest = min(six.itervalues(wdict)) + oldest = min(wdict.values()) s += "%-30s %6d oldest: %ds ago\n" % ( cls.__name__, len(wdict), now - oldest ) @@ -56,15 +55,15 @@ def print_live_refs(*a, **kw): def get_oldest(class_name): """Get the oldest object for a specific class name""" - for cls, wdict in six.iteritems(live_refs): + for cls, wdict in live_refs.items(): if cls.__name__ == class_name: if not wdict: break - return min(six.iteritems(wdict), key=itemgetter(1))[0] + return min(wdict.items(), key=itemgetter(1))[0] def iter_all(class_name): """Iterate over all objects of the same class by its class name""" - for cls, wdict in six.iteritems(live_refs): + for cls, wdict in live_refs.items(): if cls.__name__ == class_name: - return six.iterkeys(wdict) + return wdict.keys() diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 32e65bed5..d5cbef6f5 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -10,7 +10,7 @@ from . import default_settings class SettingsGlobalFuncsTest(unittest.TestCase): def test_get_settings_priority(self): - for prio_str, prio_num in six.iteritems(SETTINGS_PRIORITIES): + for prio_str, prio_num in SETTINGS_PRIORITIES.items(): self.assertEqual(get_settings_priority(prio_str), prio_num) self.assertEqual(get_settings_priority(99), 99) @@ -148,10 +148,10 @@ class BaseSettingsTest(unittest.TestCase): self.settings.setmodule( 'tests.test_settings.default_settings', 10) - self.assertCountEqual(six.iterkeys(self.settings.attributes), - six.iterkeys(ctrl_attributes)) + self.assertCountEqual(self.settings.attributes.keys(), + ctrl_attributes.keys()) - for key in six.iterkeys(ctrl_attributes): + for key in ctrl_attributes.keys(): attr = self.settings.attributes[key] ctrl_attr = ctrl_attributes[key] self.assertEqual(attr.value, ctrl_attr.value) @@ -227,7 +227,7 @@ class BaseSettingsTest(unittest.TestCase): } settings = self.settings settings.attributes = {key: SettingsAttribute(value, 0) for key, value - in six.iteritems(test_configuration)} + in test_configuration.items()} self.assertTrue(settings.getbool('TEST_ENABLED1')) self.assertTrue(settings.getbool('TEST_ENABLED2')) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index a81946490..0c04e7114 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -318,7 +318,7 @@ class WebClientTestCase(unittest.TestCase): def cleanup(passthrough): # Clean up the server which is hanging around not doing # anything. - connected = list(six.iterkeys(self.wrapper.protocols)) + connected = list(self.wrapper.protocols.keys()) # There might be nothing here if the server managed to already see # that the connection was lost. if connected: From 68bf192172e548da49613064086a6e02cf0bae54 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 3 Nov 2019 00:32:07 -0300 Subject: [PATCH 30/85] Fix bad import --- scrapy/exporters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 19af2d6e4..2ff089ce9 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -8,7 +8,7 @@ import sys import pprint import marshal import warnings -from import pickle +import pickle from xml.sax.saxutils import XMLGenerator import six From ce8e515fa8960d0229132ac33fc5f8a6e43bbd6f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 3 Nov 2019 00:36:25 -0300 Subject: [PATCH 31/85] Remove six type wrappers --- scrapy/crawler.py | 2 +- scrapy/exporters.py | 4 ++-- scrapy/http/headers.py | 6 +++--- scrapy/http/request/__init__.py | 2 +- scrapy/http/response/text.py | 6 +++--- scrapy/linkextractors/htmlparser.py | 2 +- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/linkextractors/sgml.py | 2 +- scrapy/settings/__init__.py | 10 +++++----- scrapy/spiders/crawl.py | 2 +- scrapy/spiders/sitemap.py | 4 ++-- scrapy/utils/iterators.py | 6 +++--- scrapy/utils/misc.py | 6 +++--- scrapy/utils/python.py | 14 +++++++------- 14 files changed, 34 insertions(+), 34 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 19b998e0d..b88df8f64 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -205,7 +205,7 @@ class CrawlerRunner(object): return self._create_crawler(crawler_or_spidercls) def _create_crawler(self, spidercls): - if isinstance(spidercls, six.string_types): + if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 2ff089ce9..e45317491 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -182,7 +182,7 @@ class XmlItemExporter(BaseItemExporter): for value in serialized_value: self._export_xml_field('value', value, depth=depth+1) self._beautify_indent(depth=depth) - elif isinstance(serialized_value, six.text_type): + elif isinstance(serialized_value, str): self.xg.characters(serialized_value) else: self.xg.characters(str(serialized_value)) @@ -321,7 +321,7 @@ class PythonItemExporter(BaseItemExporter): if is_listlike(value): return [self._serialize_value(v) for v in value] encode_func = to_bytes if self.binary else to_unicode - if isinstance(value, (six.text_type, bytes)): + if isinstance(value, (str, bytes)): return encode_func(value, encoding=self.encoding) return value diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 62507eb19..5bbe7d72a 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -19,7 +19,7 @@ class Headers(CaselessDict): """Normalize values to bytes""" if value is None: value = [] - elif isinstance(value, (six.text_type, bytes)): + elif isinstance(value, (str, bytes)): value = [value] elif not hasattr(value, '__iter__'): value = [value] @@ -29,10 +29,10 @@ class Headers(CaselessDict): def _tobytes(self, x): if isinstance(x, bytes): return x - elif isinstance(x, six.text_type): + elif isinstance(x, str): return x.encode(self.encoding) elif isinstance(x, int): - return six.text_type(x).encode(self.encoding) + return str(x).encode(self.encoding) else: raise TypeError('Unsupported value type: {}'.format(type(x))) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index d09eaf849..ff7a44545 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -60,7 +60,7 @@ class Request(object_ref): return self._url def _set_url(self, url): - if not isinstance(url, six.string_types): + if not isinstance(url, str): raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__) s = safe_url_string(url, self.encoding) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 69bcba577..65400803f 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -31,14 +31,14 @@ class TextResponse(Response): super(TextResponse, self).__init__(*args, **kwargs) def _set_url(self, url): - if isinstance(url, six.text_type): + if isinstance(url, str): self._url = to_unicode(url, self.encoding) else: super(TextResponse, self)._set_url(url) def _set_body(self, body): self._body = b'' # used by encoding detection - if isinstance(body, six.text_type): + if isinstance(body, str): if self._encoding is None: raise TypeError('Cannot convert unicode body - %s has no encoding' % type(self).__name__) @@ -158,7 +158,7 @@ class TextResponse(Response): def _url_from_selector(sel): # type: (parsel.Selector) -> str - if isinstance(sel.root, six.string_types): + if isinstance(sel.root, str): # e.g. ::attr(href) result return strip_html5_whitespace(sel.root) if not hasattr(sel.root, 'tag'): diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py index 623732cee..2fec35799 100644 --- a/scrapy/linkextractors/htmlparser.py +++ b/scrapy/linkextractors/htmlparser.py @@ -42,7 +42,7 @@ class HtmlParserLinkExtractor(HTMLParser): ret = [] base_url = urljoin(response_url, self.base_url) if self.base_url else response_url for link in links: - if isinstance(link.url, six.text_type): + if isinstance(link.url, str): link.url = link.url.encode(response_encoding) try: link.url = urljoin(base_url, link.url) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 5b7e709de..496ad3053 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -22,7 +22,7 @@ _collect_string_content = etree.XPath("string()") def _nons(tag): - if isinstance(tag, six.string_types): + if isinstance(tag, str): if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE: return tag.split('}')[-1] return tag diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 98bed15e9..a9dffcad3 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -49,7 +49,7 @@ class BaseSgmlLinkExtractor(SGMLParser): if base_url is None: base_url = urljoin(response_url, self.base_url) if self.base_url else response_url for link in self.links: - if isinstance(link.url, six.text_type): + if isinstance(link.url, str): link.url = link.url.encode(response_encoding) try: link.url = urljoin(base_url, link.url) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index d53b28895..c88c9c0e2 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -23,7 +23,7 @@ def get_settings_priority(priority): :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its numerical value, or directly returns a given numerical priority. """ - if isinstance(priority, six.string_types): + if isinstance(priority, str): return SETTINGS_PRIORITIES[priority] else: return priority @@ -173,7 +173,7 @@ class BaseSettings(MutableMapping): :type default: any """ value = self.get(name, default or []) - if isinstance(value, six.string_types): + if isinstance(value, str): value = value.split(',') return list(value) @@ -194,7 +194,7 @@ class BaseSettings(MutableMapping): :type default: any """ value = self.get(name, default or {}) - if isinstance(value, six.string_types): + if isinstance(value, str): value = json.loads(value) return dict(value) @@ -284,7 +284,7 @@ class BaseSettings(MutableMapping): :type priority: string or int """ self._assert_mutability() - if isinstance(module, six.string_types): + if isinstance(module, str): module = import_module(module) for key in dir(module): if key.isupper(): @@ -313,7 +313,7 @@ class BaseSettings(MutableMapping): :type priority: string or int """ self._assert_mutability() - if isinstance(values, six.string_types): + if isinstance(values, str): values = json.loads(values) if values is not None: if isinstance(values, BaseSettings): diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 03000ce54..59e2c5661 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -25,7 +25,7 @@ def _identity(request, response): def _get_method(method, spider): if callable(method): return method - elif isinstance(method, six.string_types): + elif isinstance(method, str): return getattr(spider, method, None) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 534c45c70..2917daf57 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -22,7 +22,7 @@ class SitemapSpider(Spider): super(SitemapSpider, self).__init__(*a, **kw) self._cbs = [] for r, c in self.sitemap_rules: - if isinstance(c, six.string_types): + if isinstance(c, str): c = getattr(self, c) self._cbs.append((regex(r), c)) self._follow = [regex(x) for x in self.sitemap_follow] @@ -86,7 +86,7 @@ class SitemapSpider(Spider): def regex(x): - if isinstance(x, six.string_types): + if isinstance(x, str): return re.compile(x) return x diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 9693ba768..10481fe8d 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -60,7 +60,7 @@ class _StreamReader(object): self._text, self.encoding = obj.body, obj.encoding else: self._text, self.encoding = obj, 'utf-8' - self._is_unicode = isinstance(self._text, six.text_type) + self._is_unicode = isinstance(self._text, str) def read(self, n=65535): self.read = self._read_unicode if self._is_unicode else self._read_string @@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - expected_types = (Response, six.text_type, six.binary_type) + expected_types = (Response, str, bytes) assert isinstance(obj, expected_types), \ "obj must be %s, not %s" % ( " or ".join(t.__name__ for t in expected_types), @@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True): return obj.text else: return obj.body.decode('utf-8') - elif isinstance(obj, six.text_type): + elif isinstance(obj, str): return obj if unicode else obj.encode('utf-8') else: return obj.decode('utf-8') if unicode else obj diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index f638adb25..9a44f3576 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -13,7 +13,7 @@ from scrapy.utils.python import flatten, to_unicode from scrapy.item import BaseItem -_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes +_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes def arg_to_iter(arg): @@ -83,7 +83,7 @@ def extract_regex(regex, text, encoding='utf-8'): * if the regex doesn't contain any group the entire regex matching is returned """ - if isinstance(regex, six.string_types): + if isinstance(regex, str): regex = re.compile(regex, re.UNICODE) try: @@ -92,7 +92,7 @@ def extract_regex(regex, text, encoding='utf-8'): strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) - if isinstance(text, six.text_type): + if isinstance(text, str): return [replace_entities(s, keep=['lt', 'amp']) for s in strings] else: return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp']) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 845c19fb9..18fee1964 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -68,7 +68,7 @@ def is_listlike(x): >>> is_listlike(range(5)) True """ - return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes)) + return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) def unique(list_, key=lambda x: x): @@ -87,9 +87,9 @@ def unique(list_, key=lambda x: x): def to_unicode(text, encoding=None, errors='strict'): """Return the unicode representation of a bytes object ``text``. If ``text`` is already an unicode object, return it as-is.""" - if isinstance(text, six.text_type): + if isinstance(text, str): return text - if not isinstance(text, (bytes, six.text_type)): + if not isinstance(text, (bytes, str)): raise TypeError('to_unicode must receive a bytes or str ' 'object, got %s' % type(text).__name__) if encoding is None: @@ -102,7 +102,7 @@ def to_bytes(text, encoding=None, errors='strict'): is already a bytes object, return it as-is.""" if isinstance(text, bytes): return text - if not isinstance(text, six.string_types): + if not isinstance(text, str): raise TypeError('to_bytes must receive a str or bytes ' 'object, got %s' % type(text).__name__) if encoding is None: @@ -138,7 +138,7 @@ def re_rsearch(pattern, text, chunk_size=1024): yield (text[offset:], offset) yield (text, 0) - if isinstance(pattern, six.string_types): + if isinstance(pattern, str): pattern = re.compile(pattern) for chunk, offset in _chunk_iter(): @@ -301,9 +301,9 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): """ d = {} for k, v in dict(dct_or_tuples).items(): - k = k.encode(encoding) if isinstance(k, six.text_type) else k + k = k.encode(encoding) if isinstance(k, str) else k if not keys_only: - v = v.encode(encoding) if isinstance(v, six.text_type) else v + v = v.encode(encoding) if isinstance(v, str) else v d[k] = v return d From 54a786b102a3862c94fcdb123a26b54a39892697 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 3 Nov 2019 00:58:47 -0300 Subject: [PATCH 32/85] Remove six imports --- scrapy/crawler.py | 6 +++--- scrapy/exporters.py | 2 -- scrapy/http/headers.py | 4 ---- scrapy/http/request/__init__.py | 1 - scrapy/http/response/text.py | 1 - scrapy/linkextractors/htmlparser.py | 1 - scrapy/linkextractors/lxmlhtml.py | 1 - scrapy/linkextractors/sgml.py | 1 - scrapy/settings/__init__.py | 1 - scrapy/spiders/crawl.py | 2 -- scrapy/spiders/sitemap.py | 1 - scrapy/utils/curl.py | 3 +-- scrapy/utils/iterators.py | 6 +++--- scrapy/utils/misc.py | 1 - tests/test_http_headers.py | 3 --- tests/test_http_request.py | 2 +- 16 files changed, 8 insertions(+), 28 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index b88df8f64..4d7d9bac4 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,9 +1,8 @@ -import six -import signal import logging +import signal +import sys import warnings -import sys from twisted.internet import reactor, defer from zope.interface.verify import verifyClass, DoesNotImplement @@ -22,6 +21,7 @@ from scrapy.utils.log import ( get_scrapy_root_handler, install_scrapy_root_handler) from scrapy import signals + logger = logging.getLogger(__name__) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index e45317491..f2999daea 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -11,8 +11,6 @@ import warnings import pickle from xml.sax.saxutils import XMLGenerator -import six - from scrapy.utils.serialize import ScrapyJSONEncoder from scrapy.utils.python import to_bytes, to_unicode, is_listlike from scrapy.item import BaseItem diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 5bbe7d72a..860a5c9c6 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,4 +1,3 @@ -import six from w3lib.http import headers_dict_to_raw from scrapy.utils.datatypes import CaselessDict from scrapy.utils.python import to_unicode @@ -68,9 +67,6 @@ class Headers(CaselessDict): self[key] = lst def items(self): - return list(self.iteritems()) - - def iteritems(self): return ((k, self.getlist(k)) for k in self.keys()) def values(self): diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index ff7a44545..1e4a9c166 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,7 +4,6 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ -import six from w3lib.url import safe_url_string from scrapy.http.headers import Headers diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 65400803f..1079fd6e8 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -8,7 +8,6 @@ See documentation in docs/topics/request-response.rst from urllib.parse import urljoin import parsel -import six from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding from w3lib.html import strip_html5_whitespace diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py index 2fec35799..0425d4340 100644 --- a/scrapy/linkextractors/htmlparser.py +++ b/scrapy/linkextractors/htmlparser.py @@ -5,7 +5,6 @@ import warnings from html.parser import HTMLParser from urllib.parse import urljoin -import six from w3lib.url import safe_url_string from w3lib.html import strip_html5_whitespace diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 496ad3053..cb55e805a 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -3,7 +3,6 @@ Link extractor based on lxml.html """ from urllib.parse import urljoin -import six import lxml.etree as etree from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index a9dffcad3..2ba6bca45 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -5,7 +5,6 @@ import warnings from urllib.parse import urljoin from sgmllib import SGMLParser -import six from w3lib.url import safe_url_string, canonicalize_url from w3lib.html import strip_html5_whitespace diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index c88c9c0e2..b6133619c 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,4 +1,3 @@ -import six import json import copy from collections.abc import MutableMapping diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 59e2c5661..a5eb1a518 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -8,8 +8,6 @@ See documentation in docs/topics/spiders.rst import copy import warnings -import six - from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse from scrapy.linkextractors import LinkExtractor diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 2917daf57..d368c7108 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,6 +1,5 @@ import re import logging -import six from scrapy.spiders import Spider from scrapy.http import Request, XmlResponse diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index a0a47e473..16639356e 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -4,7 +4,6 @@ from shlex import split from http.cookies import SimpleCookie from urllib.parse import urlparse -from six import string_types, iteritems from w3lib.http import basic_auth_header @@ -76,7 +75,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): name = name.strip() val = val.strip() if name.title() == 'Cookie': - for name, morsel in iteritems(SimpleCookie(val)): + for name, morsel in SimpleCookie(val).items(): cookies[name] = morsel.value else: headers.append((name, val)) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 10481fe8d..3c0cb68c3 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,13 +1,13 @@ -import re import csv -from io import StringIO import logging -import six +import re +from io import StringIO from scrapy.http import TextResponse, Response from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode + logger = logging.getLogger(__name__) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 9a44f3576..0de3f18b3 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -6,7 +6,6 @@ from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules -import six from w3lib.html import replace_entities from scrapy.utils.python import flatten, to_unicode diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 69d906fbf..50763d8f7 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -85,9 +85,6 @@ class HeadersTest(unittest.TestCase): self.assertSortedEqual(h.items(), [(b'X-Forwarded-For', [b'ip1', b'ip2']), (b'Content-Type', [b'text/html'])]) - self.assertSortedEqual(h.iteritems(), - [(b'X-Forwarded-For', [b'ip1', b'ip2']), - (b'Content-Type', [b'text/html'])]) self.assertSortedEqual(h.values(), [b'ip2', b'text/html']) def test_update(self): diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 62d2847d7..988c8a811 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -62,7 +62,7 @@ class RequestTest(unittest.TestCase): # headers must not be unicode h = Headers({'key1': u'val1', u'key2': 'val2'}) h[u'newkey'] = u'newval' - for k, v in h.iteritems(): + for k, v in h.items(): self.assertIsInstance(k, bytes) for s in v: self.assertIsInstance(s, bytes) From ac62524824c590e9dd2d323bf418f71a683ecbf4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 3 Nov 2019 01:00:54 -0300 Subject: [PATCH 33/85] Remove six.get_method_* --- scrapy/core/downloader/middleware.py | 6 +++--- scrapy/core/spidermw.py | 4 ++-- scrapy/utils/reqser.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 7a6a4dfac..72432558a 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -38,7 +38,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield method(request=request, spider=spider) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, response.__class__.__name__)) + (method.__self__.__class__.__name__, response.__class__.__name__)) if response: defer.returnValue(response) defer.returnValue((yield download_func(request=request, spider=spider))) @@ -53,7 +53,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield method(request=request, response=response, spider=spider) if not isinstance(response, (Response, Request)): raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, type(response))) + (method.__self__.__class__.__name__, type(response))) if isinstance(response, Request): defer.returnValue(response) defer.returnValue(response) @@ -65,7 +65,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield method(request=request, exception=exception, spider=spider) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, type(response))) + (method.__self__.__class__.__name__, type(response))) if response: defer.returnValue(response) defer.returnValue(_failure) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index b5f9837ff..57436d2f6 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -37,8 +37,8 @@ class SpiderMiddlewareManager(MiddlewareManager): def scrape_response(self, scrape_func, response, request, spider): fname = lambda f:'%s.%s' % ( - six.get_method_self(f).__class__.__name__, - six.get_method_function(f).__name__) + f.__self__.__class__.__name__, + f.__func__.__name__) def process_spider_input(response): for method in self.methods['process_spider_input']: diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 495564ac0..e961ffca9 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -87,12 +87,12 @@ def _mangle_private_name(obj, func, name): def _find_method(obj, func): if obj: try: - func_self = six.get_method_self(func) + func_self = func.__self__ except AttributeError: # func has no __self__ pass else: if func_self is obj: - name = six.get_method_function(func).__name__ + name = func.__func__.__name__ if _is_private_method(name): return _mangle_private_name(obj, func, name) return name From 5d8abdde59e7501e8bda25b27c8926fda66b9af1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 3 Nov 2019 01:01:10 -0300 Subject: [PATCH 34/85] Remove six.text_type from tests --- tests/test_exporters.py | 2 +- tests/test_http_response.py | 8 ++++---- tests/test_loader.py | 14 +++++++------- tests/test_logformatter.py | 4 ++-- tests/test_toplevel.py | 2 +- tests/test_utils_iterators.py | 4 ++-- tests/test_utils_python.py | 14 +++++++------- 7 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index f151a1285..8433fa4db 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -79,7 +79,7 @@ class BaseItemExporterTest(unittest.TestCase): ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') _, name = list(ie._get_serialized_fields(self.i))[0] - assert isinstance(name, six.text_type) + assert isinstance(name, str) self.assertEqual(name, u'John\xa3') def test_field_custom_serializer(self): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 80bf51647..1d121cc83 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -102,7 +102,7 @@ class BaseResponseTest(unittest.TestCase): self.assertEqual(r4.flags, []) def _assert_response_values(self, response, encoding, body): - if isinstance(body, six.text_type): + if isinstance(body, str): body_unicode = body body_bytes = body.encode(encoding) else: @@ -110,7 +110,7 @@ class BaseResponseTest(unittest.TestCase): body_bytes = body assert isinstance(response.body, bytes) - assert isinstance(response.text, six.text_type) + assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_bytes) self.assertEqual(response.body_as_unicode(), body_unicode) @@ -220,11 +220,11 @@ class TextResponseTest(BaseResponseTest): r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') # check body_as_unicode - self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type)) + self.assertTrue(isinstance(r1.body_as_unicode(), str)) self.assertEqual(r1.body_as_unicode(), unicode_string) # check response.text - self.assertTrue(isinstance(r1.text, six.text_type)) + self.assertTrue(isinstance(r1.text, str)) self.assertEqual(r1.text, unicode_string) def test_encoding(self): diff --git a/tests/test_loader.py b/tests/test_loader.py index 4a4264a2a..f1cf0114f 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -157,7 +157,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_get_value(self): il = NameItemLoader() - self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), six.text_type.upper)) + self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper)) self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$')) self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')) @@ -258,7 +258,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_extend_custom_input_processors(self): class ChildItemLoader(TestItemLoader): - name_in = MapCompose(TestItemLoader.name_in, six.text_type.swapcase) + name_in = MapCompose(TestItemLoader.name_in, str.swapcase) il = ChildItemLoader() il.add_value('name', u'marta') @@ -266,7 +266,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_extend_default_input_processors(self): class ChildDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose(DefaultedItemLoader.default_input_processor, six.text_type.swapcase) + name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase) il = ChildDefaultedItemLoader() il.add_value('name', u'marta') @@ -689,7 +689,7 @@ class ProcessorsTest(unittest.TestCase): self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) self.assertEqual(proc(['', 'hello', 'world']), u' hello world') self.assertEqual(proc(['hello', 'world']), u'hello world') - self.assertIsInstance(proc(['hello', 'world']), six.text_type) + self.assertIsInstance(proc(['hello', 'world']), str) def test_compose(self): proc = Compose(lambda v: v[0], str.upper) @@ -704,12 +704,12 @@ class ProcessorsTest(unittest.TestCase): def test_mapcompose(self): def filter_world(x): return None if x == 'world' else x - proc = MapCompose(filter_world, six.text_type.upper) + proc = MapCompose(filter_world, str.upper) self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']), [u'HELLO', u'THIS', u'IS', u'SCRAPY']) - proc = MapCompose(filter_world, six.text_type.upper) + proc = MapCompose(filter_world, str.upper) self.assertEqual(proc(None), []) - proc = MapCompose(filter_world, six.text_type.upper) + proc = MapCompose(filter_world, str.upper) self.assertRaises(ValueError, proc, [1]) proc = MapCompose(filter_world, lambda x: x + 1) self.assertRaises(ValueError, proc, 'hello') diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index eb9c4a561..ca90de9d2 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -59,7 +59,7 @@ class LoggingContribTest(unittest.TestCase): logkws = self.formatter.dropped(item, exception, response, self.spider) logline = logkws['msg'] % logkws['args'] lines = logline.splitlines() - assert all(isinstance(x, six.text_type) for x in lines) + assert all(isinstance(x, str) for x in lines) self.assertEqual(lines, [u"Dropped: \u2018", '{}']) def test_scraped(self): @@ -69,7 +69,7 @@ class LoggingContribTest(unittest.TestCase): logkws = self.formatter.scraped(item, response, self.spider) logline = logkws['msg'] % logkws['args'] lines = logline.splitlines() - assert all(isinstance(x, six.text_type) for x in lines) + assert all(isinstance(x, str) for x in lines) self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3']) diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index 91bbe43bc..6d305249a 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -6,7 +6,7 @@ import scrapy class ToplevelTestCase(TestCase): def test_version(self): - self.assertIs(type(scrapy.__version__), six.text_type) + self.assertIs(type(scrapy.__version__), str) def test_version_info(self): self.assertIs(type(scrapy.version_info), tuple) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 2d845697e..0910bd560 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -255,8 +255,8 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assertTrue(all((isinstance(k, six.text_type) for k in result_row.keys()))) - self.assertTrue(all((isinstance(v, six.text_type) for v in result_row.values()))) + self.assertTrue(all((isinstance(k, str) for k in result_row.keys()))) + self.assertTrue(all((isinstance(v, str) for v in result_row.values()))) def test_csviter_delimiter(self): body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 096aa50b7..bea431969 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -169,8 +169,8 @@ class UtilsPythonTestCase(unittest.TestCase): d2 = stringify_dict(d, keys_only=False) self.assertEqual(d, d2) self.assertIsNot(d, d2) # shouldn't modify in place - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) + self.assertFalse(any(isinstance(x, str) for x in d2.keys())) + self.assertFalse(any(isinstance(x, str) for x in d2.values())) @unittest.skipUnless(six.PY2, "deprecated function") def test_stringify_dict_tuples(self): @@ -179,8 +179,8 @@ class UtilsPythonTestCase(unittest.TestCase): d2 = stringify_dict(tuples, keys_only=False) self.assertEqual(d, d2) self.assertIsNot(d, d2) # shouldn't modify in place - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys()) - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) + self.assertFalse(any(isinstance(x, str) for x in d2.keys()), d2.keys()) + self.assertFalse(any(isinstance(x, str) for x in d2.values())) @unittest.skipUnless(six.PY2, "deprecated function") def test_stringify_dict_keys_only(self): @@ -188,7 +188,7 @@ class UtilsPythonTestCase(unittest.TestCase): d2 = stringify_dict(d) self.assertEqual(d, d2) self.assertIsNot(d, d2) # shouldn't modify in place - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) + self.assertFalse(any(isinstance(x, str) for x in d2.keys())) def test_get_func_args(self): def f1(a, b, c): @@ -227,12 +227,12 @@ class UtilsPythonTestCase(unittest.TestCase): if platform.python_implementation() == 'CPython': # TODO: how do we fix this to return the actual argument names? - self.assertEqual(get_func_args(six.text_type.split), []) + self.assertEqual(get_func_args(str.split), []) self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), []) else: self.assertEqual( - get_func_args(six.text_type.split, True), ['sep', 'maxsplit']) + get_func_args(str.split, True), ['sep', 'maxsplit']) self.assertEqual(get_func_args(" ".join, True), ['list']) self.assertEqual( get_func_args(operator.itemgetter(2), True), ['obj']) From eaeaa40b991bcb2113ba63146fbe4b2dc9e93016 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 3 Nov 2019 01:08:08 -0300 Subject: [PATCH 35/85] Remove six.PY* checks --- conftest.py | 12 +++++------- tests/test_webclient.py | 20 -------------------- 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/conftest.py b/conftest.py index 06d65ba1d..5e6a42977 100644 --- a/conftest.py +++ b/conftest.py @@ -1,4 +1,3 @@ -import six import pytest @@ -7,12 +6,11 @@ collect_ignore = [ "scrapy/utils/testsite.py", ] - -if six.PY3: - for line in open('tests/py3-ignores.txt'): - file_path = line.strip() - if file_path and file_path[0] != '#': - collect_ignore.append(file_path) +# FIXME: fix or delete these tests +for line in open('tests/py3-ignores.txt'): + file_path = line.strip() + if file_path and file_path[0] != '#': + collect_ignore.append(file_path) @pytest.fixture() diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 0c04e7114..608cfe597 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -78,26 +78,6 @@ class ParseUrlTestCase(unittest.TestCase): to_bytes(x) if not isinstance(x, int) else x for x in test) self.assertEqual(client._parse(url), test, url) - def test_externalUnicodeInterference(self): - """ - L{client._parse} should return C{str} for the scheme, host, and path - elements of its return tuple, even when passed an URL which has - previously been passed to L{urlparse} as a C{unicode} string. - """ - if not six.PY2: - raise unittest.SkipTest( - "Applies only to Py2, as urls can be ONLY unicode on Py3") - badInput = u'http://example.com/path' - goodInput = badInput.encode('ascii') - self._parse(badInput) # cache badInput in urlparse_cached - scheme, netloc, host, port, path = self._parse(goodInput) - self.assertTrue(isinstance(scheme, str)) - self.assertTrue(isinstance(netloc, str)) - self.assertTrue(isinstance(host, str)) - self.assertTrue(isinstance(path, str)) - self.assertTrue(isinstance(port, int)) - - class ScrapyHTTPPageGetterTests(unittest.TestCase): From d72444b9c8db94dd5567e763ee53b2c70a423c9a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 3 Nov 2019 01:11:23 -0300 Subject: [PATCH 36/85] Remove more six imports --- scrapy/core/downloader/middleware.py | 2 -- scrapy/core/spidermw.py | 1 - scrapy/utils/reqser.py | 2 -- tests/test_exporters.py | 1 - tests/test_http_response.py | 1 - tests/test_loader.py | 2 -- tests/test_logformatter.py | 1 - tests/test_toplevel.py | 2 +- tests/test_utils_iterators.py | 3 ++- tests/test_utils_reqser.py | 2 -- tests/test_utils_url.py | 2 -- tests/test_webclient.py | 1 - 12 files changed, 3 insertions(+), 17 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 72432558a..38608a429 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,8 +3,6 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -import six - from twisted.internet import defer from scrapy.exceptions import _InvalidOutput diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 57436d2f6..e4c6df8c7 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -5,7 +5,6 @@ See documentation in docs/topics/spider-middleware.rst """ from itertools import chain, islice -import six from twisted.python.failure import Failure from scrapy.exceptions import _InvalidOutput from scrapy.middleware import MiddlewareManager diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index e961ffca9..749bbc387 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -1,8 +1,6 @@ """ Helper functions for serializing (and deserializing) requests. """ -import six - from scrapy.http import Request from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 8433fa4db..5d1f5c182 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -8,7 +8,6 @@ from io import BytesIO from datetime import datetime import lxml.etree -import six from scrapy.item import Item, Field from scrapy.utils.python import to_unicode diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 1d121cc83..ee7177ceb 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import unittest -import six from w3lib.encoding import resolve_encoding from scrapy.http import (Request, Response, TextResponse, HtmlResponse, diff --git a/tests/test_loader.py b/tests/test_loader.py index f1cf0114f..69ded4d50 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,8 +1,6 @@ from functools import partial import unittest -import six - from scrapy.http import HtmlResponse from scrapy.item import Item, Field from scrapy.loader import ItemLoader diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index ca90de9d2..5dc077c5b 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -3,7 +3,6 @@ import unittest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase as TwistedTestCase -import six from scrapy.crawler import CrawlerRunner from scrapy.exceptions import DropItem diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index 6d305249a..fdc5df166 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -1,5 +1,5 @@ from unittest import TestCase -import six + import scrapy diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 0910bd560..4d69edb31 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- import os -import six + from twisted.trial import unittest from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml from scrapy.http import XmlResponse, TextResponse, Response from tests import get_testdata + FOOBAR_NL = u"foo\nbar" diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 92cd16de7..b5729b086 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -2,8 +2,6 @@ import unittest import sys -import six - from scrapy.http import Request, FormRequest from scrapy.spiders import Spider from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method, _mangle_private_name diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index a8e37d7b8..c5fdc752b 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -2,8 +2,6 @@ import unittest from urllib.parse import urlparse -import six - from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, add_http_if_no_scheme, guess_scheme, diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 608cfe597..746367b41 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -3,7 +3,6 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ import os -import six import shutil import OpenSSL.SSL From e461570f991cf70d17a784aa628496521c241873 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sat, 2 Nov 2019 23:13:54 -0300 Subject: [PATCH 37/85] Remove protego from requirements file --- requirements-py3.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 2c98e6f6d..cd183a525 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -2,7 +2,6 @@ parsel>=1.5.0 PyDispatcher>=2.0.5 Twisted>=17.9.0 w3lib>=1.17.0 -protego>=0.1.15 pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" From 7f3cb05d8e20048eb3e37c5e83bcf495d28c6064 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 3 Nov 2019 12:03:02 -0300 Subject: [PATCH 38/85] Remove metaclass-related six code --- scrapy/item.py | 5 +---- tests/test_item.py | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index 4e0f0ac44..1d39b48b2 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -10,8 +10,6 @@ from copy import deepcopy from pprint import pformat from warnings import warn -import six - from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref @@ -130,6 +128,5 @@ class DictItem(MutableMapping, BaseItem): return deepcopy(self) -@six.add_metaclass(ItemMeta) -class Item(DictItem): +class Item(DictItem, metaclass=ItemMeta): pass diff --git a/tests/test_item.py b/tests/test_item.py index d98c63ddd..0da8fa1ac 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -3,8 +3,6 @@ import unittest from unittest import mock from warnings import catch_warnings -import six - from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta @@ -302,7 +300,7 @@ class ItemMetaTest(unittest.TestCase): class ItemMetaClassCellRegression(unittest.TestCase): def test_item_meta_classcell_regression(self): - class MyItem(six.with_metaclass(ItemMeta, Item)): + class MyItem(Item, metaclass=ItemMeta): def __init__(self, *args, **kwargs): # This call to super() trigger the __classcell__ propagation # requirement. When not done properly raises an error: From 586b25d27e1641433d505338901fa9a1409ccdd4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 3 Nov 2019 12:10:37 -0300 Subject: [PATCH 39/85] Remove six types --- scrapy/downloadermiddlewares/ajaxcrawl.py | 3 +-- scrapy/utils/python.py | 3 +-- tests/test_utils_trackref.py | 7 ++++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 78b802673..c618e9ffc 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -2,7 +2,6 @@ import re import logging -import six from w3lib import html from scrapy.exceptions import NotConfigured @@ -66,7 +65,7 @@ class AjaxCrawlMiddleware(object): # XXX: move it to w3lib? -_ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')) +_ajax_crawlable_re = re.compile(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>') def _has_ajaxcrawlable_meta(text): """ >>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment" content="!"/></head><body></body></html>') diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 18fee1964..d32ee5a3a 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -7,7 +7,6 @@ import re import inspect import weakref import errno -import six from functools import partial, wraps from itertools import chain import sys @@ -162,7 +161,7 @@ def memoizemethod_noargs(method): return new_method -_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"} +_BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"} _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS} @deprecated("scrapy.utils.python.binary_is_text") diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 480a717e7..16e02f919 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,6 +1,7 @@ -import six import unittest +from io import StringIO from unittest import mock + from scrapy.utils import trackref @@ -38,12 +39,12 @@ Live References Bar 1 oldest: 0s ago ''') - @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_print_live_refs_empty(self, stdout): trackref.print_live_refs() self.assertEqual(stdout.getvalue(), 'Live References\n\n\n') - @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_print_live_refs_with_objects(self, stdout): o1 = Foo() # NOQA trackref.print_live_refs() From 5797aefd4c561666c2c047494b7424d77eebb469 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 3 Nov 2019 12:18:35 -0300 Subject: [PATCH 40/85] Remove six.assertCountEqual --- tests/test_cmdline/__init__.py | 7 +++---- tests/test_settings/__init__.py | 14 ++++++-------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 56cfe642a..909ea90e0 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -1,13 +1,12 @@ -from io import StringIO import json import os import pstats import shutil -import six -from subprocess import Popen, PIPE import sys import tempfile import unittest +from io import StringIO +from subprocess import Popen, PIPE from scrapy.utils.test import get_testenv @@ -65,5 +64,5 @@ class CmdlineTest(unittest.TestCase): for char in ("'", "<", ">", 'u"'): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) - six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys()) + self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) self.assertEqual(200, settingsdict[EXT_PATH]) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index d5cbef6f5..fda44653a 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,4 +1,3 @@ -import six import unittest from unittest import mock @@ -43,14 +42,14 @@ class SettingsAttributeTest(unittest.TestCase): new_dict = {'three': 11, 'four': 21} attribute.set(new_dict, 10) self.assertIsInstance(attribute.value, BaseSettings) - six.assertCountEqual(self, attribute.value, new_dict) - six.assertCountEqual(self, original_settings, original_dict) + self.assertCountEqual(attribute.value, new_dict) + self.assertCountEqual(original_settings, original_dict) new_settings = BaseSettings({'five': 12}, 0) attribute.set(new_settings, 0) # Insufficient priority - six.assertCountEqual(self, attribute.value, new_dict) + self.assertCountEqual(attribute.value, new_dict) attribute.set(new_settings, 10) - six.assertCountEqual(self, attribute.value, new_settings) + self.assertCountEqual(attribute.value, new_settings) def test_repr(self): self.assertEqual(repr(self.attribute), @@ -276,9 +275,8 @@ class BaseSettingsTest(unittest.TestCase): 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) s['TEST'].set(2, 200, 'cmdline') - six.assertCountEqual(self, s.getwithbase('TEST'), - {1: 1, 2: 200, 3: 30}) - six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE']) + self.assertCountEqual(s.getwithbase('TEST'), {1: 1, 2: 200, 3: 30}) + self.assertCountEqual(s.getwithbase('HASNOBASE'), s['HASNOBASE']) self.assertEqual(s.getwithbase('NONEXISTENT'), {}) def test_maxpriority(self): From 00b793dc59a32d13c89ac5c0d54985677b228801 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 3 Nov 2019 12:26:38 -0300 Subject: [PATCH 41/85] Remove elluding six occurrences --- scrapy/robotstxt.py | 6 ++++-- tests/test_contracts.py | 5 ++--- tests/test_pipeline_files.py | 2 +- tests/test_utils_curl.py | 7 ++----- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 397924110..0a9af3a62 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -1,12 +1,13 @@ import sys import logging from abc import ABCMeta, abstractmethod -from six import with_metaclass from scrapy.utils.python import to_unicode + logger = logging.getLogger(__name__) + def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): try: if to_native_str_type: @@ -23,7 +24,8 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): robotstxt_body = '' return robotstxt_body -class RobotParser(with_metaclass(ABCMeta)): + +class RobotParser(metaclass=ABCMeta): @classmethod @abstractmethod def from_crawler(cls, crawler, robotstxt_body): diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b2e358700..582e3d052 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,6 +1,5 @@ from unittest import TextTestResult -from six import get_unbound_function from twisted.internet import defer from twisted.python import failure from twisted.trial import unittest @@ -395,8 +394,8 @@ class ContractsManagerTest(unittest.TestCase): with MockServer() as mockserver: contract_doc = '@url {}'.format(mockserver.url('/status?n=200')) - get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc - get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc + TestSameUrlSpider.parse_first.__doc__ = contract_doc + TestSameUrlSpider.parse_second.__doc__ = contract_doc crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) yield crawler.crawl() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index dede4bf12..52f2b554e 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,12 +1,12 @@ import os import random import time +from io import BytesIO from tempfile import mkdtemp from shutil import rmtree from unittest import mock from urllib.parse import urlparse -from six import BytesIO from twisted.trial import unittest from twisted.internet import defer diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index c5655df7e..50e1bfd5f 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,7 +1,6 @@ import unittest import warnings -from six import assertRaisesRegex from w3lib.http import basic_auth_header from scrapy import Request @@ -177,8 +176,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) def test_too_few_arguments_error(self): - assertRaisesRegex( - self, + self.assertRaisesRegex( ValueError, r"too few arguments|the following arguments are required:\s*url", lambda: curl_to_request_kwargs("curl"), @@ -194,8 +192,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) # case 2: ignore_unknown_options=False (raise exception): - assertRaisesRegex( - self, + self.assertRaisesRegex( ValueError, "Unrecognized options:.*--bar.*--baz", lambda: curl_to_request_kwargs( From 0c4e5b68ea0e077033fde5da28c6dc4fdd859d92 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Sun, 3 Nov 2019 12:30:34 -0300 Subject: [PATCH 42/85] Remove six from requirements and setup files --- requirements-py3.txt | 1 - setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index cd183a525..28c649e28 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -13,5 +13,4 @@ cryptography>=2.0 # Earlier versions would fail to install cssselect>=0.9.1 lxml>=3.5.0 service_identity>=16.0.0 -six>=1.10.0 zope.interface>=4.1.3 diff --git a/setup.py b/setup.py index 8f5f14f0d..85d797f88 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,6 @@ setup( 'pyOpenSSL>=16.2.0', 'queuelib>=1.4.2', 'service_identity>=16.0.0', - 'six>=1.10.0', 'w3lib>=1.17.0', 'zope.interface>=4.1.3', 'protego>=0.1.15', From fe31695ba0266deaa94222fc01885b7270af4294 Mon Sep 17 00:00:00 2001 From: elacuesta <elacuesta@users.noreply.github.com> Date: Tue, 5 Nov 2019 15:36:19 -0300 Subject: [PATCH 43/85] Remove unused import (urllib.parse.unquote) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves <adrian@chaves.io> --- tests/test_http_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 988c8a811..05cac617c 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -6,7 +6,7 @@ import json import xmlrpc.client as xmlrpclib import warnings from unittest import mock -from urllib.parse import parse_qs, unquote, unquote_to_bytes, urlparse +from urllib.parse import parse_qs, unquote_to_bytes, urlparse from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_unicode From 44f19df3119d553aa5c002321bd901a424c4bb2c Mon Sep 17 00:00:00 2001 From: elacuesta <elacuesta@users.noreply.github.com> Date: Fri, 8 Nov 2019 11:32:50 -0300 Subject: [PATCH 44/85] [test] Update mitmproxy version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves <adrian@chaves.io> --- tests/requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index c2b16bec6..7abb66b9c 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,7 +1,7 @@ # Tests requirements jmespath mitmproxy; python_version >= '3.6' -mitmproxy==3.0.4; python_version < '3.6' +mitmproxy<4.0.0; python_version < '3.6' pytest pytest-cov pytest-twisted From 6cde428af43a5c8268208c6e4e239ab5ce507af4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Fri, 8 Nov 2019 12:26:40 -0300 Subject: [PATCH 45/85] Remove deprecated MergeDict class --- scrapy/utils/datatypes.py | 59 --------------------------------------- 1 file changed, 59 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 56d4d1b8e..e194a7613 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -235,65 +235,6 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class MergeDict(object): - """ - A simple class for creating new "virtual" dictionaries that actually look - up values in more than one dictionary, passed in the constructor. - - If a key appears in more than one of the given dictionaries, only the - first occurrence will be used. - """ - def __init__(self, *dicts): - warnings.warn( - "scrapy.utils.datatypes.MergeDict is deprecated in favor " - "of collections.ChainMap (introduced in Python 3.3)", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - self.dicts = dicts - - def __getitem__(self, key): - for dict_ in self.dicts: - try: - return dict_[key] - except KeyError: - pass - raise KeyError - - def __copy__(self): - return self.__class__(*self.dicts) - - def get(self, key, default=None): - try: - return self[key] - except KeyError: - return default - - def getlist(self, key): - for dict_ in self.dicts: - if key in dict_.keys(): - return dict_.getlist(key) - return [] - - def items(self): - item_list = [] - for dict_ in self.dicts: - item_list.extend(dict_.items()) - return item_list - - def has_key(self, key): - for dict_ in self.dicts: - if key in dict_: - return True - return False - - __contains__ = has_key - - def copy(self): - """Returns a copy of this object.""" - return self.__copy__() - - class LocalCache(collections.OrderedDict): """Dictionary with a finite number of keys. From b6bbb2819707a2202c87abd6b3dba6af13a7cc85 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Fri, 8 Nov 2019 22:13:03 -0300 Subject: [PATCH 46/85] PEP8 adjustments --- scrapy/crawler.py | 1 - scrapy/exporters.py | 1 - scrapy/http/cookies.py | 2 +- scrapy/link.py | 2 ++ tests/test_pipeline_media.py | 2 -- tests/test_proxy_connect.py | 53 ++++++++++++++++++++---------------- tests/test_utils_python.py | 2 +- 7 files changed, 34 insertions(+), 29 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 4d7d9bac4..ab62c678c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,6 +1,5 @@ import logging import signal -import sys import warnings from twisted.internet import reactor, defer diff --git a/scrapy/exporters.py b/scrapy/exporters.py index f2999daea..e31ab1780 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -4,7 +4,6 @@ Item Exporters are used to export/serialize items into different formats. import csv import io -import sys import pprint import marshal import warnings diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index c39de0b52..0903fd4f8 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -165,7 +165,7 @@ class WrappedRequest(object): def get_header(self, name, default=None): return to_unicode(self.request.headers.get(name, default), - errors='replace') + errors='replace') def header_items(self): return [ diff --git a/scrapy/link.py b/scrapy/link.py index be1888ef0..a809c5ca4 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,6 +4,8 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ + + class Link(object): """Link objects represent an extracted link by the LinkExtractor.""" diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 70f11466b..0d23f51cc 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,5 +1,3 @@ -import sys - from testfixtures import LogCapture from twisted.trial import unittest from twisted.python.failure import Failure diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index bf56136b1..4147dc944 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -28,17 +28,24 @@ from mitmproxy.tools.main import mitmdump sys.argv[0] = "mitmdump" sys.exit(mitmdump()) """ - cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'keys', 'mitmproxy-ca.pem') - self.proc = Popen([sys.executable, - '-c', script, - '--listen-host', '127.0.0.1', - '--listen-port', '0', - '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass), - '--certs', cert_path, - '--ssl-insecure', - ], - stdout=PIPE, env=get_testenv()) + cert_path = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + 'keys', + 'mitmproxy-ca.pem' + ) + self.proc = Popen( + [ + sys.executable, + '-c', script, + '--listen-host', '127.0.0.1', + '--listen-port', '0', + '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass), + '--certs', cert_path, + '--ssl-insecure', + ], + stdout=PIPE, + env=get_testenv() + ) line = self.proc.stdout.readline().decode('utf-8') host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1) address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port) @@ -75,9 +82,9 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_connect_tunnel(self): crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as logs: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) + self._assert_got_response_code(200, logs) @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') @defer.inlineCallbacks @@ -85,35 +92,35 @@ class ProxyConnectTestCase(TestCase): proxy = os.environ['https_proxy'] os.environ['https_proxy'] = proxy + '?noconnect' crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as logs: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) + self._assert_got_response_code(200, logs) @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6) @defer.inlineCallbacks def test_https_connect_tunnel_error(self): crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as logs: yield crawler.crawl("https://localhost:99999/status?n=200") - self._assert_got_tunnel_error(l) + self._assert_got_tunnel_error(logs) @defer.inlineCallbacks def test_https_tunnel_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as logs: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) # The proxy returns a 407 error code but it does not reach the client; # he just sees a TunnelError. - self._assert_got_tunnel_error(l) + self._assert_got_tunnel_error(logs) @defer.inlineCallbacks def test_https_tunnel_without_leak_proxy_authorization_header(self): request = Request(self.mockserver.url("/echo", is_secure=True)) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as l: + with LogCapture() as logs: yield crawler.crawl(seed=request) - self._assert_got_response_code(200, l) + self._assert_got_response_code(200, logs) echo = json.loads(crawler.spider.meta['responses'][0].text) self.assertTrue('Proxy-Authorization' not in echo['headers']) @@ -122,9 +129,9 @@ class ProxyConnectTestCase(TestCase): def test_https_noconnect_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as logs: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(407, l) + self._assert_got_response_code(407, logs) def _assert_got_response_code(self, code, log): print(log) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index faf0d4b73..b36c2a5e3 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -7,7 +7,7 @@ from itertools import count from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode, + WeakKeyCache, get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) From 058bdda0afe967f99a3ffd59b395566b063d9c3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 14 Nov 2019 16:51:47 +0100 Subject: [PATCH 47/85] Improve the performance of the DOWNLOAD_DELAY test --- tests/test_crawl.py | 51 +++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3fc13eeb7..a524287eb 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -30,25 +30,44 @@ class CrawlTestCase(TestCase): self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url @defer.inlineCallbacks - def test_delay(self): - # short to long delays - yield self._test_delay(0.2, False) - yield self._test_delay(1, False) - # randoms - yield self._test_delay(0.2, True) - yield self._test_delay(1, True) + def test_fixed_delay(self): + yield self._test_delay(total=3, delay=0.1) @defer.inlineCallbacks - def _test_delay(self, delay, randomize): - settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize} + def test_randomized_delay(self): + yield self._test_delay(total=3, delay=0.1, randomize=True) + + @defer.inlineCallbacks + def _test_delay(self, total, delay, randomize=False): + crawl_kwargs = dict( + maxlatency=delay * 2, + mockserver=self.mockserver, + total=total, + ) + tolerance = (1 - (0.6 if randomize else 0.2)) + + settings = {"DOWNLOAD_DELAY": delay, + 'RANDOMIZE_DOWNLOAD_DELAY': randomize} crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider) - yield crawler.crawl(maxlatency=delay * 2, mockserver=self.mockserver) - t = crawler.spider.times - totaltime = t[-1] - t[0] - avgd = totaltime / (len(t) - 1) - tolerance = 0.6 if randomize else 0.2 - self.assertTrue(avgd > delay * (1 - tolerance), - "download delay too small: %s" % avgd) + yield crawler.crawl(**crawl_kwargs) + times = crawler.spider.times + total_time = times[-1] - times[0] + average = total_time / (len(times) - 1) + self.assertTrue(average > delay * tolerance, + "download delay too small: %s" % average) + + # Ensure that the same test parameters would cause a failure if no + # download delay is set. Otherwise, it means we are using a combination + # of ``total`` and ``delay`` values that are too small for the test + # code above to have any meaning. + settings["DOWNLOAD_DELAY"] = 0 + crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider) + yield crawler.crawl(**crawl_kwargs) + times = crawler.spider.times + total_time = times[-1] - times[0] + average = total_time / (len(times) - 1) + self.assertFalse(average > delay / tolerance, + "test total or delay values are too small") @defer.inlineCallbacks def test_timeout_success(self): From 6d1667d5b8086dd12539a185680f9e119c5fdc55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 7 Nov 2019 18:32:33 +0100 Subject: [PATCH 48/85] Use the latest Python version to build the documentation --- .travis.yml | 2 +- tox.ini | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9f477e860..c9c64e990 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ matrix: - env: TOXENV=py38-extra-deps python: 3.8 - env: TOXENV=docs - python: 3.6 + python: 3.8 install: - | if [ "$TOXENV" = "pypy3" ]; then diff --git a/tox.ini b/tox.ini index fd75d18e2..195cc106a 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,9 @@ [tox] envlist = py35 +[latest] +basepython = python3.8 + [testenv] deps = -ctests/constraints.txt @@ -63,14 +66,14 @@ commands = py.test {posargs:--durations=10 docs scrapy tests} [testenv:security] -basepython = python3.8 +basepython = {[latest]basepython} deps = bandit commands = bandit -r -c .bandit.yml {posargs:scrapy} [testenv:flake8] -basepython = python3.8 +basepython = {[latest]basepython} deps = {[testenv]deps} pytest-flake8 @@ -83,18 +86,21 @@ deps = -rdocs/requirements.txt [testenv:docs] +basepython = {[latest]basepython} changedir = {[docs]changedir} deps = {[docs]deps} commands = sphinx-build -W -b html . {envtmpdir}/html [testenv:docs-coverage] +basepython = {[latest]basepython} changedir = {[docs]changedir} deps = {[docs]deps} commands = sphinx-build -b coverage . {envtmpdir}/coverage [testenv:docs-links] +basepython = {[latest]basepython} changedir = {[docs]changedir} deps = {[docs]deps} commands = From e1af85619f63fe9312920f0dea7454bb76fcc23f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 18 Nov 2019 11:06:25 +0100 Subject: [PATCH 49/85] Add a configuration file for Read the Docs --- .readthedocs.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 000000000..3c1c3e8be --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,7 @@ +version: 2 +sphinx: + configuration: docs/conf.py +python: + version: 3.8 + install: + - requirements: docs/requirements.txt From fc3af54dbd5b7fdcbc44c82ba4ced528a104cd88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 20 Nov 2019 07:57:09 +0100 Subject: [PATCH 50/85] Make tox configuration more user friendly --- .travis.yml | 5 ++-- docs/contributing.rst | 14 ++-------- tox.ini | 64 ++++++++++++++++++------------------------- 3 files changed, 31 insertions(+), 52 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9f477e860..4e28d6f11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,9 @@ matrix: - env: TOXENV=flake8 python: 3.8 - env: TOXENV=pypy3 - python: 3.5 - env: TOXENV=py35 python: 3.5 - - env: TOXENV=py35-pinned + - env: TOXENV=pinned python: 3.5 - env: TOXENV=py36 python: 3.6 @@ -23,7 +22,7 @@ matrix: python: 3.7 - env: TOXENV=py38 python: 3.8 - - env: TOXENV=py38-extra-deps + - env: TOXENV=extra-deps python: 3.8 - env: TOXENV=docs python: 3.6 diff --git a/docs/contributing.rst b/docs/contributing.rst index f084bd23d..c4cb605ab 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -202,17 +202,9 @@ tests requires `tox`_. Running tests ------------- -Make sure you have a recent enough `tox`_ installation: +To run all tests:: - ``tox --version`` - -If your version is older than 1.7.0, please update it first: - - ``pip install -U tox`` - -To run all tests go to the root directory of Scrapy source code and run: - - ``tox`` + tox To run a specific test (say ``tests/test_loader.py``) use: @@ -227,7 +219,7 @@ environment name from ``tox.ini``. For example, to run the tests with Python You can also specify a comma-separated list of environmets, and use `tox’s parallel mode`_ to run the tests on multiple environments in parallel:: - tox -e py27,py36 -p auto + tox -e py37,py38 -p auto To pass command-line options to pytest_, add them after ``--`` in your call to tox_. Using ``--`` overrides the default positional arguments defined in diff --git a/tox.ini b/tox.ini index fd75d18e2..0b41f6cc3 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,8 @@ # and then run "tox" from this directory. [tox] -envlist = py35 +envlist = security,flake8,py3 +minversion = 1.7.0 [testenv] deps = @@ -23,11 +24,28 @@ passenv = commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} -[testenv:py35] -basepython = python3.5 +[testenv:security] +basepython = python3 +deps = + bandit +commands = + bandit -r -c .bandit.yml {posargs:scrapy} -[testenv:py35-pinned] -basepython = python3.5 +[testenv:flake8] +basepython = python3 +deps = + {[testenv]deps} + pytest-flake8 +commands = + py.test --flake8 {posargs:docs scrapy tests} + +[testenv:pypy3] +basepython = pypy3 +commands = + py.test {posargs:--durations=10 docs scrapy tests} + +[testenv:pinned] +basepython = python3 deps = -ctests/constraints.txt cryptography==2.0 @@ -48,34 +66,11 @@ deps = botocore==1.3.23 Pillow==3.4.2 -[testenv:py36] -basepython = python3.6 - -[testenv:py37] -basepython = python3.7 - -[testenv:py38] -basepython = python3.8 - -[testenv:pypy3] -basepython = pypy3 -commands = - py.test {posargs:--durations=10 docs scrapy tests} - -[testenv:security] -basepython = python3.8 -deps = - bandit -commands = - bandit -r -c .bandit.yml {posargs:scrapy} - -[testenv:flake8] -basepython = python3.8 +[testenv:extra-deps] deps = {[testenv]deps} - pytest-flake8 -commands = - py.test --flake8 {posargs:docs scrapy tests} + reppy + robotexclusionrulesparser [docs] changedir = docs @@ -99,10 +94,3 @@ changedir = {[docs]changedir} deps = {[docs]deps} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck - -[testenv:py38-extra-deps] -basepython = python3.8 -deps = - {[testenv]deps} - reppy - robotexclusionrulesparser From 1718e450ef9549a4fc71b01dba1e6faf7a63238a Mon Sep 17 00:00:00 2001 From: Mabel Villalba <mabelvj@gmail.com> Date: Mon, 18 Nov 2019 12:33:55 +0100 Subject: [PATCH 51/85] [start_url] Fixes #4133: Raise AttributeError error when empty 'start_urls' and 'start_url' found. Added test. --- scrapy/spiders/__init__.py | 5 +++++ tests/test_spider.py | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index e9c131e3b..5a35fcdb6 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -68,6 +68,11 @@ class Spider(object_ref): def start_requests(self): cls = self.__class__ + if not self.start_urls and hasattr(self, 'start_url'): + raise AttributeError( + "Crawling could not start: 'start_urls' not found " + "or empty (but found 'start_url' attribute instead, " + "did you miss an 's'?)") if method_is_overridden(cls, Spider, 'make_requests_from_url'): warnings.warn( "Spider.make_requests_from_url method is deprecated; it " diff --git a/tests/test_spider.py b/tests/test_spider.py index 83fb68c2f..0a6640cec 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -391,6 +391,13 @@ class CrawlSpiderTest(SpiderTest): self.assertTrue(hasattr(spider, '_follow_links')) self.assertFalse(spider._follow_links) + def test_start_url(self): + spider = self.spider_class("example.com") + spider.start_url = 'https://www.example.com' + + with self.assertRaisesRegex(AttributeError, + r'^Crawling could not start.*$'): + list(spider.start_requests()) class SitemapSpiderTest(SpiderTest): From 55cc5c9068a7a39908f90e6403e6bd8c8ef11a4d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Fri, 22 Nov 2019 12:41:31 -0300 Subject: [PATCH 52/85] Skip pickle in bandit check --- .bandit.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.bandit.yml b/.bandit.yml index 00554587a..cc7db3a66 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,6 +1,7 @@ skips: - B101 - B105 +- B301 - B303 - B306 - B307 @@ -8,6 +9,7 @@ skips: - B320 - B321 - B402 +- B403 - B404 - B406 - B410 From 54b056c4be045099f9f872c5455c003d68b578a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Mon, 25 Nov 2019 12:13:31 +0100 Subject: [PATCH 53/85] Make developer-tools doctests pass --- docs/_tests/quotes.html | 281 ++++++++++++++++++++++++++++++++ docs/topics/developer-tools.rst | 40 +++-- pytest.ini | 1 - 3 files changed, 308 insertions(+), 14 deletions(-) create mode 100644 docs/_tests/quotes.html diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html new file mode 100644 index 000000000..71aff8847 --- /dev/null +++ b/docs/_tests/quotes.html @@ -0,0 +1,281 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <title>Quotes to Scrape + + + + +
+
+
+

+ Quotes to Scrape +

+
+
+

+ + Login + +

+
+
+ + +
+
+ +
+ “The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.” + by + (about) + +
+ Tags: + + + change + + deep-thoughts + + thinking + + world + +
+
+ +
+ “It is our choices, Harry, that show what we truly are, far more than our abilities.” + by + (about) + +
+ Tags: + + + abilities + + choices + +
+
+ +
+ “There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.” + by + (about) + +
+ Tags: + + + inspirational + + life + + live + + miracle + + miracles + +
+
+ +
+ “The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.” + by + (about) + +
+ Tags: + + + aliteracy + + books + + classic + + humor + +
+
+ +
+ “Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.” + by + (about) + +
+ Tags: + + + be-yourself + + inspirational + +
+
+ +
+ “Try not to become a man of success. Rather become a man of value.” + by + (about) + +
+ Tags: + + + adulthood + + success + + value + +
+
+ +
+ “It is better to be hated for what you are than to be loved for what you are not.” + by + (about) + +
+ Tags: + + + life + + love + +
+
+ +
+ “I have not failed. I've just found 10,000 ways that won't work.” + by + (about) + +
+ Tags: + + + edison + + failure + + inspirational + + paraphrased + +
+
+ +
+ “A woman is like a tea bag; you never know how strong it is until it's in hot water.” + by + (about) + +
+ Tags: + + + misattributed-eleanor-roosevelt + +
+
+ +
+ “A day without sunshine is like, you know, night.” + by + (about) + +
+ Tags: + + + humor + + obvious + + simile + +
+
+ + +
+
+ +

Top Ten tags

+ + + love + + + + inspirational + + + + life + + + + humor + + + + books + + + + reading + + + + friendship + + + + friends + + + + truth + + + + simile + + + +
+
+ +
+ + + \ No newline at end of file diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index bf14643be..e67ce55f9 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -39,7 +39,7 @@ Therefore, you should keep in mind the following things: .. _topics-inspector: Inspecting a website -=================================== +==================== By far the most handy feature of the Developer Tools is the `Inspector` feature, which allows you to inspect the underlying HTML code of @@ -79,13 +79,23 @@ sections and tags of a webpage, which greatly improves readability. You can expand and collapse a tag by clicking on the arrow in front of it or by double clicking directly on the tag. If we expand the ``span`` tag with the ``class= "text"`` we will see the quote-text we clicked on. The `Inspector` lets you -copy XPaths to selected elements. Let's try it out: Right-click on the ``span`` -tag, select ``Copy > XPath`` and paste it in the scrapy shell like so:: +copy XPaths to selected elements. Let's try it out. + +First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal: + +.. code-block:: none $ scrapy shell "http://quotes.toscrape.com/" - (...) - >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() - ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”] + +Then, back to your web browser, right-click on the ``span`` tag, select +``Copy > XPath`` and paste it in the Scrapy shell like so: + +.. invisible-code-block: python + + response = load_response('http://quotes.toscrape.com/', 'quotes.html') + +>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall() +['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'] Adding ``text()`` at the end we are able to extract the first quote with this basic selector. But this XPath is not really that clever. All it does is @@ -112,13 +122,13 @@ see each quote: With this knowledge we can refine our XPath: Instead of a path to follow, we'll simply select all ``span`` tags with the ``class="text"`` by using -the `has-class-extension`_:: +the `has-class-extension`_: - >>> response.xpath('//span[has-class("text")]/text()').getall() - ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”, - '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', - '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', - (...)] +>>> response.xpath('//span[has-class("text")]/text()').getall() +['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', +'“It is our choices, Harry, that show what we truly are, far more than our abilities.”', +'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', +...] And with one simple, cleverer XPath we are able to extract all quotes from the page. We could have constructed a loop over our first XPath to increase @@ -159,7 +169,11 @@ The page is quite similar to the basic `quotes.toscrape.com`_-page, but instead of the above-mentioned ``Next`` button, the page automatically loads new quotes when you scroll to the bottom. We could go ahead and try out different XPaths directly, but instead -we'll check another quite useful command from the scrapy shell:: +we'll check another quite useful command from the scrapy shell: + +.. skip: next + +.. code-block:: none $ scrapy shell "quotes.toscrape.com/scroll" (...) diff --git a/pytest.ini b/pytest.ini index 33c34b8e8..0e830076d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,7 +8,6 @@ addopts = --ignore=docs/_ext --ignore=docs/conf.py --ignore=docs/news.rst - --ignore=docs/topics/developer-tools.rst --ignore=docs/topics/dynamic-content.rst --ignore=docs/topics/items.rst --ignore=docs/topics/leaks.rst From 63546cbf3e02380819732441ad55d95725dca7c5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 27 Nov 2019 22:42:52 +0500 Subject: [PATCH 54/85] Deprecate the HTTPS proxy noconnect mode. --- scrapy/core/downloader/handlers/http11.py | 7 ++++++ tests/test_downloader_handlers.py | 10 --------- tests/test_proxy_connect.py | 26 ----------------------- 3 files changed, 7 insertions(+), 36 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 7d917cb74..782eca89e 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -16,6 +16,7 @@ from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.client import Agent, ResponseDone, HTTPConnectionPool, ResponseFailed, URI from twisted.internet.endpoints import TCP4ClientEndpoint +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse @@ -285,6 +286,12 @@ class ScrapyAgent(object): scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) omitConnectTunnel = b'noconnect' in proxyParams + if omitConnectTunnel: + warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " + "If you use Crawlera, it doesn't require this mode anymore, " + "so you should update scrapy-crawlera to 1.3.0+ " + "and remove '?noconnect' from the Crawlera URL.", + ScrapyDeprecationWarning) if scheme == b'https' and not omitConnectTunnel: proxyAuth = request.headers.get(b'Proxy-Authorization', None) proxyConf = (proxyHost, proxyPort, proxyAuth) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 60124b93f..45d4aa952 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -687,16 +687,6 @@ class HttpProxyTestCase(unittest.TestCase): request = Request('http://example.com', meta={'proxy': http_proxy}) return self.download_request(request, Spider('foo')).addCallback(_test) - def test_download_with_proxy_https_noconnect(self): - def _test(response): - self.assertEqual(response.status, 200) - self.assertEqual(response.url, request.url) - self.assertEqual(response.body, b'https://example.com') - - http_proxy = '%s?noconnect' % self.getURL('') - request = Request('https://example.com', meta={'proxy': http_proxy}) - return self.download_request(request, Spider('foo')).addCallback(_test) - def test_download_without_proxy(self): def _test(response): self.assertEqual(response.status, 200) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 277455751..05d371b65 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -108,32 +108,6 @@ class ProxyConnectTestCase(TestCase): echo = json.loads(crawler.spider.meta['responses'][0].text) self.assertTrue('Proxy-Authorization' not in echo['headers']) - # The noconnect mode isn't supported by the current mitmproxy, it returns - # "Invalid request scheme: https" as it doesn't seem to support full URLs in GET at all, - # and it's not clear what behavior is intended by Scrapy and by mitmproxy here. - # https://github.com/mitmproxy/mitmproxy/issues/848 may be related. - # The Scrapy noconnect mode was required, at least in the past, to work with Crawlera, - # and https://github.com/scrapy-plugins/scrapy-crawlera/pull/44 seems to be related. - - @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') - @defer.inlineCallbacks - def test_https_noconnect(self): - proxy = os.environ['https_proxy'] - os.environ['https_proxy'] = proxy + '?noconnect' - crawler = get_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) - - @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') - @defer.inlineCallbacks - def test_https_noconnect_auth_error(self): - os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' - crawler = get_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(407, l) - def _assert_got_response_code(self, code, log): print(log) self.assertEqual(str(log).count('Crawled (%d)' % code), 1) From 5d8d4bb7d7d998ae2324c4995bafaafdef752572 Mon Sep 17 00:00:00 2001 From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com> Date: Thu, 5 Dec 2019 00:22:10 +1100 Subject: [PATCH 55/85] Re-arrange the imports in the httpproxy module (#4210) This commit re-arranges the imports in the httpproxy module to follow pep8 --- scrapy/downloadermiddlewares/httpproxy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 2212d9688..5e4542b6c 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,7 +1,8 @@ import base64 +from urllib.request import _parse_proxy + from six.moves.urllib.parse import unquote, urlunparse from six.moves.urllib.request import getproxies, proxy_bypass -from urllib.request import _parse_proxy from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached From 702333478d072c3c043c64ec7ad3997befb87943 Mon Sep 17 00:00:00 2001 From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com> Date: Thu, 5 Dec 2019 00:23:28 +1100 Subject: [PATCH 56/85] Re-arrange the imports in httpcache module (#4209) This commit re-arrange the imports in httpcache module to follow pep8 --- scrapy/downloadermiddlewares/httpcache.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 495b103d1..4e06f8236 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,11 +1,19 @@ from email.utils import formatdate + from twisted.internet import defer -from twisted.internet.error import TimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost, TCPTimedOutError +from twisted.internet.error import ( + ConnectError, + ConnectionDone, + ConnectionLost, + ConnectionRefusedError, + DNSLookupError, + TCPTimedOutError, + TimeoutError, +) from twisted.web.client import ResponseFailed + from scrapy import signals -from scrapy.exceptions import NotConfigured, IgnoreRequest +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.utils.misc import load_object From 74627033c4a1701f3e197216f9f2801d497f5535 Mon Sep 17 00:00:00 2001 From: Grammy Jiang <719388+grammy-jiang@users.noreply.github.com> Date: Thu, 5 Dec 2019 00:24:14 +1100 Subject: [PATCH 57/85] Remove the used import and re-arrange the imports (#4208) This commit removes unused import and re-arrange the imports in cookies module --- scrapy/downloadermiddlewares/cookies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 0d2b9900c..aeb7578b8 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -1,8 +1,8 @@ -import os -import six import logging from collections import defaultdict +import six + from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar From 9b4b43f8ac3a39d5dd6137223699d3d9e7dacec2 Mon Sep 17 00:00:00 2001 From: Grammy Jiang Date: Thu, 5 Dec 2019 11:25:19 +1100 Subject: [PATCH 58/85] Convert the relative imports to absolute imports This commits converts the relative imports to absolute imports in the entire package --- scrapy/__init__.py | 2 +- scrapy/contracts/default.py | 2 +- scrapy/core/downloader/__init__.py | 4 ++-- scrapy/core/downloader/handlers/http.py | 6 ++++-- scrapy/core/downloader/handlers/s3.py | 2 +- scrapy/linkextractors/__init__.py | 2 +- scrapy/linkextractors/regex.py | 2 +- 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 230e5cee3..fb8357f3c 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted') del warnings # Apply monkey patches to fix issues in external libraries -from . import _monkeypatches +from scrapy import _monkeypatches del _monkeypatches from twisted import version as _txv diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 24f6c2e77..e0d425874 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -4,7 +4,7 @@ from scrapy.item import BaseItem from scrapy.http import Request from scrapy.exceptions import ContractFail -from . import Contract +from scrapy.contracts import Contract # contracts diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 213268741..c5474a57f 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -10,8 +10,8 @@ from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached from scrapy.resolver import dnscache from scrapy import signals -from .middleware import DownloaderMiddlewareManager -from .handlers import DownloadHandlers +from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.core.downloader.handlers import DownloadHandlers class Slot(object): diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 6111e132a..52535bd8b 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,2 +1,4 @@ -from .http10 import HTTP10DownloadHandler -from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler +from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler +from scrapy.core.downloader.handlers.http11 import ( + HTTP11DownloadHandler as HTTPDownloadHandler, +) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 808d1bf21..f4a42ce12 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -3,7 +3,7 @@ from six.moves.urllib.parse import unquote from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.boto import is_botocore -from .http import HTTPDownloadHandler +from scrapy.core.downloader.handlers.http import HTTPDownloadHandler def _get_boto_connection(): diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 8c3693f04..8411c4d59 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -118,4 +118,4 @@ class FilteringLinkExtractor(object): # Top-level imports -from .lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401 +from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401 diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py index e689b4727..49aa2be46 100644 --- a/scrapy/linkextractors/regex.py +++ b/scrapy/linkextractors/regex.py @@ -4,7 +4,7 @@ from six.moves.urllib.parse import urljoin from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url from scrapy.link import Link -from .sgml import SgmlLinkExtractor +from scrapy.linkextractors.sgml import SgmlLinkExtractor linkre = re.compile( "|\s.*?>)(.*?)<[/ ]?a>", From af624ef414ab10d833925b4d6f9048468be01273 Mon Sep 17 00:00:00 2001 From: Wang Qin <37098874+dqwerter@users.noreply.github.com> Date: Thu, 5 Dec 2019 09:29:12 +0800 Subject: [PATCH 59/85] Update overview.rst | Fix an inconsistency There exists an inconsistency between the code (line 37 - 38) and the output 'quotes.json' (line 56 - 68). Note that even though according to line 53 - 54 'quotes.json' is "reformatted here for better readability", it cannot explain why the "author" field precedes the "text" field. Intended output for the code BEFORE change: [{ "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d", "author": "Jane Austen" }, { "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d", "author": "Groucho Marx" }, { "text": "\u201cA day without sunshine is like, you know, night.\u201d", "author": "Steve Martin" }, ...] Intended output for the code After change (the inconsistency is fixed): [{ "author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d" }, { "author": "Groucho Marx", "text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d" }, { "author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d" }, ...] --- docs/intro/overview.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 8b2fef065..01986b594 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').get(), 'author': quote.xpath('span/small/text()').get(), + 'text': quote.css('span.text::text').get(), } next_page = response.css('li.next a::attr("href")').get() From 83b8046fdcae660276634a03b99d51c629fa8b7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Nov 2019 16:26:46 +0100 Subject: [PATCH 60/85] Do not indent doctests from the documentation unnecessarily --- docs/intro/tutorial.rst | 120 +++---- docs/news.rst | 12 +- docs/topics/developer-tools.rst | 12 +- docs/topics/dynamic-content.rst | 28 +- docs/topics/items.rst | 106 +++--- docs/topics/leaks.rst | 140 ++++---- docs/topics/loaders.rst | 114 ++++--- docs/topics/selectors.rst | 565 ++++++++++++++++---------------- docs/topics/shell.rst | 77 +++-- docs/topics/stats.rst | 12 +- 10 files changed, 589 insertions(+), 597 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 6b15a5fbd..33b1a969b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -252,30 +252,30 @@ The result of running ``response.css('title')`` is a list-like object called and allow you to run further queries to fine-grain the selection or extract the data. -To extract the text from the title above, you can do:: +To extract the text from the title above, you can do: - >>> response.css('title::text').getall() - ['Quotes to Scrape'] +>>> response.css('title::text').getall() +['Quotes to Scrape'] There are two things to note here: one is that we've added ``::text`` to the CSS query, to mean we want to select only the text elements directly inside ```` element. If we don't specify ``::text``, we'd get the full title -element, including its tags:: +element, including its tags: - >>> response.css('title').getall() - ['<title>Quotes to Scrape'] +>>> response.css('title').getall() +['Quotes to Scrape'] The other thing is that the result of calling ``.getall()`` is a list: it is possible that a selector returns more than one result, so we extract them all. -When you know you just want the first result, as in this case, you can do:: +When you know you just want the first result, as in this case, you can do: - >>> response.css('title::text').get() - 'Quotes to Scrape' +>>> response.css('title::text').get() +'Quotes to Scrape' -As an alternative, you could've written:: +As an alternative, you could've written: - >>> response.css('title::text')[0].get() - 'Quotes to Scrape' +>>> response.css('title::text')[0].get() +'Quotes to Scrape' However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList` instance avoids an ``IndexError`` and returns ``None`` when it doesn't @@ -288,14 +288,14 @@ to be scraped, you can at least get **some** data. Besides the :meth:`~scrapy.selector.SelectorList.getall` and :meth:`~scrapy.selector.SelectorList.get` methods, you can also use the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular -expressions`_:: +expressions`_: - >>> response.css('title::text').re(r'Quotes.*') - ['Quotes to Scrape'] - >>> response.css('title::text').re(r'Q\w+') - ['Quotes'] - >>> response.css('title::text').re(r'(\w+) to (\w+)') - ['Quotes', 'Scrape'] +>>> response.css('title::text').re(r'Quotes.*') +['Quotes to Scrape'] +>>> response.css('title::text').re(r'Q\w+') +['Quotes'] +>>> response.css('title::text').re(r'(\w+) to (\w+)') +['Quotes', 'Scrape'] In order to find the proper CSS selectors to use, you might find useful opening the response page from the shell in your web browser using ``view(response)``. @@ -312,12 +312,12 @@ visually selected elements, which works in many browsers. XPath: a brief intro ^^^^^^^^^^^^^^^^^^^^ -Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:: +Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions: - >>> response.xpath('//title') - [] - >>> response.xpath('//title/text()').get() - 'Quotes to Scrape' +>>> response.xpath('//title') +[] +>>> response.xpath('//title/text()').get() +'Quotes to Scrape' XPath expressions are very powerful, and are the foundation of Scrapy Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You @@ -372,35 +372,35 @@ we want:: $ scrapy shell 'http://quotes.toscrape.com' -We get a list of selectors for the quote HTML elements with:: +We get a list of selectors for the quote HTML elements with: - >>> response.css("div.quote") - [, - , - ...] +>>> response.css("div.quote") +[, + , + ...] Each of the selectors returned by the query above allows us to run further queries over their sub-elements. Let's assign the first selector to a -variable, so that we can run our CSS selectors directly on a particular quote:: +variable, so that we can run our CSS selectors directly on a particular quote: - >>> quote = response.css("div.quote")[0] +>>> quote = response.css("div.quote")[0] Now, let's extract ``text``, ``author`` and the ``tags`` from that quote -using the ``quote`` object we just created:: +using the ``quote`` object we just created: - >>> text = quote.css("span.text::text").get() - >>> text - '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' - >>> author = quote.css("small.author::text").get() - >>> author - 'Albert Einstein' +>>> text = quote.css("span.text::text").get() +>>> text +'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' +>>> author = quote.css("small.author::text").get() +>>> author +'Albert Einstein' Given that the tags are a list of strings, we can use the ``.getall()`` method -to get all of them:: +to get all of them: - >>> tags = quote.css("div.tags a.tag::text").getall() - >>> tags - ['change', 'deep-thoughts', 'thinking', 'world'] +>>> tags = quote.css("div.tags a.tag::text").getall() +>>> tags +['change', 'deep-thoughts', 'thinking', 'world'] .. invisible-code-block: python @@ -409,16 +409,16 @@ to get all of them:: .. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output") Having figured out how to extract each bit, we can now iterate over all the -quotes elements and put them together into a Python dictionary:: +quotes elements and put them together into a Python dictionary: - >>> for quote in response.css("div.quote"): - ... text = quote.css("span.text::text").get() - ... author = quote.css("small.author::text").get() - ... tags = quote.css("div.tags a.tag::text").getall() - ... print(dict(text=text, author=author, tags=tags)) - {'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']} - {'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']} - ... +>>> for quote in response.css("div.quote"): +... text = quote.css("span.text::text").get() +... author = quote.css("small.author::text").get() +... tags = quote.css("div.tags a.tag::text").getall() +... print(dict(text=text, author=author, tags=tags)) +{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']} +{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']} +... Extracting data in our spider ----------------------------- @@ -516,23 +516,23 @@ markup: -We can try extracting it in the shell:: +We can try extracting it in the shell: - >>> response.css('li.next a').get() - 'Next ' +>>> response.css('li.next a').get() +'Next ' This gets the anchor element, but we want the attribute ``href``. For that, Scrapy supports a CSS extension that lets you select the attribute contents, -like this:: +like this: - >>> response.css('li.next a::attr(href)').get() - '/page/2/' +>>> response.css('li.next a::attr(href)').get() +'/page/2/' There is also an ``attrib`` property available -(see :ref:`selecting-attributes` for more):: +(see :ref:`selecting-attributes` for more): - >>> response.css('li.next a').attrib['href'] - '/page/2/' +>>> response.css('li.next a').attrib['href'] +'/page/2/' Let's see now our spider modified to recursively follow the link to the next page, extracting data from it:: diff --git a/docs/news.rst b/docs/news.rst index 9dfd28508..217382c57 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -47,13 +47,13 @@ Backward-incompatible changes (:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`) * :class:`~scrapy.loader.ItemLoader` now turns the values of its input item - into lists:: + into lists: - >>> item = MyItem() - >>> item['field'] = 'value1' - >>> loader = ItemLoader(item=item) - >>> item['field'] - ['value1'] + >>> item = MyItem() + >>> item['field'] = 'value1' + >>> loader = ItemLoader(item=item) + >>> item['field'] + ['value1'] This is needed to allow adding values to existing fields (``loader.add_value('field', 'value2')``). diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index bf14643be..1c9315cd8 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -112,13 +112,13 @@ see each quote: With this knowledge we can refine our XPath: Instead of a path to follow, we'll simply select all ``span`` tags with the ``class="text"`` by using -the `has-class-extension`_:: +the `has-class-extension`_: - >>> response.xpath('//span[has-class("text")]/text()').getall() - ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”, - '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', - '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', - (...)] +>>> response.xpath('//span[has-class("text")]/text()').getall() +['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', + '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', + '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”', + ...] And with one simple, cleverer XPath we are able to extract all quotes from the page. We could have constructed a loop over our first XPath to increase diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 8334ddcec..1c3607860 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -172,27 +172,27 @@ data from it: data in JSON format, which you can then parse with `json.loads`_. For example, if the JavaScript code contains a separate line like - ``var data = {"field": "value"};`` you can extract that data as follows:: + ``var data = {"field": "value"};`` you can extract that data as follows: - >>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n' - >>> json_data = response.css('script::text').re_first(pattern) - >>> json.loads(json_data) - {'field': 'value'} + >>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n' + >>> json_data = response.css('script::text').re_first(pattern) + >>> json.loads(json_data) + {'field': 'value'} - Otherwise, use js2xml_ to convert the JavaScript code into an XML document that you can parse using :ref:`selectors `. For example, if the JavaScript code contains - ``var data = {field: "value"};`` you can extract that data as follows:: + ``var data = {field: "value"};`` you can extract that data as follows: - >>> import js2xml - >>> import lxml.etree - >>> from parsel import Selector - >>> javascript = response.css('script::text').get() - >>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode') - >>> selector = Selector(text=xml) - >>> selector.css('var[name="data"]').get() - 'value' + >>> import js2xml + >>> import lxml.etree + >>> from parsel import Selector + >>> javascript = response.css('script::text').get() + >>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode') + >>> selector = Selector(text=xml) + >>> selector.css('var[name="data"]').get() + 'value' .. _topics-javascript-rendering: diff --git a/docs/topics/items.rst b/docs/topics/items.rst index cdf60208e..15313775b 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -84,77 +84,74 @@ notice the API is very similar to the `dict API`_. Creating items -------------- -:: +>>> product = Product(name='Desktop PC', price=1000) +>>> print(product) +Product(name='Desktop PC', price=1000) - >>> product = Product(name='Desktop PC', price=1000) - >>> print(product) - Product(name='Desktop PC', price=1000) Getting field values -------------------- -:: +>>> product['name'] +Desktop PC +>>> product.get('name') +Desktop PC - >>> product['name'] - Desktop PC - >>> product.get('name') - Desktop PC +>>> product['price'] +1000 - >>> product['price'] - 1000 +>>> product['last_updated'] +Traceback (most recent call last): + ... +KeyError: 'last_updated' - >>> product['last_updated'] - Traceback (most recent call last): - ... - KeyError: 'last_updated' +>>> product.get('last_updated', 'not set') +not set - >>> product.get('last_updated', 'not set') - not set +>>> product['lala'] # getting unknown field +Traceback (most recent call last): + ... +KeyError: 'lala' - >>> product['lala'] # getting unknown field - Traceback (most recent call last): - ... - KeyError: 'lala' +>>> product.get('lala', 'unknown field') +'unknown field' - >>> product.get('lala', 'unknown field') - 'unknown field' +>>> 'name' in product # is name field populated? +True - >>> 'name' in product # is name field populated? - True +>>> 'last_updated' in product # is last_updated populated? +False - >>> 'last_updated' in product # is last_updated populated? - False +>>> 'last_updated' in product.fields # is last_updated a declared field? +True - >>> 'last_updated' in product.fields # is last_updated a declared field? - True +>>> 'lala' in product.fields # is lala a declared field? +False - >>> 'lala' in product.fields # is lala a declared field? - False Setting field values -------------------- -:: +>>> product['last_updated'] = 'today' +>>> product['last_updated'] +today - >>> product['last_updated'] = 'today' - >>> product['last_updated'] - today +>>> product['lala'] = 'test' # setting unknown field +Traceback (most recent call last): + ... +KeyError: 'Product does not support field: lala' - >>> product['lala'] = 'test' # setting unknown field - Traceback (most recent call last): - ... - KeyError: 'Product does not support field: lala' Accessing all populated values ------------------------------ -To access all populated values, just use the typical `dict API`_:: +To access all populated values, just use the typical `dict API`_: - >>> product.keys() - ['price', 'name'] +>>> product.keys() +['price', 'name'] - >>> product.items() - [('price', 1000), ('name', 'Desktop PC')] +>>> product.items() +[('price', 1000), ('name', 'Desktop PC')] .. _copying-items: @@ -194,20 +191,21 @@ To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead Other common tasks ------------------ -Creating dicts from items:: +Creating dicts from items: - >>> dict(product) # create a dict from all populated values - {'price': 1000, 'name': 'Desktop PC'} +>>> dict(product) # create a dict from all populated values +{'price': 1000, 'name': 'Desktop PC'} -Creating items from dicts:: +Creating items from dicts: - >>> Product({'name': 'Laptop PC', 'price': 1500}) - Product(price=1500, name='Laptop PC') +>>> Product({'name': 'Laptop PC', 'price': 1500}) +Product(price=1500, name='Laptop PC') + +>>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict +Traceback (most recent call last): + ... +KeyError: 'Product does not support field: lala' - >>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict - Traceback (most recent call last): - ... - KeyError: 'Product does not support field: lala' Extending Items =============== diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 793636f59..87d9d262f 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -132,21 +132,21 @@ and check the code of the spider to discover the nasty line that is generating the leaks (passing response references inside requests). Sometimes extra information about live objects can be helpful. -Let's check the oldest response:: +Let's check the oldest response: - >>> from scrapy.utils.trackref import get_oldest - >>> r = get_oldest('HtmlResponse') - >>> r.url - 'http://www.somenastyspider.com/product.php?pid=123' +>>> from scrapy.utils.trackref import get_oldest +>>> r = get_oldest('HtmlResponse') +>>> r.url +'http://www.somenastyspider.com/product.php?pid=123' If you want to iterate over all objects, instead of getting the oldest one, you -can use the :func:`scrapy.utils.trackref.iter_all` function:: +can use the :func:`scrapy.utils.trackref.iter_all` function: - >>> from scrapy.utils.trackref import iter_all - >>> [r.url for r in iter_all('HtmlResponse')] - ['http://www.somenastyspider.com/product.php?pid=123', - 'http://www.somenastyspider.com/product.php?pid=584', - ... +>>> from scrapy.utils.trackref import iter_all +>>> [r.url for r in iter_all('HtmlResponse')] +['http://www.somenastyspider.com/product.php?pid=123', + 'http://www.somenastyspider.com/product.php?pid=584', +...] Too many spiders? ----------------- @@ -155,10 +155,10 @@ If your project has too many spiders executed in parallel, the output of :func:`prefs()` can be difficult to read. For this reason, that function has a ``ignore`` argument which can be used to ignore a particular class (and all its subclases). For -example, this won't show any live references to spiders:: +example, this won't show any live references to spiders: - >>> from scrapy.spiders import Spider - >>> prefs(ignore=Spider) +>>> from scrapy.spiders import Spider +>>> prefs(ignore=Spider) .. module:: scrapy.utils.trackref :synopsis: Track references of live objects @@ -214,41 +214,41 @@ If you use ``pip``, you can install Guppy with the following command:: The telnet console also comes with a built-in shortcut (``hpy``) for accessing Guppy heap objects. Here's an example to view all Python objects available in -the heap using Guppy:: +the heap using Guppy: - >>> x = hpy.heap() - >>> x.bytype - Partition of a set of 297033 objects. Total size = 52587824 bytes. - Index Count % Size % Cumulative % Type - 0 22307 8 16423880 31 16423880 31 dict - 1 122285 41 12441544 24 28865424 55 str - 2 68346 23 5966696 11 34832120 66 tuple - 3 227 0 5836528 11 40668648 77 unicode - 4 2461 1 2222272 4 42890920 82 type - 5 16870 6 2024400 4 44915320 85 function - 6 13949 5 1673880 3 46589200 89 types.CodeType - 7 13422 5 1653104 3 48242304 92 list - 8 3735 1 1173680 2 49415984 94 _sre.SRE_Pattern - 9 1209 0 456936 1 49872920 95 scrapy.http.headers.Headers - <1676 more rows. Type e.g. '_.more' to view.> +>>> x = hpy.heap() +>>> x.bytype +Partition of a set of 297033 objects. Total size = 52587824 bytes. + Index Count % Size % Cumulative % Type + 0 22307 8 16423880 31 16423880 31 dict + 1 122285 41 12441544 24 28865424 55 str + 2 68346 23 5966696 11 34832120 66 tuple + 3 227 0 5836528 11 40668648 77 unicode + 4 2461 1 2222272 4 42890920 82 type + 5 16870 6 2024400 4 44915320 85 function + 6 13949 5 1673880 3 46589200 89 types.CodeType + 7 13422 5 1653104 3 48242304 92 list + 8 3735 1 1173680 2 49415984 94 _sre.SRE_Pattern + 9 1209 0 456936 1 49872920 95 scrapy.http.headers.Headers +<1676 more rows. Type e.g. '_.more' to view.> You can see that most space is used by dicts. Then, if you want to see from -which attribute those dicts are referenced, you could do:: +which attribute those dicts are referenced, you could do: - >>> x.bytype[0].byvia - Partition of a set of 22307 objects. Total size = 16423880 bytes. - Index Count % Size % Cumulative % Referred Via: - 0 10982 49 9416336 57 9416336 57 '.__dict__' - 1 1820 8 2681504 16 12097840 74 '.__dict__', '.func_globals' - 2 3097 14 1122904 7 13220744 80 - 3 990 4 277200 2 13497944 82 "['cookies']" - 4 987 4 276360 2 13774304 84 "['cache']" - 5 985 4 275800 2 14050104 86 "['meta']" - 6 897 4 251160 2 14301264 87 '[2]' - 7 1 0 196888 1 14498152 88 "['moduleDict']", "['modules']" - 8 672 3 188160 1 14686312 89 "['cb_kwargs']" - 9 27 0 155016 1 14841328 90 '[1]' - <333 more rows. Type e.g. '_.more' to view.> +>>> x.bytype[0].byvia +Partition of a set of 22307 objects. Total size = 16423880 bytes. + Index Count % Size % Cumulative % Referred Via: + 0 10982 49 9416336 57 9416336 57 '.__dict__' + 1 1820 8 2681504 16 12097840 74 '.__dict__', '.func_globals' + 2 3097 14 1122904 7 13220744 80 + 3 990 4 277200 2 13497944 82 "['cookies']" + 4 987 4 276360 2 13774304 84 "['cache']" + 5 985 4 275800 2 14050104 86 "['meta']" + 6 897 4 251160 2 14301264 87 '[2]' + 7 1 0 196888 1 14498152 88 "['moduleDict']", "['modules']" + 8 672 3 188160 1 14686312 89 "['cb_kwargs']" + 9 27 0 155016 1 14841328 90 '[1]' +<333 more rows. Type e.g. '_.more' to view.> As you can see, the Guppy module is very powerful but also requires some deep knowledge about Python internals. For more info about Guppy, refer to the @@ -269,32 +269,32 @@ If you use ``pip``, you can install muppy with the following command:: pip install Pympler Here's an example to view all Python objects available in -the heap using muppy:: +the heap using muppy: - >>> from pympler import muppy - >>> all_objects = muppy.get_objects() - >>> len(all_objects) - 28667 - >>> from pympler import summary - >>> suml = summary.summarize(all_objects) - >>> summary.print_(suml) - types | # objects | total size - ==================================== | =========== | ============ - >> from pympler import muppy +>>> all_objects = muppy.get_objects() +>>> len(all_objects) +28667 +>>> from pympler import summary +>>> suml = summary.summarize(all_objects) +>>> summary.print_(suml) + types | # objects | total size +==================================== | =========== | ============ + >> from scrapy.loader import ItemLoader - >>> il = ItemLoader(item=Product()) - >>> il.add_value('name', [u'Welcome to my', u'website']) - >>> il.add_value('price', [u'€', u'1000']) - >>> il.load_item() - {'name': u'Welcome to my website', 'price': u'1000'} +>>> from scrapy.loader import ItemLoader +>>> il = ItemLoader(item=Product()) +>>> il.add_value('name', [u'Welcome to my', u'website']) +>>> il.add_value('price', [u'€', u'1000']) +>>> il.load_item() +{'name': u'Welcome to my website', 'price': u'1000'} The precedence order, for both input and output processors, is as follows: @@ -314,11 +312,11 @@ ItemLoader objects applied before processors :type re: str or compiled regex - Examples:: + Examples: - >>> from scrapy.loader.processors import TakeFirst - >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)') - 'FOO` + >>> from scrapy.loader.processors import TakeFirst + >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)') + 'FOO` .. method:: add_value(field_name, value, \*processors, \**kwargs) @@ -639,12 +637,12 @@ Here is a list of all built-in processors: values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts. - Example:: + Example: - >>> from scrapy.loader.processors import Identity - >>> proc = Identity() - >>> proc(['one', 'two', 'three']) - ['one', 'two', 'three'] + >>> from scrapy.loader.processors import Identity + >>> proc = Identity() + >>> proc(['one', 'two', 'three']) + ['one', 'two', 'three'] .. class:: TakeFirst @@ -652,12 +650,12 @@ Here is a list of all built-in processors: so it's typically used as an output processor to single-valued fields. It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts. - Example:: + Example: - >>> from scrapy.loader.processors import TakeFirst - >>> proc = TakeFirst() - >>> proc(['', 'one', 'two', 'three']) - 'one' + >>> from scrapy.loader.processors import TakeFirst + >>> proc = TakeFirst() + >>> proc(['', 'one', 'two', 'three']) + 'one' .. class:: Join(separator=u' ') @@ -667,15 +665,15 @@ Here is a list of all built-in processors: When using the default separator, this processor is equivalent to the function: ``u' '.join`` - Examples:: + Examples: - >>> from scrapy.loader.processors import Join - >>> proc = Join() - >>> proc(['one', 'two', 'three']) - 'one two three' - >>> proc = Join('
') - >>> proc(['one', 'two', 'three']) - 'one
two
three' + >>> from scrapy.loader.processors import Join + >>> proc = Join() + >>> proc(['one', 'two', 'three']) + 'one two three' + >>> proc = Join('
') + >>> proc(['one', 'two', 'three']) + 'one
two
three' .. class:: Compose(\*functions, \**default_loader_context) @@ -688,12 +686,12 @@ Here is a list of all built-in processors: By default, stop process on ``None`` value. This behaviour can be changed by passing keyword argument ``stop_on_none=False``. - Example:: + Example: - >>> from scrapy.loader.processors import Compose - >>> proc = Compose(lambda v: v[0], str.upper) - >>> proc(['hello', 'world']) - 'HELLO' + >>> from scrapy.loader.processors import Compose + >>> proc = Compose(lambda v: v[0], str.upper) + >>> proc(['hello', 'world']) + 'HELLO' Each function can optionally receive a ``loader_context`` parameter. For those which do, this processor will pass the currently active :ref:`Loader @@ -732,15 +730,15 @@ Here is a list of all built-in processors: :meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors `, which returns a list of unicode strings. - The example below should clarify how it works:: + The example below should clarify how it works: - >>> def filter_world(x): - ... return None if x == 'world' else x - ... - >>> from scrapy.loader.processors import MapCompose - >>> proc = MapCompose(filter_world, str.upper) - >>> proc(['hello', 'world', 'this', 'is', 'scrapy']) - ['HELLO, 'THIS', 'IS', 'SCRAPY'] + >>> def filter_world(x): + ... return None if x == 'world' else x + ... + >>> from scrapy.loader.processors import MapCompose + >>> proc = MapCompose(filter_world, str.upper) + >>> proc(['hello', 'world', 'this', 'is', 'scrapy']) + ['HELLO, 'THIS', 'IS', 'SCRAPY'] As with the Compose processor, functions can receive Loader contexts, and ``__init__`` method keyword arguments are used as default context values. See @@ -752,21 +750,21 @@ Here is a list of all built-in processors: Requires jmespath (https://github.com/jmespath/jmespath.py) to run. This processor takes only one input at a time. - Example:: + Example: - >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose - >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries - >>> proc({'foo': 'bar'}) - 'bar' - >>> proc({'foo': {'bar': 'baz'}}) - {'bar': 'baz'} + >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose + >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries + >>> proc({'foo': 'bar'}) + 'bar' + >>> proc({'foo': {'bar': 'baz'}}) + {'bar': 'baz'} - Working with Json:: + Working with Json: - >>> import json - >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo")) - >>> proc_single_json_str('{"foo": "bar"}') - 'bar' - >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo'))) - >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]') - ['bar'] + >>> import json + >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo")) + >>> proc_single_json_str('{"foo": "bar"}') + 'bar' + >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo'))) + >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]') + ['bar'] diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 282a585d4..8ec758b0e 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -51,18 +51,18 @@ Constructing selectors .. highlight:: python Response objects expose a :class:`~scrapy.selector.Selector` instance -on ``.selector`` attribute:: +on ``.selector`` attribute: - >>> response.selector.xpath('//span/text()').get() - 'good' +>>> response.selector.xpath('//span/text()').get() +'good' Querying responses using XPath and CSS is so common that responses include two -more shortcuts: ``response.xpath()`` and ``response.css()``:: +more shortcuts: ``response.xpath()`` and ``response.css()``: - >>> response.xpath('//span/text()').get() - 'good' - >>> response.css('span::text').get() - 'good' +>>> response.xpath('//span/text()').get() +'good' +>>> response.css('span::text').get() +'good' Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class constructed by passing either :class:`~scrapy.http.TextResponse` object or @@ -74,21 +74,21 @@ shortcuts. By using ``response.selector`` or one of these shortcuts you can also ensure the response body is parsed only once. But if required, it is possible to use ``Selector`` directly. -Constructing from text:: +Constructing from text: - >>> from scrapy.selector import Selector - >>> body = 'good' - >>> Selector(text=body).xpath('//span/text()').get() - 'good' +>>> from scrapy.selector import Selector +>>> body = 'good' +>>> Selector(text=body).xpath('//span/text()').get() +'good' Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of -:class:`~scrapy.http.TextResponse` subclasses:: +:class:`~scrapy.http.TextResponse` subclasses: - >>> from scrapy.selector import Selector - >>> from scrapy.http import HtmlResponse - >>> response = HtmlResponse(url='http://example.com', body=body) - >>> Selector(response=response).xpath('//span/text()').get() - 'good' +>>> from scrapy.selector import Selector +>>> from scrapy.http import HtmlResponse +>>> response = HtmlResponse(url='http://example.com', body=body) +>>> Selector(response=response).xpath('//span/text()').get() +'good' ``Selector`` automatically chooses the best parsing rules (XML vs HTML) based on input type. @@ -123,118 +123,118 @@ Since we're dealing with HTML, the selector will automatically use an HTML parse .. highlight:: python So, by looking at the :ref:`HTML code ` of that -page, let's construct an XPath for selecting the text inside the title tag:: +page, let's construct an XPath for selecting the text inside the title tag: - >>> response.xpath('//title/text()') - [] +>>> response.xpath('//title/text()') +[] To actually extract the textual data, you must call the selector ``.get()`` -or ``.getall()`` methods, as follows:: +or ``.getall()`` methods, as follows: - >>> response.xpath('//title/text()').getall() - ['Example website'] - >>> response.xpath('//title/text()').get() - 'Example website' +>>> response.xpath('//title/text()').getall() +['Example website'] +>>> response.xpath('//title/text()').get() +'Example website' ``.get()`` always returns a single result; if there are several matches, content of a first match is returned; if there are no matches, None is returned. ``.getall()`` returns a list with all results. Notice that CSS selectors can select text or attribute nodes using CSS3 -pseudo-elements:: +pseudo-elements: - >>> response.css('title::text').get() - 'Example website' +>>> response.css('title::text').get() +'Example website' As you can see, ``.xpath()`` and ``.css()`` methods return a :class:`~scrapy.selector.SelectorList` instance, which is a list of new -selectors. This API can be used for quickly selecting nested data:: +selectors. This API can be used for quickly selecting nested data: - >>> response.css('img').xpath('@src').getall() - ['image1_thumb.jpg', - 'image2_thumb.jpg', - 'image3_thumb.jpg', - 'image4_thumb.jpg', - 'image5_thumb.jpg'] +>>> response.css('img').xpath('@src').getall() +['image1_thumb.jpg', + 'image2_thumb.jpg', + 'image3_thumb.jpg', + 'image4_thumb.jpg', + 'image5_thumb.jpg'] If you want to extract only the first matched element, you can call the selector ``.get()`` (or its alias ``.extract_first()`` commonly used in -previous Scrapy versions):: +previous Scrapy versions): - >>> response.xpath('//div[@id="images"]/a/text()').get() - 'Name: My image 1 ' +>>> response.xpath('//div[@id="images"]/a/text()').get() +'Name: My image 1 ' -It returns ``None`` if no element was found:: +It returns ``None`` if no element was found: - >>> response.xpath('//div[@id="not-exists"]/text()').get() is None - True +>>> response.xpath('//div[@id="not-exists"]/text()').get() is None +True A default return value can be provided as an argument, to be used instead of ``None``: - >>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found') - 'not-found' +>>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found') +'not-found' Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes -using ``.attrib`` property of a :class:`~scrapy.selector.Selector`:: +using ``.attrib`` property of a :class:`~scrapy.selector.Selector`: - >>> [img.attrib['src'] for img in response.css('img')] - ['image1_thumb.jpg', - 'image2_thumb.jpg', - 'image3_thumb.jpg', - 'image4_thumb.jpg', - 'image5_thumb.jpg'] +>>> [img.attrib['src'] for img in response.css('img')] +['image1_thumb.jpg', + 'image2_thumb.jpg', + 'image3_thumb.jpg', + 'image4_thumb.jpg', + 'image5_thumb.jpg'] As a shortcut, ``.attrib`` is also available on SelectorList directly; -it returns attributes for the first matching element:: +it returns attributes for the first matching element: - >>> response.css('img').attrib['src'] - 'image1_thumb.jpg' +>>> response.css('img').attrib['src'] +'image1_thumb.jpg' This is most useful when only a single result is expected, e.g. when selecting -by id, or selecting unique elements on a web page:: +by id, or selecting unique elements on a web page: - >>> response.css('base').attrib['href'] - 'http://example.com/' +>>> response.css('base').attrib['href'] +'http://example.com/' -Now we're going to get the base URL and some image links:: +Now we're going to get the base URL and some image links: - >>> response.xpath('//base/@href').get() - 'http://example.com/' +>>> response.xpath('//base/@href').get() +'http://example.com/' - >>> response.css('base::attr(href)').get() - 'http://example.com/' +>>> response.css('base::attr(href)').get() +'http://example.com/' - >>> response.css('base').attrib['href'] - 'http://example.com/' +>>> response.css('base').attrib['href'] +'http://example.com/' - >>> response.xpath('//a[contains(@href, "image")]/@href').getall() - ['image1.html', - 'image2.html', - 'image3.html', - 'image4.html', - 'image5.html'] +>>> response.xpath('//a[contains(@href, "image")]/@href').getall() +['image1.html', + 'image2.html', + 'image3.html', + 'image4.html', + 'image5.html'] - >>> response.css('a[href*=image]::attr(href)').getall() - ['image1.html', - 'image2.html', - 'image3.html', - 'image4.html', - 'image5.html'] +>>> response.css('a[href*=image]::attr(href)').getall() +['image1.html', + 'image2.html', + 'image3.html', + 'image4.html', + 'image5.html'] - >>> response.xpath('//a[contains(@href, "image")]/img/@src').getall() - ['image1_thumb.jpg', - 'image2_thumb.jpg', - 'image3_thumb.jpg', - 'image4_thumb.jpg', - 'image5_thumb.jpg'] +>>> response.xpath('//a[contains(@href, "image")]/img/@src').getall() +['image1_thumb.jpg', + 'image2_thumb.jpg', + 'image3_thumb.jpg', + 'image4_thumb.jpg', + 'image5_thumb.jpg'] - >>> response.css('a[href*=image] img::attr(src)').getall() - ['image1_thumb.jpg', - 'image2_thumb.jpg', - 'image3_thumb.jpg', - 'image4_thumb.jpg', - 'image5_thumb.jpg'] +>>> response.css('a[href*=image] img::attr(src)').getall() +['image1_thumb.jpg', + 'image2_thumb.jpg', + 'image3_thumb.jpg', + 'image4_thumb.jpg', + 'image5_thumb.jpg'] .. _topics-selectors-css-extensions: @@ -259,47 +259,47 @@ that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**: Examples: -* ``title::text`` selects children text nodes of a descendant ```` element:: +* ``title::text`` selects children text nodes of a descendant ``<title>`` element: - >>> response.css('title::text').get() - 'Example website' +>>> response.css('title::text').get() +'Example website' -* ``*::text`` selects all descendant text nodes of the current selector context:: +* ``*::text`` selects all descendant text nodes of the current selector context: - >>> response.css('#images *::text').getall() - ['\n ', - 'Name: My image 1 ', - '\n ', - 'Name: My image 2 ', - '\n ', - 'Name: My image 3 ', - '\n ', - 'Name: My image 4 ', - '\n ', - 'Name: My image 5 ', - '\n '] +>>> response.css('#images *::text').getall() +['\n ', + 'Name: My image 1 ', + '\n ', + 'Name: My image 2 ', + '\n ', + 'Name: My image 3 ', + '\n ', + 'Name: My image 4 ', + '\n ', + 'Name: My image 5 ', + '\n '] * ``foo::text`` returns no results if ``foo`` element exists, but contains - no text (i.e. text is empty):: + no text (i.e. text is empty): - >>> response.css('img::text').getall() - [] +>>> response.css('img::text').getall() +[] This means ``.css('foo::text').get()`` could return None even if an element - exists. Use ``default=''`` if you always want a string:: + exists. Use ``default=''`` if you always want a string: - >>> response.css('img::text').get() - >>> response.css('img::text').get(default='') - '' +>>> response.css('img::text').get() +>>> response.css('img::text').get(default='') +'' -* ``a::attr(href)`` selects the *href* attribute value of descendant links:: +* ``a::attr(href)`` selects the *href* attribute value of descendant links: - >>> response.css('a::attr(href)').getall() - ['image1.html', - 'image2.html', - 'image3.html', - 'image4.html', - 'image5.html'] +>>> response.css('a::attr(href)').getall() +['image1.html', + 'image2.html', + 'image3.html', + 'image4.html', + 'image5.html'] .. note:: See also: :ref:`selecting-attributes`. @@ -318,25 +318,24 @@ Nesting selectors The selection methods (``.xpath()`` or ``.css()``) return a list of selectors of the same type, so you can call the selection methods for those selectors -too. Here's an example:: +too. Here's an example: - >>> links = response.xpath('//a[contains(@href, "image")]') - >>> links.getall() - ['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>', - '<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>', - '<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>', - '<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>', - '<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>'] +>>> links = response.xpath('//a[contains(@href, "image")]') +>>> links.getall() +['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>', + '<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>', + '<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>', + '<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>', + '<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>'] - >>> for index, link in enumerate(links): - ... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get()) - ... print('Link number %d points to url %r and image %r' % args) - - Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg' - Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg' - Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg' - Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg' - Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg' +>>> for index, link in enumerate(links): +... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get()) +... print('Link number %d points to url %r and image %r' % args) +Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg' +Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg' +Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg' +Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg' +Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg' .. _selecting-attributes: @@ -344,42 +343,42 @@ Selecting element attributes ---------------------------- There are several ways to get a value of an attribute. First, one can use -XPath syntax:: +XPath syntax: - >>> response.xpath("//a/@href").getall() - ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] +>>> response.xpath("//a/@href").getall() +['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] XPath syntax has a few advantages: it is a standard XPath feature, and ``@attributes`` can be used in other parts of an XPath expression - e.g. it is possible to filter by attribute value. Scrapy also provides an extension to CSS selectors (``::attr(...)``) -which allows to get attribute values:: +which allows to get attribute values: - >>> response.css('a::attr(href)').getall() - ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] +>>> response.css('a::attr(href)').getall() +['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] In addition to that, there is a ``.attrib`` property of Selector. You can use it if you prefer to lookup attributes in Python -code, without using XPaths or CSS extensions:: +code, without using XPaths or CSS extensions: - >>> [a.attrib['href'] for a in response.css('a')] - ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] +>>> [a.attrib['href'] for a in response.css('a')] +['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] This property is also available on SelectorList; it returns a dictionary with attributes of a first matching element. It is convenient to use when a selector is expected to give a single result (e.g. when selecting by element -ID, or when selecting an unique element on a page):: +ID, or when selecting an unique element on a page): - >>> response.css('base').attrib - {'href': 'http://example.com/'} - >>> response.css('base').attrib['href'] - 'http://example.com/' +>>> response.css('base').attrib +{'href': 'http://example.com/'} +>>> response.css('base').attrib['href'] +'http://example.com/' -``.attrib`` property of an empty SelectorList is empty:: +``.attrib`` property of an empty SelectorList is empty: - >>> response.css('foo').attrib - {} +>>> response.css('foo').attrib +{} Using selectors with regular expressions ---------------------------------------- @@ -390,21 +389,21 @@ data using regular expressions. However, unlike using ``.xpath()`` or can't construct nested ``.re()`` calls. Here's an example used to extract image names from the :ref:`HTML code -<topics-selectors-htmlcode>` above:: +<topics-selectors-htmlcode>` above: - >>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') - ['My image 1', - 'My image 2', - 'My image 3', - 'My image 4', - 'My image 5'] +>>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') +['My image 1', + 'My image 2', + 'My image 3', + 'My image 4', + 'My image 5'] There's an additional helper reciprocating ``.get()`` (and its alias ``.extract_first()``) for ``.re()``, named ``.re_first()``. -Use it to extract just the first matching string:: +Use it to extract just the first matching string: - >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)') - 'My image 1' +>>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)') +'My image 1' .. _old-extraction-api: @@ -422,28 +421,28 @@ and readable code. The following examples show how these methods map to each other. -1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``:: +1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``: - >>> response.css('a::attr(href)').get() - 'image1.html' - >>> response.css('a::attr(href)').extract_first() - 'image1.html' + >>> response.css('a::attr(href)').get() + 'image1.html' + >>> response.css('a::attr(href)').extract_first() + 'image1.html' -2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``:: +2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``: - >>> response.css('a::attr(href)').getall() - ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] - >>> response.css('a::attr(href)').extract() - ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] + >>> response.css('a::attr(href)').getall() + ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] + >>> response.css('a::attr(href)').extract() + ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] -3. ``Selector.get()`` is the same as ``Selector.extract()``:: +3. ``Selector.get()`` is the same as ``Selector.extract()``: - >>> response.css('a::attr(href)')[0].get() - 'image1.html' - >>> response.css('a::attr(href)')[0].extract() - 'image1.html' + >>> response.css('a::attr(href)')[0].get() + 'image1.html' + >>> response.css('a::attr(href)')[0].extract() + 'image1.html' -4. For consistency, there is also ``Selector.getall()``, which returns a list:: +4. For consistency, there is also ``Selector.getall()``, which returns a list: >>> response.css('a::attr(href)')[0].getall() ['image1.html'] @@ -481,26 +480,26 @@ with ``/``, that XPath will be absolute to the document and not relative to the ``Selector`` you're calling it from. For example, suppose you want to extract all ``<p>`` elements inside ``<div>`` -elements. First, you would get all ``<div>`` elements:: +elements. First, you would get all ``<div>`` elements: - >>> divs = response.xpath('//div') +>>> divs = response.xpath('//div') At first, you may be tempted to use the following approach, which is wrong, as it actually extracts all ``<p>`` elements from the document, not only those -inside ``<div>`` elements:: +inside ``<div>`` elements: - >>> for p in divs.xpath('//p'): # this is wrong - gets all <p> from the whole document - ... print(p.get()) +>>> for p in divs.xpath('//p'): # this is wrong - gets all <p> from the whole document +... print(p.get()) -This is the proper way to do it (note the dot prefixing the ``.//p`` XPath):: +This is the proper way to do it (note the dot prefixing the ``.//p`` XPath): - >>> for p in divs.xpath('.//p'): # extracts all <p> inside - ... print(p.get()) +>>> for p in divs.xpath('.//p'): # extracts all <p> inside +... print(p.get()) -Another common case would be to extract all direct ``<p>`` children:: +Another common case would be to extract all direct ``<p>`` children: - >>> for p in divs.xpath('p'): - ... print(p.get()) +>>> for p in divs.xpath('p'): +... print(p.get()) For more details about relative XPaths see the `Location Paths`_ section in the XPath specification. @@ -521,12 +520,12 @@ for that you may end up with more elements that you want, if they have a differe class name that shares the string ``someclass``. As it turns out, Scrapy selectors allow you to chain selectors, so most of the time -you can just select by class using CSS and then switch to XPath when needed:: +you can just select by class using CSS and then switch to XPath when needed: - >>> from scrapy import Selector - >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>') - >>> sel.css('.shout').xpath('./time/@datetime').getall() - ['2014-07-23 19:00'] +>>> from scrapy import Selector +>>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>') +>>> sel.css('.shout').xpath('./time/@datetime').getall() +['2014-07-23 19:00'] This is cleaner than using the verbose XPath trick shown above. Just remember to use the ``.`` in the XPath expressions that will follow. @@ -538,41 +537,41 @@ Beware of the difference between //node[1] and (//node)[1] ``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them. -Example:: +Example: - >>> from scrapy import Selector - >>> sel = Selector(text=""" - ....: <ul class="list"> - ....: <li>1</li> - ....: <li>2</li> - ....: <li>3</li> - ....: </ul> - ....: <ul class="list"> - ....: <li>4</li> - ....: <li>5</li> - ....: <li>6</li> - ....: </ul>""") - >>> xp = lambda x: sel.xpath(x).getall() +>>> from scrapy import Selector +>>> sel = Selector(text=""" +....: <ul class="list"> +....: <li>1</li> +....: <li>2</li> +....: <li>3</li> +....: </ul> +....: <ul class="list"> +....: <li>4</li> +....: <li>5</li> +....: <li>6</li> +....: </ul>""") +>>> xp = lambda x: sel.xpath(x).getall() -This gets all first ``<li>`` elements under whatever it is its parent:: +This gets all first ``<li>`` elements under whatever it is its parent: - >>> xp("//li[1]") - ['<li>1</li>', '<li>4</li>'] +>>> xp("//li[1]") +['<li>1</li>', '<li>4</li>'] -And this gets the first ``<li>`` element in the whole document:: +And this gets the first ``<li>`` element in the whole document: - >>> xp("(//li)[1]") - ['<li>1</li>'] +>>> xp("(//li)[1]") +['<li>1</li>'] -This gets all first ``<li>`` elements under an ``<ul>`` parent:: +This gets all first ``<li>`` elements under an ``<ul>`` parent: - >>> xp("//ul/li[1]") - ['<li>1</li>', '<li>4</li>'] +>>> xp("//ul/li[1]") +['<li>1</li>', '<li>4</li>'] -And this gets the first ``<li>`` element under an ``<ul>`` parent in the whole document:: +And this gets the first ``<li>`` element under an ``<ul>`` parent in the whole document: - >>> xp("(//ul/li)[1]") - ['<li>1</li>'] +>>> xp("(//ul/li)[1]") +['<li>1</li>'] Using text nodes in a condition ------------------------------- @@ -584,34 +583,34 @@ This is because the expression ``.//text()`` yields a collection of text element And when a node-set is converted to a string, which happens when it is passed as argument to a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only. -Example:: +Example: - >>> from scrapy import Selector - >>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>') +>>> from scrapy import Selector +>>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>') -Converting a *node-set* to string:: +Converting a *node-set* to string: - >>> sel.xpath('//a//text()').getall() # take a peek at the node-set - ['Click here to go to the ', 'Next Page'] - >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string - ['Click here to go to the '] +>>> sel.xpath('//a//text()').getall() # take a peek at the node-set +['Click here to go to the ', 'Next Page'] +>>> sel.xpath("string(//a[1]//text())").getall() # convert it to string +['Click here to go to the '] -A *node* converted to a string, however, puts together the text of itself plus of all its descendants:: +A *node* converted to a string, however, puts together the text of itself plus of all its descendants: - >>> sel.xpath("//a[1]").getall() # select the first node - ['<a href="#">Click here to go to the <strong>Next Page</strong></a>'] - >>> sel.xpath("string(//a[1])").getall() # convert it to string - ['Click here to go to the Next Page'] +>>> sel.xpath("//a[1]").getall() # select the first node +['<a href="#">Click here to go to the <strong>Next Page</strong></a>'] +>>> sel.xpath("string(//a[1])").getall() # convert it to string +['Click here to go to the Next Page'] -So, using the ``.//text()`` node-set won't select anything in this case:: +So, using the ``.//text()`` node-set won't select anything in this case: - >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall() - [] +>>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall() +[] -But using the ``.`` to mean the node, works:: +But using the ``.`` to mean the node, works: - >>> sel.xpath("//a[contains(., 'Next Page')]").getall() - ['<a href="#">Click here to go to the <strong>Next Page</strong></a>'] +>>> sel.xpath("//a[contains(., 'Next Page')]").getall() +['<a href="#">Click here to go to the <strong>Next Page</strong></a>'] .. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions @@ -627,17 +626,17 @@ some arguments in your queries with placeholders like ``?``, which are then substituted with values passed with the query. Here's an example to match an element based on its "id" attribute value, -without hard-coding it (that was shown previously):: +without hard-coding it (that was shown previously): - >>> # `$val` used in the expression, a `val` argument needs to be passed - >>> response.xpath('//div[@id=$val]/a/text()', val='images').get() - 'Name: My image 1 ' +>>> # `$val` used in the expression, a `val` argument needs to be passed +>>> response.xpath('//div[@id=$val]/a/text()', val='images').get() +'Name: My image 1 ' Here's another example, to find the "id" attribute of a ``<div>`` tag containing -five ``<a>`` children (here we pass the value ``5`` as an integer):: +five ``<a>`` children (here we pass the value ``5`` as an integer): - >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get() - 'images' +>>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get() +'images' All variable references must have a binding value when calling ``.xpath()`` (otherwise you'll get a ``ValueError: XPath error:`` exception). @@ -687,19 +686,19 @@ You can see several namespace declarations including a default .. highlight:: python Once in the shell we can try selecting all ``<link>`` objects and see that it -doesn't work (because the Atom XML namespace is obfuscating those nodes):: +doesn't work (because the Atom XML namespace is obfuscating those nodes): - >>> response.xpath("//link") - [] +>>> response.xpath("//link") +[] But once we call the :meth:`Selector.remove_namespaces` method, all -nodes can be accessed directly by their names:: +nodes can be accessed directly by their names: - >>> response.selector.remove_namespaces() - >>> response.xpath("//link") - [<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>, - <Selector xpath='//link' data='<link rel="next" type="application/atom+'>, - ... +>>> response.selector.remove_namespaces() +>>> response.xpath("//link") +[<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>, + <Selector xpath='//link' data='<link rel="next" type="application/atom+'>, + ... If you wonder why the namespace removal procedure isn't always called by default instead of having to call it manually, this is because of two reasons, which, in order @@ -734,26 +733,25 @@ Regular expressions The ``test()`` function, for example, can prove quite useful when XPath's ``starts-with()`` or ``contains()`` are not sufficient. -Example selecting links in list item with a "class" attribute ending with a digit:: +Example selecting links in list item with a "class" attribute ending with a digit: - >>> from scrapy import Selector - >>> doc = u""" - ... <div> - ... <ul> - ... <li class="item-0"><a href="link1.html">first item</a></li> - ... <li class="item-1"><a href="link2.html">second item</a></li> - ... <li class="item-inactive"><a href="link3.html">third item</a></li> - ... <li class="item-1"><a href="link4.html">fourth item</a></li> - ... <li class="item-0"><a href="link5.html">fifth item</a></li> - ... </ul> - ... </div> - ... """ - >>> sel = Selector(text=doc, type="html") - >>> sel.xpath('//li//@href').getall() - ['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html'] - >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall() - ['link1.html', 'link2.html', 'link4.html', 'link5.html'] - >>> +>>> from scrapy import Selector +>>> doc = u""" +... <div> +... <ul> +... <li class="item-0"><a href="link1.html">first item</a></li> +... <li class="item-1"><a href="link2.html">second item</a></li> +... <li class="item-inactive"><a href="link3.html">third item</a></li> +... <li class="item-1"><a href="link4.html">fourth item</a></li> +... <li class="item-0"><a href="link5.html">fifth item</a></li> +... </ul> +... </div> +... """ +>>> sel = Selector(text=doc, type="html") +>>> sel.xpath('//li//@href').getall() +['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html'] +>>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall() +['link1.html', 'link2.html', 'link4.html', 'link5.html'] .. warning:: C library ``libxslt`` doesn't natively support EXSLT regular expressions so `lxml`_'s implementation uses hooks to Python's ``re`` module. @@ -849,7 +847,6 @@ with groups of itemscopes and corresponding itemprops:: current scope: ['http://schema.org/Rating'] properties: ['worstRating', 'ratingValue', 'bestRating'] - >>> Here we first iterate over ``itemscope`` elements, and for each one, we look for all ``itemprops`` elements and exclude those that are themselves @@ -877,15 +874,15 @@ For the following HTML:: .. highlight:: python -You can use it like this:: +You can use it like this: - >>> response.xpath('//p[has-class("foo")]') - [<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>, - <Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>] - >>> response.xpath('//p[has-class("foo", "bar-baz")]') - [<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>] - >>> response.xpath('//p[has-class("foo", "bar")]') - [] +>>> response.xpath('//p[has-class("foo")]') +[<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>, + <Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>] +>>> response.xpath('//p[has-class("foo", "bar-baz")]') +[<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>] +>>> response.xpath('//p[has-class("foo", "bar")]') +[] So XPath ``//p[has-class("foo", "bar-baz")]`` is roughly equivalent to CSS ``p.foo.bar-baz``. Please note, that it is slower in most of the cases, diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 68a0b19b5..c1fdfd221 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -177,47 +177,46 @@ all start with the ``[s]`` prefix):: >>> -After that, we can start playing with the objects:: +After that, we can start playing with the objects: - >>> response.xpath('//title/text()').get() - 'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' +>>> response.xpath('//title/text()').get() +'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' - >>> fetch("https://reddit.com") +>>> fetch("https://reddit.com") - >>> response.xpath('//title/text()').get() - 'reddit: the front page of the internet' +>>> response.xpath('//title/text()').get() +'reddit: the front page of the internet' - >>> request = request.replace(method="POST") +>>> request = request.replace(method="POST") - >>> fetch(request) +>>> fetch(request) - >>> response.status - 404 +>>> response.status +404 - >>> from pprint import pprint +>>> from pprint import pprint - >>> pprint(response.headers) - {'Accept-Ranges': ['bytes'], - 'Cache-Control': ['max-age=0, must-revalidate'], - 'Content-Type': ['text/html; charset=UTF-8'], - 'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'], - 'Server': ['snooserv'], - 'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', - 'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', - 'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', - 'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'], - 'Vary': ['accept-encoding'], - 'Via': ['1.1 varnish'], - 'X-Cache': ['MISS'], - 'X-Cache-Hits': ['0'], - 'X-Content-Type-Options': ['nosniff'], - 'X-Frame-Options': ['SAMEORIGIN'], - 'X-Moose': ['majestic'], - 'X-Served-By': ['cache-cdg8730-CDG'], - 'X-Timer': ['S1481214079.394283,VS0,VE159'], - 'X-Ua-Compatible': ['IE=edge'], - 'X-Xss-Protection': ['1; mode=block']} - >>> +>>> pprint(response.headers) +{'Accept-Ranges': ['bytes'], + 'Cache-Control': ['max-age=0, must-revalidate'], + 'Content-Type': ['text/html; charset=UTF-8'], + 'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'], + 'Server': ['snooserv'], + 'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'], + 'Vary': ['accept-encoding'], + 'Via': ['1.1 varnish'], + 'X-Cache': ['MISS'], + 'X-Cache-Hits': ['0'], + 'X-Content-Type-Options': ['nosniff'], + 'X-Frame-Options': ['SAMEORIGIN'], + 'X-Moose': ['majestic'], + 'X-Served-By': ['cache-cdg8730-CDG'], + 'X-Timer': ['S1481214079.394283,VS0,VE159'], + 'X-Ua-Compatible': ['IE=edge'], + 'X-Xss-Protection': ['1; mode=block']} .. _topics-shell-inspect-response: @@ -263,16 +262,16 @@ When you run the spider, you will get something similar to this:: >>> response.url 'http://example.org' -Then, you can check if the extraction code is working:: +Then, you can check if the extraction code is working: - >>> response.xpath('//h1[@class="fn"]') - [] +>>> response.xpath('//h1[@class="fn"]') +[] Nope, it doesn't. So you can open the response in your web browser and see if -it's the response you were expecting:: +it's the response you were expecting: - >>> view(response) - True +>>> view(response) +True Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the crawling:: diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 38648ec55..3dd829ebe 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -57,15 +57,15 @@ Set stat value only if lower than previous:: stats.min_value('min_free_memory_percent', value) -Get stat value:: +Get stat value: - >>> stats.get_value('custom_count') - 1 +>>> stats.get_value('custom_count') +1 -Get all stats:: +Get all stats: - >>> stats.get_stats() - {'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} +>>> stats.get_stats() +{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} Available Stats Collectors ========================== From 07b8cd28aa84fb322a072467b49e2557d4aa3881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 5 Dec 2019 14:48:31 +0100 Subject: [PATCH 61/85] =?UTF-8?q?Mark=20bandit=E2=80=99s=20402=20check=20a?= =?UTF-8?q?s=20addressed=20by=20#4180=20(#4181)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bandit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bandit.yml b/.bandit.yml index cc7db3a66..243379b0b 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -8,7 +8,7 @@ skips: - B311 - B320 - B321 -- B402 +- B402 # https://github.com/scrapy/scrapy/issues/4180 - B403 - B404 - B406 From 02cdc53fb82e3cc5e51771180f1f79186b52670a Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin <wrar@wrar.name> Date: Fri, 13 Dec 2019 18:04:05 +0500 Subject: [PATCH 62/85] Add a test for a CrawlerProcess script. (#4218) * Add a test for a CrawlerProcess script. * Add tests/CrawlerProcess to collect_ignore. * Remove an extra line. * Fix/improve conftest.py. --- conftest.py | 9 ++++++++- tests/CrawlerProcess/simple.py | 15 +++++++++++++++ tests/test_crawler.py | 20 ++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 tests/CrawlerProcess/simple.py diff --git a/conftest.py b/conftest.py index d54ce155c..d37c22436 100644 --- a/conftest.py +++ b/conftest.py @@ -1,12 +1,19 @@ +from pathlib import Path + import pytest +def _py_files(folder): + return (str(p) for p in Path(folder).rglob('*.py')) + + collect_ignore = [ # not a test, but looks like a test "scrapy/utils/testsite.py", + # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess + *_py_files("tests/CrawlerProcess") ] - for line in open('tests/ignores.txt'): file_path = line.strip() if file_path and file_path[0] != '#': diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py new file mode 100644 index 000000000..5f6f1ae30 --- /dev/null +++ b/tests/CrawlerProcess/simple.py @@ -0,0 +1,15 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 8eb2389e2..e37a2ff0e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,4 +1,7 @@ import logging +import os +import subprocess +import sys import warnings from twisted.internet import defer @@ -14,6 +17,7 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet +from scrapy.utils.test import get_testenv class BaseCrawlerTest(unittest.TestCase): @@ -245,3 +249,19 @@ class CrawlerRunnerHasSpider(unittest.TestCase): yield runner.crawl(NoRequestsSpider) self.assertEqual(runner.bootstrap_failed, True) + + +class CrawlerProcessSubprocess(unittest.TestCase): + script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess') + + def run_script(self, script_name): + script_path = os.path.join(self.script_dir, script_name) + args = (sys.executable, script_path) + p = subprocess.Popen(args, env=get_testenv(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + return stderr.decode('utf-8') + + def test_simple(self): + log = self.run_script('simple.py') + self.assertIn('Spider closed (finished)', log) From b5c4c2cae89714479d6fbb5a497fe43d48d886bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Fri, 13 Dec 2019 14:20:48 +0100 Subject: [PATCH 63/85] Keep 2 spaces between code and inline comments (#4195) --- pytest.ini | 35 +++++++++++++------------- scrapy/core/downloader/webclient.py | 2 +- scrapy/core/engine.py | 2 +- scrapy/core/scraper.py | 2 +- scrapy/exporters.py | 2 +- scrapy/settings/default_settings.py | 4 +-- scrapy/spiders/feed.py | 4 +-- scrapy/utils/console.py | 8 +++--- tests/test_command_fetch.py | 2 +- tests/test_downloader_handlers.py | 2 +- tests/test_engine.py | 1 - tests/test_http_request.py | 2 +- tests/test_pipeline_media.py | 6 ++--- tests/test_spidermiddleware_referer.py | 20 +++++++-------- tests/test_utils_defer.py | 8 +++--- tests/test_utils_spider.py | 13 ++++------ 16 files changed, 54 insertions(+), 59 deletions(-) diff --git a/pytest.ini b/pytest.ini index 33c34b8e8..1b23595c0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -26,8 +26,8 @@ flake8-ignore = scrapy/http/__init__.py F401 # Issues pending a review: # extras - extras/qps-bench-server.py E261 E501 - extras/qpsclient.py E501 E261 E501 + extras/qps-bench-server.py E501 + extras/qpsclient.py E501 E501 # scrapy/commands scrapy/commands/__init__.py E128 E501 scrapy/commands/check.py E501 @@ -45,15 +45,15 @@ flake8-ignore = scrapy/contracts/__init__.py E501 W504 scrapy/contracts/default.py E502 E128 # scrapy/core - scrapy/core/engine.py E261 E501 E128 E127 E306 E502 + scrapy/core/engine.py E501 E128 E127 E306 E502 scrapy/core/scheduler.py E501 - scrapy/core/scraper.py E501 E306 E261 E128 W504 + scrapy/core/scraper.py E501 E306 E128 W504 scrapy/core/spidermw.py E501 E731 E502 E126 E226 scrapy/core/downloader/__init__.py E501 scrapy/core/downloader/contextfactory.py E501 E128 E126 scrapy/core/downloader/middleware.py E501 E502 scrapy/core/downloader/tls.py E501 E305 E241 - scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226 + scrapy/core/downloader/webclient.py E731 E501 E502 E128 E126 E226 scrapy/core/downloader/handlers/__init__.py E501 scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127 scrapy/core/downloader/handlers/http10.py E501 @@ -102,7 +102,7 @@ flake8-ignore = scrapy/selector/unified.py E501 E111 # scrapy/settings scrapy/settings/__init__.py E501 - scrapy/settings/default_settings.py E501 E261 E114 E116 E226 + scrapy/settings/default_settings.py E501 E114 E116 E226 scrapy/settings/deprecated.py E501 # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 @@ -112,17 +112,16 @@ flake8-ignore = # scrapy/spiders scrapy/spiders/__init__.py E501 E402 scrapy/spiders/crawl.py E501 - scrapy/spiders/feed.py E501 E261 + scrapy/spiders/feed.py E501 scrapy/spiders/sitemap.py E501 # scrapy/utils scrapy/utils/benchserver.py E501 scrapy/utils/conf.py E402 E502 E501 - scrapy/utils/console.py E261 E306 E305 + scrapy/utils/console.py E306 E305 scrapy/utils/datatypes.py E501 E226 scrapy/utils/decorators.py E501 scrapy/utils/defer.py E501 E128 scrapy/utils/deprecate.py E128 E501 E127 E502 - scrapy/utils/engine.py E261 scrapy/utils/gz.py E305 E501 W504 scrapy/utils/http.py F403 E226 scrapy/utils/httpobj.py E501 @@ -150,7 +149,7 @@ flake8-ignore = scrapy/crawler.py E501 scrapy/dupefilters.py E501 E202 scrapy/exceptions.py E501 - scrapy/exporters.py E501 E261 E226 + scrapy/exporters.py E501 E226 scrapy/interfaces.py E501 scrapy/item.py E501 E128 scrapy/link.py E501 @@ -171,7 +170,7 @@ flake8-ignore = tests/pipelines.py F841 E226 tests/spiders.py E501 E127 tests/test_closespider.py E501 E127 - tests/test_command_fetch.py E501 E261 + tests/test_command_fetch.py E501 tests/test_command_parse.py E501 E128 E303 E226 tests/test_command_shell.py E501 E128 tests/test_commands.py E128 E501 @@ -179,7 +178,7 @@ flake8-ignore = tests/test_crawl.py E501 E741 E265 tests/test_crawler.py F841 E306 E501 tests/test_dependencies.py F841 E501 E305 - tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123 + tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E502 E701 E126 E226 E123 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126 @@ -194,13 +193,13 @@ flake8-ignore = tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124 - tests/test_engine.py E401 E501 E502 E128 E261 + tests/test_engine.py E401 E501 E502 E128 tests/test_exporters.py E501 E731 E306 E128 E124 tests/test_extension_telnet.py F841 tests/test_feedexport.py E501 F841 E241 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 - tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123 + tests/test_http_request.py E402 E501 E127 E128 W293 E502 E128 E502 E126 E123 tests/test_http_response.py E501 E301 E502 E128 E265 tests/test_item.py E701 E128 F841 E306 tests/test_link.py E501 @@ -212,7 +211,7 @@ flake8-ignore = tests/test_pipeline_crawl.py E131 E501 E128 E126 tests/test_pipeline_files.py E501 W293 E303 E272 E226 tests/test_pipeline_images.py F841 E501 E303 - tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502 + tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502 tests/test_proxy_connect.py E501 E741 tests/test_request_cb_kwargs.py E501 tests/test_responsetypes.py E501 E305 @@ -224,12 +223,12 @@ flake8-ignore = tests/test_spidermiddleware_httperror.py E128 E501 E127 E121 tests/test_spidermiddleware_offsite.py E501 E128 E111 W293 tests/test_spidermiddleware_output_chain.py E501 W293 E226 - tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121 + tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121 tests/test_squeues.py E501 E701 E741 tests/test_utils_conf.py E501 E303 E128 tests/test_utils_curl.py E501 tests/test_utils_datatypes.py E402 E501 E305 - tests/test_utils_defer.py E306 E261 E501 F841 E226 + tests/test_utils_defer.py E306 E501 F841 E226 tests/test_utils_deprecate.py F841 E306 E501 tests/test_utils_http.py E501 E502 E128 W504 tests/test_utils_iterators.py E501 E128 E129 E303 E241 @@ -240,7 +239,7 @@ flake8-ignore = tests/test_utils_response.py E501 tests/test_utils_signal.py E741 F841 E731 E226 tests/test_utils_sitemap.py E128 E501 E124 - tests/test_utils_spider.py E261 E305 + tests/test_utils_spider.py E305 tests/test_utils_template.py E305 tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123 tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126 diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 798346f19..f368c3bae 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -42,7 +42,7 @@ class ScrapyHTTPPageGetter(HTTPClient): delimiter = b'\n' def connectionMade(self): - self.headers = Headers() # bucket for response headers + self.headers = Headers() # bucket for response headers # Method command self.sendCommand(self.factory.method, self.factory.path) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index fa913e528..829e69993 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -25,7 +25,7 @@ class Slot(object): def __init__(self, start_requests, close_if_idle, nextcall, scheduler): self.closing = False - self.inprogress = set() # requests in progress + self.inprogress = set() # requests in progress self.start_requests = iter(start_requests) self.close_if_idle = close_if_idle self.nextcall = nextcall diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index db463f989..b3d585cce 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -123,7 +123,7 @@ class Scraper(object): callback/errback""" assert isinstance(response, (Response, Failure)) - dfd = self._scrape2(response, request, spider) # returns spiders processed output + dfd = self._scrape2(response, request, spider) # returns spiders processed output dfd.addErrback(self.handle_spider_error, request, response, spider) dfd.addCallback(self.handle_spider_output, request, response, spider) return dfd diff --git a/scrapy/exporters.py b/scrapy/exporters.py index fcb55da67..5bf131312 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -199,7 +199,7 @@ class CsvItemExporter(BaseItemExporter): line_buffering=False, write_through=True, encoding=self.encoding, - newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 + newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 ) self.csv_writer = csv.writer(self.stream, **kwargs) self._headers_not_written = True diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 5c9678c01..1e163e1fb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -84,8 +84,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader' DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory' DOWNLOADER_CLIENT_TLS_CIPHERS = 'DEFAULT' -DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform, - # also allowing negotiation +# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation: +DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False DOWNLOADER_MIDDLEWARES = {} diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 197812a26..c566f0236 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -100,8 +100,8 @@ class CSVFeedSpider(Spider): and the file's headers. """ - delimiter = None # When this is None, python's csv module's default delimiter is used - quotechar = None # When this is None, python's csv module's default quotechar is used + delimiter = None # When this is None, python's csv module's default delimiter is used + quotechar = None # When this is None, python's csv module's default quotechar is used headers = None def process_results(self, response, results): diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 688e28c34..7eb40f0ce 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -47,7 +47,7 @@ def _embed_ptpython_shell(namespace={}, banner=''): def _embed_standard_shell(namespace={}, banner=''): """Start a standard python shell""" import code - try: # readline module is only available on unix systems + try: # readline module is only available on unix systems import readline except ImportError: pass @@ -72,9 +72,9 @@ def get_shell_embed_func(shells=None, known_shells=None): """Return the first acceptable shell-embed function from a given list of shell names. """ - if shells is None: # list, preference order of shells + if shells is None: # list, preference order of shells shells = DEFAULT_PYTHON_SHELLS.keys() - if known_shells is None: # available embeddable shells + if known_shells is None: # available embeddable shells known_shells = DEFAULT_PYTHON_SHELLS.copy() for shell in shells: if shell in known_shells: @@ -97,5 +97,5 @@ def start_python_console(namespace=None, banner='', shells=None): shell = get_shell_embed_func(shells) if shell is not None: shell(namespace=namespace, banner=banner) - except SystemExit: # raised when using exit() in python code.interact + except SystemExit: # raised when using exit() in python code.interact pass diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 3fa3ed930..9d3c8fe73 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -29,6 +29,6 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url('/text'), '--headers']) - out = out.replace(b'\r', b'') # required on win32 + out = out.replace(b'\r', b'') # required on win32 assert b'Server: TwistedWeb' in out, out assert b'Content-Type: text/plain' in out diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 60124b93f..2db2417e8 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -349,7 +349,7 @@ class HttpTestCase(unittest.TestCase): return self.download_request(request, Spider('foo')).addCallback(_test) def test_payload(self): - body = b'1'*100 # PayloadResource requires body length to be 100 + body = b'1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) diff --git a/tests/test_engine.py b/tests/test_engine.py index 002c4e6bb..537df8d91 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -271,7 +271,6 @@ class EngineTest(unittest.TestCase): self.run.signals_catched[signals.spider_opened]) self.assertEqual({'spider': self.run.spider}, self.run.signals_catched[signals.spider_idle]) - self.run.signals_catched[signals.spider_closed].pop('spider_stats', None) # XXX: remove for scrapy 0.17 self.assertEqual({'spider': self.run.spider, 'reason': 'finished'}, self.run.signals_catched[signals.spider_closed]) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index a98aa1e6f..57e7b457d 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -149,7 +149,7 @@ class RequestTest(unittest.TestCase): r2 = self.request_class(url="http://www.example.com/", body=b"") assert isinstance(r2.body, bytes) - self.assertEqual(r2.encoding, 'utf-8') # default encoding + self.assertEqual(r2.encoding, 'utf-8') # default encoding r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8') assert isinstance(r3.body, bytes) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 0d23f51cc..1fcc5799e 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -238,10 +238,10 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): self.assertEqual(new_item['results'], [(True, rsp1), (False, fail)]) m = self.pipe._mockcalled # only once - self.assertEqual(m[0], 'get_media_requests') # first hook called + self.assertEqual(m[0], 'get_media_requests') # first hook called self.assertEqual(m.count('get_media_requests'), 1) self.assertEqual(m.count('item_completed'), 1) - self.assertEqual(m[-1], 'item_completed') # last hook called + self.assertEqual(m[-1], 'item_completed') # last hook called # twice, one per request self.assertEqual(m.count('media_to_download'), 2) # one to handle success and other for failure @@ -252,7 +252,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def test_get_media_requests(self): # returns single Request (without callback) req = Request('http://url') - item = dict(requests=req) # pass a single item + item = dict(requests=req) # pass a single item new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item assert request_fingerprint(req) in self.info.downloaded diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index ecec6135d..7cc17600c 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -548,8 +548,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware): (301, 'http://scrapytest.org/3'), (301, 'http://scrapytest.org/4'), ), - b'http://scrapytest.org/1', # expected initial referer - b'http://scrapytest.org/1', # expected referer for the redirection request + b'http://scrapytest.org/1', # expected initial referer + b'http://scrapytest.org/1', # expected referer for the redirection request ), ( 'https://scrapytest.org/1', 'https://scrapytest.org/2', @@ -609,8 +609,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): (301, 'http://scrapytest.org/3'), (301, 'http://scrapytest.org/4'), ), - None, # expected initial "Referer" - None, # expected "Referer" for the redirection request + None, # expected initial "Referer" + None, # expected "Referer" for the redirection request ), ( 'https://scrapytest.org/1', 'https://scrapytest.org/2', @@ -648,8 +648,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect): (301, 'http://scrapytest.org/103'), (301, 'http://scrapytest.org/104'), ), - b'http://scrapytest.org/101', # expected initial "Referer" - b'http://scrapytest.org/101', # expected referer for the redirection request + b'http://scrapytest.org/101', # expected initial "Referer" + b'http://scrapytest.org/101', # expected referer for the redirection request ), ( 'https://scrapytest.org/201', 'https://scrapytest.org/202', @@ -757,8 +757,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect): (301, 'http://scrapytest.org/103'), (301, 'http://scrapytest.org/104'), ), - b'http://scrapytest.org/101', # expected initial referer - b'http://scrapytest.org/101', # expected referer for the redirection request + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request ), ( 'https://scrapytest.org/201', 'https://scrapytest.org/202', @@ -827,8 +827,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect): (301, 'http://scrapytest.org/103'), (301, 'http://scrapytest.org/104'), ), - b'http://scrapytest.org/101', # expected initial referer - b'http://scrapytest.org/101', # expected referer for the redirection request + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request ), ( 'https://scrapytest.org/201', 'https://scrapytest.org/202', diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 49c2befb5..dfbe71ae2 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -14,8 +14,8 @@ class MustbeDeferredTest(unittest.TestCase): return steps dfd = mustbe_deferred(_append, 1) - dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred - steps.append(2) # add another value, that should be catched by assertEqual + dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred + steps.append(2) # add another value, that should be catched by assertEqual return dfd def test_unfired_deferred(self): @@ -27,8 +27,8 @@ class MustbeDeferredTest(unittest.TestCase): return dfd dfd = mustbe_deferred(_append, 1) - dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred - steps.append(2) # add another value, that should be catched by assertEqual + dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred + steps.append(2) # add another value, that should be catched by assertEqual return dfd diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index edeeacc80..ee7d17062 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -1,20 +1,16 @@ import unittest + +from scrapy import Spider from scrapy.http import Request from scrapy.item import BaseItem from scrapy.utils.spider import iterate_spider_output, iter_spider_classes -from scrapy.spiders import CrawlSpider - -class MyBaseSpider(CrawlSpider): - pass # abstract spider - - -class MySpider1(MyBaseSpider): +class MySpider1(Spider): name = 'myspider1' -class MySpider2(MyBaseSpider): +class MySpider2(Spider): name = 'myspider2' @@ -35,5 +31,6 @@ class UtilsSpidersTestCase(unittest.TestCase): it = iter_spider_classes(tests.test_utils_spider) self.assertEqual(set(it), {MySpider1, MySpider2}) + if __name__ == "__main__": unittest.main() From a4ef9750f9058fbe1041bad0fb28f39c693e5659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Fri, 13 Dec 2019 14:32:06 +0100 Subject: [PATCH 64/85] Fix Flake8-reported issues --- pytest.ini | 2 +- tests/test_linkextractors.py | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pytest.ini b/pytest.ini index 33c34b8e8..02014d1a5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -88,7 +88,7 @@ flake8-ignore = scrapy/http/response/__init__.py E501 E128 W293 W291 scrapy/http/response/text.py E501 W293 E128 E124 # scrapy/linkextractors - scrapy/linkextractors/__init__.py E731 E502 E501 E402 + scrapy/linkextractors/__init__.py E731 E502 E501 E402 W504 scrapy/linkextractors/lxmlhtml.py E501 E731 E226 # scrapy/loader scrapy/loader/__init__.py E501 E502 E128 diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index ebe497913..0ffeaecc3 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -514,25 +514,27 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): """Make sure the FilteringLinkExtractor deprecation warning is not issued for LxmlLinkExtractor""" with catch_warnings(record=True) as warnings: - extractor = LxmlLinkExtractor() + LxmlLinkExtractor() self.assertEqual(len(warnings), 0) + class SubclassedItem(LxmlLinkExtractor): pass - subclassed_extractor = SubclassedItem() + + SubclassedItem() self.assertEqual(len(warnings), 0) class FilteringLinkExtractorTest(unittest.TestCase): def test_deprecation_warning(self): - args = [None]*10 + args = [None] * 10 with catch_warnings(record=True) as warnings: - extractor = FilteringLinkExtractor(*args) + FilteringLinkExtractor(*args) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) with catch_warnings(record=True) as warnings: class SubclassedFilteringLinkExtractor(FilteringLinkExtractor): pass - subclassed_extractor = SubclassedFilteringLinkExtractor(*args) + SubclassedFilteringLinkExtractor(*args) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) From e3a3ad4aaff6d07252bb4f65be4667e21070aea1 Mon Sep 17 00:00:00 2001 From: marc <Marc> Date: Sat, 14 Dec 2019 10:34:31 +0100 Subject: [PATCH 65/85] remove reference to old (Python 2.7) environment --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 234c4bcee..eaaf86c29 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -229,7 +229,7 @@ You can also specify a comma-separated list of environmets, and use :ref:`tox’ parallel mode <tox:parallel_mode>` to run the tests on multiple environments in parallel:: - tox -e py27,py36 -p auto + tox -e py36,py38 -p auto To pass command-line options to :doc:`pytest <pytest:index>`, add them after ``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the From 1aab20e1cec9e2e9aac4ce9efd37b07811b8f7d6 Mon Sep 17 00:00:00 2001 From: marc <Marc> Date: Sat, 14 Dec 2019 10:37:31 +0100 Subject: [PATCH 66/85] update copyright notice year --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 914d1d05f..5fdaa4e4c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -50,7 +50,7 @@ master_doc = 'index' # General information about the project. project = u'Scrapy' -copyright = u'2008–2018, Scrapy developers' +copyright = u'2008–2020, Scrapy developers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From a59bb279d18966e6a47f71d21c690598fc69678e Mon Sep 17 00:00:00 2001 From: marc <Marc> Date: Sun, 15 Dec 2019 17:33:00 +0100 Subject: [PATCH 67/85] add year through code --- docs/conf.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 5fdaa4e4c..ed56c5cd1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ # serve to show the default. import sys +from datetime import datetime from os import path # If your extensions are in another directory, add it here. If the directory @@ -49,8 +50,8 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'Scrapy' -copyright = u'2008–2020, Scrapy developers' +project = 'Scrapy' +copyright = '2008–{}, Scrapy developers'.format(datetime.now().year) # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -194,8 +195,8 @@ htmlhelp_basename = 'Scrapydoc' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). latex_documents = [ - ('index', 'Scrapy.tex', u'Scrapy Documentation', - u'Scrapy developers', 'manual'), + ('index', 'Scrapy.tex', 'Scrapy Documentation', + 'Scrapy developers', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of From 451e7a616e6d23e88528836296a9abdc887834c6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta <eugenio.lacuesta@gmail.com> Date: Fri, 12 Jul 2019 00:26:01 -0300 Subject: [PATCH 68/85] Scan callbacks/errbacks for return statements with values different than None --- scrapy/core/scraper.py | 19 ++++-- scrapy/utils/datatypes.py | 31 ++++++++- scrapy/utils/misc.py | 46 +++++++++++++ tests/test_utils_datatypes.py | 66 ++++++++++++++++++- ...t_return_with_argument_inside_generator.py | 37 +++++++++++ 5 files changed, 190 insertions(+), 9 deletions(-) create mode 100644 tests/test_utils_misc/test_return_with_argument_inside_generator.py diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index b3d585cce..99114d3bb 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -9,7 +9,7 @@ from twisted.internet import defer from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback from scrapy.utils.spider import iterate_spider_output -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.log import logformatter_adapter, failure_to_exc_info from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest from scrapy import signals @@ -18,6 +18,7 @@ from scrapy.item import BaseItem from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.utils.request import referer_str + logger = logging.getLogger(__name__) @@ -99,11 +100,13 @@ class Scraper(object): def enqueue_scrape(self, response, request, spider): slot = self.slot dfd = slot.add_response_request(response, request) + def finish_scraping(_): slot.finish_response(response, request) self._check_if_closing(spider, slot) self._scrape_next(spider, slot) return _ + dfd.addBoth(finish_scraping) dfd.addErrback( lambda f: logger.error('Scraper bug processing %(request)s', @@ -123,7 +126,7 @@ class Scraper(object): callback/errback""" assert isinstance(response, (Response, Failure)) - dfd = self._scrape2(response, request, spider) # returns spiders processed output + dfd = self._scrape2(response, request, spider) # returns spider's processed output dfd.addErrback(self.handle_spider_error, request, response, spider) dfd.addCallback(self.handle_spider_output, request, response, spider) return dfd @@ -142,7 +145,10 @@ class Scraper(object): def call_spider(self, result, request, spider): result.request = request dfd = defer_result(result) - dfd.addCallbacks(callback=request.callback or spider.parse, + callback = request.callback or spider.parse + warn_on_generator_with_return_value(spider, callback) + warn_on_generator_with_return_value(spider, request.errback) + dfd.addCallbacks(callback=callback, errback=request.errback, callbackKeywords=request.cb_kwargs) return dfd.addCallback(iterate_spider_output) @@ -172,8 +178,8 @@ class Scraper(object): if not result: return defer_succeed(None) it = iter_errback(result, self.handle_spider_error, request, response, spider) - dfd = parallel(it, self.concurrent_items, - self._process_spidermw_output, request, response, spider) + dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, + request, response, spider) return dfd def _process_spidermw_output(self, output, request, response, spider): @@ -200,8 +206,7 @@ class Scraper(object): """Log and silence errors that come from the engine (typically download errors that got propagated thru here) """ - if (isinstance(download_failure, Failure) and - not download_failure.check(IgnoreRequest)): + if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest): if download_failure.frames: logger.error('Error downloading %(request)s', {'request': request}, diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index ffd1537c3..a52bbc70e 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -8,6 +8,7 @@ This module must not depend on any module outside the Standard Library. import collections import copy import warnings +import weakref from collections.abc import Mapping from scrapy.exceptions import ScrapyDeprecationWarning @@ -240,7 +241,6 @@ class LocalCache(collections.OrderedDict): """Dictionary with a finite number of keys. Older items expires first. - """ def __init__(self, limit=None): @@ -254,6 +254,35 @@ class LocalCache(collections.OrderedDict): super(LocalCache, self).__setitem__(key, value) +class LocalWeakReferencedCache(weakref.WeakKeyDictionary): + """ + A weakref.WeakKeyDictionary implementation that uses LocalCache as its + underlying data structure, making it ordered and capable of being size-limited. + + Useful for memoization, while avoiding keeping received + arguments in memory only because of the cached references. + + Note: like LocalCache and unlike weakref.WeakKeyDictionary, + it cannot be instantiated with an initial dictionary. + """ + + def __init__(self, limit=None): + super(LocalWeakReferencedCache, self).__init__() + self.data = LocalCache(limit=limit) + + def __setitem__(self, key, value): + try: + super(LocalWeakReferencedCache, self).__setitem__(key, value) + except TypeError: + pass # key is not weak-referenceable, skip caching + + def __getitem__(self, key): + try: + return super(LocalWeakReferencedCache, self).__getitem__(key) + except TypeError: + return None # key is not weak-referenceable, it's not cached + + class SequenceExclude(object): """Object to test if an item is NOT within some sequence.""" diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 9955fb1e7..cb0ee5af3 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,13 +1,18 @@ """Helper functions which don't fit anywhere else""" +import ast +import inspect import os import re import hashlib +import warnings from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules +from textwrap import dedent from w3lib.html import replace_entities +from scrapy.utils.datatypes import LocalWeakReferencedCache from scrapy.utils.python import flatten, to_unicode from scrapy.item import BaseItem @@ -161,3 +166,44 @@ def set_environ(**kwargs): del os.environ[k] else: os.environ[k] = v + + +_generator_callbacks_cache = LocalWeakReferencedCache(limit=128) + + +def is_generator_with_return_value(callable): + """ + Returns True if a callable is a generator function which includes a + 'return' statement with a value different than None, False otherwise + """ + if callable in _generator_callbacks_cache: + return _generator_callbacks_cache[callable] + + def returns_none(return_node): + value = return_node.value + return value is None or isinstance(value, ast.NameConstant) and value.value is None + + if inspect.isgeneratorfunction(callable): + tree = ast.parse(dedent(inspect.getsource(callable))) + for node in ast.walk(tree): + if isinstance(node, ast.Return) and not returns_none(node): + _generator_callbacks_cache[callable] = True + return _generator_callbacks_cache[callable] + + _generator_callbacks_cache[callable] = False + return _generator_callbacks_cache[callable] + + +def warn_on_generator_with_return_value(spider, callable): + """ + Logs a warning if a callable is a generator function and includes + a 'return' statement with a value different than None + """ + if is_generator_with_return_value(callable): + warnings.warn( + 'The "{}.{}" method is a generator and includes a "return" statement with a ' + 'value different than None. This could lead to unexpected behaviour. Please see ' + 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement ' + 'for details about the semantics of the "return" statement within generators' + .format(spider.__class__.__name__, callable.__name__), stacklevel=2, + ) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 38a25778e..e5aa56eb9 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -2,7 +2,9 @@ import copy import unittest from collections.abc import Mapping, MutableMapping -from scrapy.utils.datatypes import CaselessDict, LocalCache, SequenceExclude +from scrapy.http import Request +from scrapy.utils.datatypes import CaselessDict, LocalCache, LocalWeakReferencedCache, SequenceExclude +from scrapy.utils.python import garbage_collect __doctests__ = ['scrapy.utils.datatypes'] @@ -255,5 +257,67 @@ class LocalCacheTest(unittest.TestCase): self.assertEqual(cache[str(x)], x) +class LocalWeakReferencedCacheTest(unittest.TestCase): + + def test_cache_with_limit(self): + cache = LocalWeakReferencedCache(limit=2) + r1 = Request('https://example.org') + r2 = Request('https://example.com') + r3 = Request('https://example.net') + cache[r1] = 1 + cache[r2] = 2 + cache[r3] = 3 + self.assertEqual(len(cache), 2) + self.assertNotIn(r1, cache) + self.assertIn(r2, cache) + self.assertIn(r3, cache) + self.assertEqual(cache[r2], 2) + self.assertEqual(cache[r3], 3) + del r2 + + # PyPy takes longer to collect dead references + garbage_collect() + + self.assertEqual(len(cache), 1) + + def test_cache_non_weak_referenceable_objects(self): + cache = LocalWeakReferencedCache() + k1 = None + k2 = 1 + k3 = [1, 2, 3] + cache[k1] = 1 + cache[k2] = 2 + cache[k3] = 3 + self.assertNotIn(k1, cache) + self.assertNotIn(k2, cache) + self.assertNotIn(k3, cache) + self.assertEqual(len(cache), 0) + + def test_cache_without_limit(self): + max = 10**4 + cache = LocalWeakReferencedCache() + refs = [] + for x in range(max): + refs.append(Request('https://example.org/{}'.format(x))) + cache[refs[-1]] = x + self.assertEqual(len(cache), max) + for i, r in enumerate(refs): + self.assertIn(r, cache) + self.assertEqual(cache[r], i) + del r # delete reference to the last object in the list + + # delete half of the objects, make sure that is reflected in the cache + for _ in range(max // 2): + refs.pop() + + # PyPy takes longer to collect dead references + garbage_collect() + + self.assertEqual(len(cache), max // 2) + for i, r in enumerate(refs): + self.assertIn(r, cache) + self.assertEqual(cache[r], i) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py new file mode 100644 index 000000000..bdbec1beb --- /dev/null +++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py @@ -0,0 +1,37 @@ +import unittest + +from scrapy.utils.misc import is_generator_with_return_value + + +class UtilsMiscPy3TestCase(unittest.TestCase): + + def test_generators_with_return_statements(self): + def f(): + yield 1 + return 2 + + def g(): + yield 1 + return 'asdf' + + def h(): + yield 1 + return None + + def i(): + yield 1 + return + + def j(): + yield 1 + + def k(): + yield 1 + yield from g() + + assert is_generator_with_return_value(f) + assert is_generator_with_return_value(g) + assert not is_generator_with_return_value(h) + assert not is_generator_with_return_value(i) + assert not is_generator_with_return_value(j) + assert not is_generator_with_return_value(k) # not recursive From 5980d3bbff67d060a1a1b15372293ced972dbe8b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Tue, 10 Sep 2019 14:23:11 +0500 Subject: [PATCH 69/85] Add simple tests for pipelines. --- tests/test_pipelines.py | 71 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 tests/test_pipelines.py diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py new file mode 100644 index 000000000..bc53f5427 --- /dev/null +++ b/tests/test_pipelines.py @@ -0,0 +1,71 @@ +from twisted.internet import defer +from twisted.internet.defer import Deferred +from twisted.trial import unittest + +from scrapy import Spider, signals, Request +from scrapy.utils.test import get_crawler + +from tests.mockserver import MockServer + + +class SimplePipeline: + def process_item(self, item, spider): + item['pipeline_passed'] = True + return item + + +class DeferredPipeline: + def cb(self, item): + item['pipeline_passed'] = True + return item + + def process_item(self, item, spider): + d = Deferred() + d.addCallback(self.cb) + d.callback(item) + return d + + +class ItemSpider(Spider): + name = 'itemspider' + + def start_requests(self): + yield Request(self.mockserver.url('/status?n=200')) + + def parse(self, response): + return {'field': 42} + + +class PipelineTestCase(unittest.TestCase): + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + def _on_item_scraped(self, item): + self.assertIsInstance(item, dict) + self.assertTrue(item.get('pipeline_passed')) + self.items.append(item) + + def _create_crawler(self, pipeline_class): + settings = { + 'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1}, + } + crawler = get_crawler(ItemSpider, settings) + crawler.signals.connect(self._on_item_scraped, signals.item_scraped) + self.items = [] + return crawler + + @defer.inlineCallbacks + def test_simple_pipeline(self): + crawler = self._create_crawler(SimplePipeline) + yield crawler.crawl(mockserver=self.mockserver) + self.assertEqual(len(self.items), 1) + + @defer.inlineCallbacks + def test_deferred_pipeline(self): + crawler = self._create_crawler(DeferredPipeline) + yield crawler.crawl(mockserver=self.mockserver) + self.assertEqual(len(self.items), 1) From 7d0096da6e37e508280e3af7388122bcdf0e3dcf Mon Sep 17 00:00:00 2001 From: apu <1173372284@qq.com> Date: Tue, 17 Dec 2019 09:47:01 +0800 Subject: [PATCH 70/85] Fix mail attachs tcmime *** (#4229) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the file name consists of alphanumeric characters, it is normal to receive the attachment name. However，However, problems will occur if the file name is changed to Chinese. This has nothing to do with the file type --- scrapy/mail.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 891bb5e09..9655b8114 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -73,8 +73,7 @@ class MailSender(object): part = MIMEBase(*mimetype.split('/')) part.set_payload(f.read()) Encoders.encode_base64(part) - part.add_header('Content-Disposition', 'attachment; filename="%s"' \ - % attach_name) + part.add_header('Content-Disposition', 'attachment', filename=attach_name) msg.attach(part) else: msg.set_payload(body) From 63cf5c75c850e48b6f267574e4a8f7ae3293deac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= <noviluni@gmail.com> Date: Tue, 17 Dec 2019 13:53:15 +0100 Subject: [PATCH 71/85] Fix E502: backslash is redundant between brackets (#4238) --- pytest.ini | 40 +++++------ scrapy/cmdline.py | 4 +- scrapy/commands/fetch.py | 11 ++- scrapy/commands/startproject.py | 4 +- scrapy/contracts/default.py | 4 +- scrapy/core/downloader/handlers/s3.py | 4 +- scrapy/core/downloader/webclient.py | 6 +- scrapy/core/spidermw.py | 6 +- .../downloadermiddlewares/httpcompression.py | 5 +- scrapy/extensions/closespider.py | 6 +- scrapy/linkextractors/__init__.py | 3 +- scrapy/middleware.py | 4 +- scrapy/utils/conf.py | 2 +- tests/test_cmdline/__init__.py | 10 ++- tests/test_downloader_handlers.py | 72 ++++++++++--------- tests/test_downloadermiddleware_retry.py | 6 +- tests/test_engine.py | 4 +- tests/test_http_request.py | 34 +++++---- tests/test_http_response.py | 4 +- tests/test_utils_http.py | 8 +-- 20 files changed, 121 insertions(+), 116 deletions(-) diff --git a/pytest.ini b/pytest.ini index 1b23595c0..f088e10ef 100644 --- a/pytest.ini +++ b/pytest.ini @@ -33,45 +33,45 @@ flake8-ignore = scrapy/commands/check.py E501 scrapy/commands/crawl.py E501 scrapy/commands/edit.py E501 - scrapy/commands/fetch.py E401 E501 E128 E502 E731 + scrapy/commands/fetch.py E401 E501 E128 E731 scrapy/commands/genspider.py E128 E501 E502 scrapy/commands/parse.py E128 E501 E731 E226 scrapy/commands/runspider.py E501 scrapy/commands/settings.py E128 scrapy/commands/shell.py E128 E501 E502 - scrapy/commands/startproject.py E502 E127 E501 E128 + scrapy/commands/startproject.py E127 E501 E128 scrapy/commands/version.py E501 E128 # scrapy/contracts scrapy/contracts/__init__.py E501 W504 - scrapy/contracts/default.py E502 E128 + scrapy/contracts/default.py E128 # scrapy/core scrapy/core/engine.py E501 E128 E127 E306 E502 scrapy/core/scheduler.py E501 scrapy/core/scraper.py E501 E306 E128 W504 - scrapy/core/spidermw.py E501 E731 E502 E126 E226 + scrapy/core/spidermw.py E501 E731 E126 E226 scrapy/core/downloader/__init__.py E501 scrapy/core/downloader/contextfactory.py E501 E128 E126 scrapy/core/downloader/middleware.py E501 E502 scrapy/core/downloader/tls.py E501 E305 E241 - scrapy/core/downloader/webclient.py E731 E501 E502 E128 E126 E226 + scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226 scrapy/core/downloader/handlers/__init__.py E501 scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127 scrapy/core/downloader/handlers/http10.py E501 scrapy/core/downloader/handlers/http11.py E501 - scrapy/core/downloader/handlers/s3.py E501 E502 E128 E126 + scrapy/core/downloader/handlers/s3.py E501 E128 E126 # scrapy/downloadermiddlewares scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226 scrapy/downloadermiddlewares/decompression.py E501 scrapy/downloadermiddlewares/defaultheaders.py E501 scrapy/downloadermiddlewares/httpcache.py E501 E126 - scrapy/downloadermiddlewares/httpcompression.py E502 E128 + scrapy/downloadermiddlewares/httpcompression.py E501 E128 scrapy/downloadermiddlewares/httpproxy.py E501 scrapy/downloadermiddlewares/redirect.py E501 W504 scrapy/downloadermiddlewares/retry.py E501 E126 scrapy/downloadermiddlewares/robotstxt.py E501 scrapy/downloadermiddlewares/stats.py E501 # scrapy/extensions - scrapy/extensions/closespider.py E501 E502 E128 E123 + scrapy/extensions/closespider.py E501 E128 E123 scrapy/extensions/corestats.py E501 scrapy/extensions/feedexport.py E128 E501 scrapy/extensions/httpcache.py E128 E501 E303 @@ -88,10 +88,10 @@ flake8-ignore = scrapy/http/response/__init__.py E501 E128 W293 W291 scrapy/http/response/text.py E501 W293 E128 E124 # scrapy/linkextractors - scrapy/linkextractors/__init__.py E731 E502 E501 E402 + scrapy/linkextractors/__init__.py E731 E501 E402 scrapy/linkextractors/lxmlhtml.py E501 E731 E226 # scrapy/loader - scrapy/loader/__init__.py E501 E502 E128 + scrapy/loader/__init__.py E501 E128 scrapy/loader/processors.py E501 # scrapy/pipelines scrapy/pipelines/files.py E116 E501 E266 @@ -116,7 +116,7 @@ flake8-ignore = scrapy/spiders/sitemap.py E501 # scrapy/utils scrapy/utils/benchserver.py E501 - scrapy/utils/conf.py E402 E502 E501 + scrapy/utils/conf.py E402 E501 scrapy/utils/console.py E306 E305 scrapy/utils/datatypes.py E501 E226 scrapy/utils/decorators.py E501 @@ -145,7 +145,7 @@ flake8-ignore = # scrapy scrapy/__init__.py E402 E501 scrapy/_monkeypatches.py W293 - scrapy/cmdline.py E502 E501 + scrapy/cmdline.py E501 scrapy/crawler.py E501 scrapy/dupefilters.py E501 E202 scrapy/exceptions.py E501 @@ -155,7 +155,7 @@ flake8-ignore = scrapy/link.py E501 scrapy/logformatter.py E501 W293 scrapy/mail.py E402 E128 E501 E502 - scrapy/middleware.py E502 E128 E501 + scrapy/middleware.py E128 E501 scrapy/pqueues.py E501 scrapy/responsetypes.py E128 E501 E305 scrapy/robotstxt.py E501 @@ -178,7 +178,7 @@ flake8-ignore = tests/test_crawl.py E501 E741 E265 tests/test_crawler.py F841 E306 E501 tests/test_dependencies.py F841 E501 E305 - tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E502 E701 E126 E226 E123 + tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126 @@ -189,18 +189,18 @@ flake8-ignore = tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123 tests/test_downloadermiddleware_httpproxy.py E501 E128 tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305 - tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126 + tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E303 E126 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124 - tests/test_engine.py E401 E501 E502 E128 + tests/test_engine.py E401 E501 E128 tests/test_exporters.py E501 E731 E306 E128 E124 tests/test_extension_telnet.py F841 tests/test_feedexport.py E501 F841 E241 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 - tests/test_http_request.py E402 E501 E127 E128 W293 E502 E128 E502 E126 E123 - tests/test_http_response.py E501 E301 E502 E128 E265 + tests/test_http_request.py E402 E501 E127 E128 W293 E128 E126 E123 + tests/test_http_response.py E501 E301 E128 E265 tests/test_item.py E701 E128 F841 E306 tests/test_link.py E501 tests/test_linkextractors.py E501 E128 E124 @@ -230,7 +230,7 @@ flake8-ignore = tests/test_utils_datatypes.py E402 E501 E305 tests/test_utils_defer.py E306 E501 F841 E226 tests/test_utils_deprecate.py F841 E306 E501 - tests/test_utils_http.py E501 E502 E128 W504 + tests/test_utils_http.py E501 E128 W504 tests/test_utils_iterators.py E501 E128 E129 E303 E241 tests/test_utils_log.py E741 E226 tests/test_utils_python.py E501 E303 E731 E701 E305 @@ -243,7 +243,7 @@ flake8-ignore = tests/test_utils_template.py E305 tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123 tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126 - tests/test_cmdline/__init__.py E502 E501 + tests/test_cmdline/__init__.py E501 tests/test_settings/__init__.py E501 E128 tests/test_spiderloader/__init__.py E128 E501 tests/test_utils_misc/__init__.py E501 diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 69e917004..ec78f7c91 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -67,7 +67,7 @@ def _pop_command_name(argv): def _print_header(settings, inproject): if inproject: - print("Scrapy %s - project: %s\n" % (scrapy.__version__, \ + print("Scrapy %s - project: %s\n" % (scrapy.__version__, settings['BOT_NAME'])) else: print("Scrapy %s - no active project\n" % scrapy.__version__) @@ -123,7 +123,7 @@ def execute(argv=None, settings=None): inproject = inside_project() cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) - parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \ + parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), conflict_handler='resolve') if not cmdname: _print_commands(settings, inproject) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 8a22ebabe..0e149941d 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -24,12 +24,11 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--spider", dest="spider", - help="use this spider") - parser.add_option("--headers", dest="headers", action="store_true", \ - help="print response HTTP headers instead of body") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \ - default=False, help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_option("--spider", dest="spider", help="use this spider") + parser.add_option("--headers", dest="headers", action="store_true", + help="print response HTTP headers instead of body") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", + default=False, help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index e65131ae8..b123e5c84 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -43,8 +43,8 @@ class Command(ScrapyCommand): return False if not re.search(r'^[_a-zA-Z]\w*$', project_name): - print('Error: Project names must begin with a letter and contain'\ - ' only\nletters, numbers and underscores') + print('Error: Project names must begin with a letter and contain' + ' only\nletters, numbers and underscores') elif _module_exists(project_name): print('Error: Module %r already exists' % project_name) else: diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index e0d425874..3002fc702 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -86,8 +86,8 @@ class ReturnsContract(Contract): else: expected = '%s..%s' % (self.min_bound, self.max_bound) - raise ContractFail("Returned %s %s, expected %s" % \ - (occurrences, self.obj_name, expected)) + raise ContractFail("Returned %s %s, expected %s" % + (occurrences, self.obj_name, expected)) class ScrapesContract(Contract): diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index e2a07bdef..d6fbd54ee 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -32,8 +32,8 @@ def _get_boto_connection(): class S3DownloadHandler(object): - def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \ - httpdownloadhandler=HTTPDownloadHandler, **kw): + def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, + httpdownloadhandler=HTTPDownloadHandler, **kw): if not aws_access_key_id: aws_access_key_id = settings['AWS_ACCESS_KEY_ID'] diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index f368c3bae..fc796e8bb 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -88,9 +88,9 @@ class ScrapyHTTPPageGetter(HTTPClient): if self.factory.url.startswith(b'https'): self.transport.stopProducing() - self.factory.noPage(\ - defer.TimeoutError("Getting %s took longer than %s seconds." % \ - (self.factory.url, self.factory.timeout))) + self.factory.noPage( + defer.TimeoutError("Getting %s took longer than %s seconds." % + (self.factory.url, self.factory.timeout))) class ScrapyHTTPClientFactory(HTTPClientFactory): diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index e2ade8256..097a374bf 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -44,7 +44,7 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \ + raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' .format(fname(method), type(result))) except _InvalidOutput: raise @@ -69,7 +69,7 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \ + raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' .format(fname(method), type(result))) return _failure @@ -103,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = evaluate_iterable(result, method_index) else: - raise _InvalidOutput('Middleware {} must return an iterable, got {}' \ + raise _InvalidOutput('Middleware {} must return an iterable, got {}' .format(fname(method), type(result))) return chain(result, recovered) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 203dee42d..65b652953 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -37,8 +37,9 @@ class HttpCompressionMiddleware(object): if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) - respcls = responsetypes.from_args(headers=response.headers, \ - url=response.url, body=decoded_body) + respcls = responsetypes.from_args( + headers=response.headers, url=response.url, body=decoded_body + ) kwargs = dict(cls=respcls, body=decoded_body) if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 9ccf356ec..afb2ed049 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -54,9 +54,9 @@ class CloseSpider(object): self.crawler.engine.close_spider(spider, 'closespider_pagecount') def spider_opened(self, spider): - self.task = reactor.callLater(self.close_on['timeout'], \ - self.crawler.engine.close_spider, spider, \ - reason='closespider_timeout') + self.task = reactor.callLater(self.close_on['timeout'], + self.crawler.engine.close_spider, spider, + reason='closespider_timeout') def item_scraped(self, item, spider): self.counter['itemcount'] += 1 diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 4a3e74fbe..bc65f41cc 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -44,8 +44,7 @@ IGNORED_EXTENSIONS = [ _re_type = type(re.compile("", 0)) _matches = lambda url, regexs: any(r.search(url) for r in regexs) -_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', \ - 'file', 'ftp'} +_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'} class FilteringLinkExtractor(object): diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 1cfd8a782..53fa435bb 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -65,8 +65,8 @@ class MiddlewareManager(object): return process_chain(self.methods[methodname], obj, *args) def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args): - return process_chain_both(self.methods[cb_methodname], \ - self.methods[eb_methodname], obj, *args) + return process_chain_both(self.methods[cb_methodname], + self.methods[eb_methodname], obj, *args) def open_spider(self, spider): return self._process_parallel('open_spider', spider) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 7a15e77ff..23306ca28 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -37,7 +37,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): """Fail if a value in the components dict is not a real number or None.""" for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): - raise ValueError('Invalid value {} for component {}, please provide ' \ + raise ValueError('Invalid value {} for component {}, please provide ' 'a real number or None instead'.format(value, name)) # BEGIN Backward compatibility for old (base, custom) call signature diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 909ea90e0..da99a6be8 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -25,17 +25,15 @@ class CmdlineTest(unittest.TestCase): return comm.decode(encoding) def test_default_settings(self): - self.assertEqual(self._execute('settings', '--get', 'TEST1'), \ - 'default') + self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'default') def test_override_settings_using_set_arg(self): - self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s', 'TEST1=override'), \ - 'override') + self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s', + 'TEST1=override'), 'override') def test_override_settings_using_envvar(self): self.env['SCRAPY_TEST1'] = 'override' - self.assertEqual(self._execute('settings', '--get', 'TEST1'), \ - 'override') + self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override') def test_profiling(self): path = tempfile.mkdtemp() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 2db2417e8..82d7b18d6 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -800,8 +800,8 @@ class S3TestCase(unittest.TestCase): req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date}) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], \ - b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=') + self.assertEqual(httpreq.headers['Authorization'], + b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=') def test_request_signing2(self): # puts an object into the johnsmith bucket. @@ -813,21 +813,22 @@ class S3TestCase(unittest.TestCase): }) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], \ - b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=') + self.assertEqual(httpreq.headers['Authorization'], + b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=') def test_request_signing3(self): # lists the content of the johnsmith bucket. date = 'Tue, 27 Mar 2007 19:42:41 +0000' - req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \ - method='GET', headers={ - 'User-Agent': 'Mozilla/5.0', - 'Date': date, - }) + req = Request( + 's3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', + method='GET', headers={ + 'User-Agent': 'Mozilla/5.0', + 'Date': date, + }) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], \ - b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=') + self.assertEqual(httpreq.headers['Authorization'], + b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=') def test_request_signing4(self): # fetches the access control policy sub-resource for the 'johnsmith' bucket. @@ -836,8 +837,8 @@ class S3TestCase(unittest.TestCase): method='GET', headers={'Date': date}) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], \ - b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') + self.assertEqual(httpreq.headers['Authorization'], + b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') def test_request_signing5(self): try: @@ -850,11 +851,11 @@ class S3TestCase(unittest.TestCase): # deletes an object from the 'johnsmith' bucket using the # path-style and Date alternative. date = 'Tue, 27 Mar 2007 21:20:27 +0000' - req = Request('s3://johnsmith/photos/puppy.jpg', \ - method='DELETE', headers={ - 'Date': date, - 'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000', - }) + req = Request( + 's3://johnsmith/photos/puppy.jpg', method='DELETE', headers={ + 'Date': date, + 'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000', + }) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) # botocore does not override Date with x-amz-date @@ -864,25 +865,26 @@ class S3TestCase(unittest.TestCase): def test_request_signing6(self): # uploads an object to a CNAME style virtual hosted bucket with metadata. date = 'Tue, 27 Mar 2007 21:06:08 +0000' - req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \ - method='PUT', headers={ - 'User-Agent': 'curl/7.15.5', - 'Host': 'static.johnsmith.net:8080', - 'Date': date, - 'x-amz-acl': 'public-read', - 'content-type': 'application/x-download', - 'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==', - 'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net', - 'X-Amz-Meta-FileChecksum': '0x02661779', - 'X-Amz-Meta-ChecksumAlgorithm': 'crc32', - 'Content-Disposition': 'attachment; filename=database.dat', - 'Content-Encoding': 'gzip', - 'Content-Length': '5913339', - }) + req = Request( + 's3://static.johnsmith.net:8080/db-backup.dat.gz', + method='PUT', headers={ + 'User-Agent': 'curl/7.15.5', + 'Host': 'static.johnsmith.net:8080', + 'Date': date, + 'x-amz-acl': 'public-read', + 'content-type': 'application/x-download', + 'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==', + 'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net', + 'X-Amz-Meta-FileChecksum': '0x02661779', + 'X-Amz-Meta-ChecksumAlgorithm': 'crc32', + 'Content-Disposition': 'attachment; filename=database.dat', + 'Content-Encoding': 'gzip', + 'Content-Length': '5913339', + }) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], \ - b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=') + self.assertEqual(httpreq.headers['Authorization'], + b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=') def test_request_signing7(self): # ensure that spaces are quoted properly before signing diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 51b79b6c3..e09d66086 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -165,12 +165,12 @@ class MaxRetryTimesTest(unittest.TestCase): # SETTINGS: meta(max_retry_times) = 4 meta_max_retry_times = 4 - req = Request(self.invalid_url, meta= \ - {'max_retry_times': meta_max_retry_times, 'dont_retry': True}) + req = Request(self.invalid_url, meta={ + 'max_retry_times': meta_max_retry_times, 'dont_retry': True + }) self._test_retry(req, DNSLookupError('foo'), 0) - def _test_retry(self, req, exception, max_retry_times): for i in range(0, max_retry_times): diff --git a/tests/test_engine.py b/tests/test_engine.py index 537df8d91..a48b63025 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -91,8 +91,8 @@ def start_test_site(debug=False): port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") if debug: - print("Test server running at http://localhost:%d/ - hit Ctrl-C to finish." \ - % port.getHost().port) + print("Test server running at http://localhost:%d/ - hit Ctrl-C to finish." + % port.getHost().port) return port diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 57e7b457d..e30417b30 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -622,8 +622,9 @@ class FormRequestTest(RequestTest): <input type="hidden" name="two" value="3"> <input type="submit" name="clickable2" value="clicked2"> </form>""") - req = self.request_class.from_response(response, formdata={'two': '2'}, \ - clickdata={'name': 'clickable2'}) + req = self.request_class.from_response( + response, formdata={'two': '2'}, clickdata={'name': 'clickable2'} + ) fs = _qs(req) self.assertEqual(fs[b'clickable2'], [b'clicked2']) self.assertFalse(b'clickable1' in fs, fs) @@ -671,8 +672,9 @@ class FormRequestTest(RequestTest): <input type="hidden" name="one" value="clicked1"> <input type="hidden" name="two" value="clicked2"> </form>""") - req = self.request_class.from_response(response, \ - clickdata={u'name': u'clickable', u'value': u'clicked2'}) + req = self.request_class.from_response( + response, clickdata={u'name': u'clickable', u'value': u'clicked2'} + ) fs = _qs(req) self.assertEqual(fs[b'clickable'], [b'clicked2']) self.assertEqual(fs[b'one'], [b'clicked1']) @@ -686,8 +688,9 @@ class FormRequestTest(RequestTest): <input type="hidden" name="poundsign" value="\u00a3"> <input type="hidden" name="eurosign" value="\u20ac"> </form>""") - req = self.request_class.from_response(response, \ - clickdata={u'name': u'price in \u00a3'}) + req = self.request_class.from_response( + response, clickdata={u'name': u'price in \u00a3'} + ) fs = _qs(req, to_unicode=True) self.assertTrue(fs[u'price in \u00a3']) @@ -700,8 +703,9 @@ class FormRequestTest(RequestTest): <input type="hidden" name="yensign" value="\u00a5"> </form>""", encoding='latin1') - req = self.request_class.from_response(response, \ - clickdata={u'name': u'price in \u00a5'}) + req = self.request_class.from_response( + response, clickdata={u'name': u'price in \u00a5'} + ) fs = _qs(req, to_unicode=True, encoding='latin1') self.assertTrue(fs[u'price in \u00a5']) @@ -716,8 +720,9 @@ class FormRequestTest(RequestTest): <input type="hidden" name="field2" value="value2"> </form> """) - req = self.request_class.from_response(response, formname='form2', \ - clickdata={u'name': u'clickable'}) + req = self.request_class.from_response( + response, formname='form2', clickdata={u'name': u'clickable'} + ) fs = _qs(req) self.assertEqual(fs[b'clickable'], [b'clicked2']) self.assertEqual(fs[b'field2'], [b'value2']) @@ -725,8 +730,9 @@ class FormRequestTest(RequestTest): def test_from_response_override_clickable(self): response = _buildresponse('''<form><input type="submit" name="clickme" value="one"> </form>''') - req = self.request_class.from_response(response, \ - formdata={'clickme': 'two'}, clickdata={'name': 'clickme'}) + req = self.request_class.from_response( + response, formdata={'clickme': 'two'}, clickdata={'name': 'clickme'} + ) fs = _qs(req) self.assertEqual(fs[b'clickme'], [b'two']) @@ -853,7 +859,7 @@ class FormRequestTest(RequestTest): <form name="form2" action="post.php" method="POST"> <input type="hidden" name="two" value="2"> </form>""") - self.assertRaises(IndexError, self.request_class.from_response, \ + self.assertRaises(IndexError, self.request_class.from_response, response, formname="form3", formnumber=2) def test_from_response_formid_exists(self): @@ -907,7 +913,7 @@ class FormRequestTest(RequestTest): <form id="form2" name="form2" action="post.php" method="POST"> <input type="hidden" name="two" value="2"> </form>""") - self.assertRaises(IndexError, self.request_class.from_response, \ + self.assertRaises(IndexError, self.request_class.from_response, response, formid="form3", formnumber=2) def test_from_response_select(self): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 79bb745cc..960ecea3e 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -316,8 +316,8 @@ class TextResponseTest(BaseResponseTest): assert u'SUFFIX' in r.text, repr(r.text) # Do not destroy html tags due to encoding bugs - r = self.response_class("http://example.com", encoding='utf-8', \ - body=b'\xf0<span>value</span>') + r = self.response_class("http://example.com", encoding='utf-8', + body=b'\xf0<span>value</span>') assert u'<span>value</span>' in r.text, repr(r.text) # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py index f9af4bf87..2fac3da1f 100644 --- a/tests/test_utils_http.py +++ b/tests/test_utils_http.py @@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase): chunked_body += "8\r\n" + "sequence\r\n" chunked_body += "0\r\n\r\n" body = decode_chunked_transfer(chunked_body) - self.assertEqual(body, \ - "This is the data in the first chunk\r\n" + - "and this is the second one\r\n" + - "consequence") + self.assertEqual(body, + "This is the data in the first chunk\r\n" + + "and this is the second one\r\n" + + "consequence") From 2d92a39003a5a0b01b265b87cf722a61f121ca60 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 18 Dec 2019 12:07:08 +0500 Subject: [PATCH 72/85] Restore test_download_with_proxy_https_noconnect, check for a warning there. --- tests/test_downloader_handlers.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 45d4aa952..87ab4c9e5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -33,7 +33,7 @@ from scrapy.responsetypes import responsetypes from scrapy.settings import Settings from scrapy.utils.test import get_crawler, skip_if_no_boto from scrapy.utils.python import to_bytes -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from tests.mockserver import MockServer, ssl_context_factory, Echo from tests.spiders import SingleRequestSpider @@ -687,6 +687,18 @@ class HttpProxyTestCase(unittest.TestCase): request = Request('http://example.com', meta={'proxy': http_proxy}) return self.download_request(request, Spider('foo')).addCallback(_test) + def test_download_with_proxy_https_noconnect(self): + def _test(response): + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, b'https://example.com') + + http_proxy = '%s?noconnect' % self.getURL('') + request = Request('https://example.com', meta={'proxy': http_proxy}) + with self.assertWarnsRegex(ScrapyDeprecationWarning, + r'Using HTTPS proxies in the noconnect mode is deprecated'): + return self.download_request(request, Spider('foo')).addCallback(_test) + def test_download_without_proxy(self): def _test(response): self.assertEqual(response.status, 200) From bb2ff13e4c7c80cfc7925f60eadc97dec0d69026 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 18 Dec 2019 15:39:08 +0500 Subject: [PATCH 73/85] Skip Http10ProxyTestCase.test_download_with_proxy_https_noconnect --- tests/test_downloader_handlers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 87ab4c9e5..412a9c084 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -712,6 +712,8 @@ class HttpProxyTestCase(unittest.TestCase): class Http10ProxyTestCase(HttpProxyTestCase): download_handler_cls = HTTP10DownloadHandler + def test_download_with_proxy_https_noconnect(self): + raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler') class Http11ProxyTestCase(HttpProxyTestCase): download_handler_cls = HTTP11DownloadHandler From ac302c3f615d339ed36000231ca3e1e1c347478a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 18 Dec 2019 15:43:05 +0500 Subject: [PATCH 74/85] Fix a flake8 problem. --- tests/test_downloader_handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 412a9c084..7412d7ebf 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -715,6 +715,7 @@ class Http10ProxyTestCase(HttpProxyTestCase): def test_download_with_proxy_https_noconnect(self): raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler') + class Http11ProxyTestCase(HttpProxyTestCase): download_handler_cls = HTTP11DownloadHandler From 12f9ffeb5d1c1f4d02f616852dcae82ef633e8d9 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita <whalebot.helmsman@gmail.com> Date: Wed, 18 Dec 2019 10:53:27 +0000 Subject: [PATCH 75/85] remove requirements-py3.txt --- requirements-py3.txt | 16 ---------------- tox.ini | 1 - 2 files changed, 17 deletions(-) delete mode 100644 requirements-py3.txt diff --git a/requirements-py3.txt b/requirements-py3.txt deleted file mode 100644 index 28c649e28..000000000 --- a/requirements-py3.txt +++ /dev/null @@ -1,16 +0,0 @@ -parsel>=1.5.0 -PyDispatcher>=2.0.5 -Twisted>=17.9.0 -w3lib>=1.17.0 - -pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" -queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" -cryptography>=2.0 # Earlier versions would fail to install - -# Reference versions taken from -# https://packages.ubuntu.com/xenial/python/ -# https://packages.ubuntu.com/xenial/zope/ -cssselect>=0.9.1 -lxml>=3.5.0 -service_identity>=16.0.0 -zope.interface>=4.1.3 diff --git a/tox.ini b/tox.ini index fd75d18e2..f37c381d0 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,6 @@ envlist = py35 [testenv] deps = -ctests/constraints.txt - -rrequirements-py3.txt -rtests/requirements-py3.txt # Extras botocore>=1.3.23 From ee9881d2704798c9cd61b6da503bb0694227c58c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 18 Dec 2019 12:08:34 +0100 Subject: [PATCH 76/85] Improve FilteringLinkExtractor.__new__ --- scrapy/linkextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index a510fef70..7254bd79c 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -61,7 +61,7 @@ class FilteringLinkExtractor(object): warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' 'please use scrapy.linkextractors.LinkExtractor instead', ScrapyDeprecationWarning, stacklevel=2) - return super(FilteringLinkExtractor, cls).__new__(cls) + return super().__new__(cls, *args, **kwargs) def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text): From 174769a3f08fcd84eaec8a88217a05f8ebc3f2cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Wed, 18 Dec 2019 12:09:03 +0100 Subject: [PATCH 77/85] Use a better name for the LxmlLinkExtractor subclassing test --- tests/test_linkextractors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 0ffeaecc3..cfd4c6b85 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -517,10 +517,10 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): LxmlLinkExtractor() self.assertEqual(len(warnings), 0) - class SubclassedItem(LxmlLinkExtractor): + class SubclassedLxmlLinkExtractor(LxmlLinkExtractor): pass - SubclassedItem() + SubclassedLxmlLinkExtractor() self.assertEqual(len(warnings), 0) From 012533924a799c7bf1d3f2d4a489d8ca1ac462f8 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita <whalebot.helmsman@gmail.com> Date: Wed, 18 Dec 2019 11:13:36 +0000 Subject: [PATCH 78/85] remove requirements from here too --- docs/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 0ed11c4dc..773b92cea 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,3 @@ --r ../requirements-py3.txt Sphinx>=2.1 sphinx-hoverxref sphinx-notfound-page From 7ccb169a27ccbedf3145b824e1bd655cb6902f32 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin <wrar@wrar.name> Date: Wed, 18 Dec 2019 19:41:16 +0500 Subject: [PATCH 79/85] Split a long test in test_engine.py into three. --- tests/test_engine.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index a48b63025..25dee7c1f 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -179,7 +179,6 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_crawler(self): - for spider in TestSpider, DictItemsSpider: self.run = CrawlerRun(spider) yield self.run.run() @@ -189,11 +188,15 @@ class EngineTest(unittest.TestCase): self._assert_scraped_items() self._assert_signals_catched() + @defer.inlineCallbacks + def test_crawler_dupefilter(self): self.run = CrawlerRun(TestDupeFilterSpider) yield self.run.run() self._assert_scheduled_requests(urls_to_visit=7) self._assert_dropped_requests() + @defer.inlineCallbacks + def test_crawler_itemerror(self): self.run = CrawlerRun(ItemZeroDivisionErrorSpider) yield self.run.run() self._assert_items_error() From 916382e109dc06a57f4631448555953d1fa540b5 Mon Sep 17 00:00:00 2001 From: elacuesta <elacuesta@users.noreply.github.com> Date: Wed, 18 Dec 2019 12:05:33 -0300 Subject: [PATCH 80/85] Add errback parameter to scrapy.spiders.crawl.Rule (#4000) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add errback parameter to scrapy.spiders.crawl.Rule * CrawlSpider: optimize by reducing iterations * [test] Rule.errback * [doc] Rule.errback * [doc] Use autoclass in docs/topics/spiders.rst Co-Authored-By: Adrián Chaves <adrian@chaves.io> * Rule.process_links takes a list * Fix aesthetic issue reported by Flake8 --- docs/topics/spiders.rst | 6 +++++ scrapy/spiders/crawl.py | 60 ++++++++++++++++++++++++++--------------- tests/mockserver.py | 7 +++++ tests/spiders.py | 36 +++++++++++++++++++++++-- tests/test_crawl.py | 18 ++++++++++--- 5 files changed, 101 insertions(+), 26 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index d65a43afd..b0fb14e24 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -414,6 +414,12 @@ Crawling rules from which the request originated as second argument. It must return a ``Request`` object or ``None`` (to filter out the request). + ``errback`` is a callable or a string (in which case a method from the spider + object with that name will be used) to be called if any exception is + raised while processing a request generated by the rule. + It receives a :class:`Twisted Failure <twisted.python.failure.Failure>` + instance as first parameter. + CrawlSpider example ~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index a5eb1a518..a2c364c0e 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -16,7 +16,11 @@ from scrapy.utils.python import get_func_args from scrapy.utils.spider import iterate_spider_output -def _identity(request, response): +def _identity(x): + return x + + +def _identity_process_request(request, response): return request @@ -32,17 +36,20 @@ _default_link_extractor = LinkExtractor() class Rule(object): - def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None): + def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None, + process_links=None, process_request=None, errback=None): self.link_extractor = link_extractor or _default_link_extractor self.callback = callback + self.errback = errback self.cb_kwargs = cb_kwargs or {} - self.process_links = process_links - self.process_request = process_request or _identity + self.process_links = process_links or _identity + self.process_request = process_request or _identity_process_request self.process_request_argcount = None self.follow = follow if follow is not None else not callback def _compile(self, spider): self.callback = _get_method(self.callback, spider) + self.errback = _get_method(self.errback, spider) self.process_links = _get_method(self.process_links, spider) self.process_request = _get_method(self.process_request, spider) self.process_request_argcount = len(get_func_args(self.process_request)) @@ -76,48 +83,59 @@ class CrawlSpider(Spider): def process_results(self, response, results): return results - def _build_request(self, rule, link): - r = Request(url=link.url, callback=self._response_downloaded) - r.meta.update(rule=rule, link_text=link.text) - return r + def _build_request(self, rule_index, link): + return Request( + url=link.url, + callback=self._callback, + errback=self._errback, + meta=dict(rule=rule_index, link_text=link.text), + ) def _requests_to_follow(self, response): if not isinstance(response, HtmlResponse): return seen = set() - for n, rule in enumerate(self._rules): + for rule_index, rule in enumerate(self._rules): links = [lnk for lnk in rule.link_extractor.extract_links(response) if lnk not in seen] - if links and rule.process_links: - links = rule.process_links(links) - for link in links: + for link in rule.process_links(links): seen.add(link) - request = self._build_request(n, link) + request = self._build_request(rule_index, link) yield rule._process_request(request, response) - def _response_downloaded(self, response): + def _callback(self, response): rule = self._rules[response.meta['rule']] return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow) + def _errback(self, failure): + rule = self._rules[failure.request.meta['rule']] + return self._handle_failure(failure, rule.errback) + def _parse_response(self, response, callback, cb_kwargs, follow=True): if callback: cb_res = callback(response, **cb_kwargs) or () cb_res = self.process_results(response, cb_res) - for requests_or_item in iterate_spider_output(cb_res): - yield requests_or_item + for request_or_item in iterate_spider_output(cb_res): + yield request_or_item if follow and self._follow_links: for request_or_item in self._requests_to_follow(response): yield request_or_item + def _handle_failure(self, failure, errback): + if errback: + results = errback(failure) or () + for request_or_item in iterate_spider_output(results): + yield request_or_item + def _compile_rules(self): - self._rules = [copy.copy(r) for r in self.rules] - for rule in self._rules: - rule._compile(self) + self._rules = [] + for rule in self.rules: + self._rules.append(copy.copy(rule)) + self._rules[-1]._compile(self) @classmethod def from_crawler(cls, crawler, *args, **kwargs): spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs) - spider._follow_links = crawler.settings.getbool( - 'CRAWLSPIDER_FOLLOW_LINKS', True) + spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True) return spider diff --git a/tests/mockserver.py b/tests/mockserver.py index fe28176d4..a45277db9 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -164,6 +164,12 @@ class Drop(Partial): request.finish() +class ArbitraryLengthPayloadResource(LeafResource): + + def render(self, request): + return request.content.read() + + class Root(Resource): def __init__(self): @@ -177,6 +183,7 @@ class Root(Resource): self.putChild(b"echo", Echo()) self.putChild(b"payload", PayloadResource()) self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) + self.putChild(b"alpayload", ArbitraryLengthPayloadResource()) try: from tests import tests_datadir self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/'))) diff --git a/tests/spiders.py b/tests/spiders.py index 981bd2eb8..39c8da0b6 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -1,14 +1,14 @@ """ Some spiders used for testing and benchmarking """ - import time from urllib.parse import urlencode -from scrapy.spiders import Spider from scrapy.http import Request from scrapy.item import Item from scrapy.linkextractors import LinkExtractor +from scrapy.spiders import Spider +from scrapy.spiders.crawl import CrawlSpider, Rule class MockServerSpider(Spider): @@ -184,3 +184,35 @@ class DuplicateStartRequestsSpider(MockServerSpider): def parse(self, response): self.visited += 1 + + +class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider): + name = 'crawl_spider_with_errback' + custom_settings = { + 'RETRY_HTTP_CODES': [], # no need to retry + } + rules = ( + Rule(LinkExtractor(), callback='callback', errback='errback', follow=True), + ) + + def start_requests(self): + test_body = b""" + <html> + <head><title>Page title<title></head> + <body> + <p><a href="/status?n=200">Item 200</a></p> <!-- callback --> + <p><a href="/status?n=201">Item 201</a></p> <!-- callback --> + <p><a href="/status?n=404">Item 404</a></p> <!-- errback --> + <p><a href="/status?n=500">Item 500</a></p> <!-- errback --> + <p><a href="/status?n=501">Item 501</a></p> <!-- errback --> + </body> + </html> + """ + url = self.mockserver.url("/alpayload") + yield Request(url, method="POST", body=test_body) + + def callback(self, response): + self.logger.info('[callback] status %i', response.status) + + def errback(self, failure): + self.logger.info('[errback] status %i', failure.value.response.status) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3307899b7..76f87458b 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -5,12 +5,12 @@ from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase -from scrapy.http import Request from scrapy.crawler import CrawlerRunner +from scrapy.http import Request from scrapy.utils.python import to_unicode -from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \ - BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider from tests.mockserver import MockServer +from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider, + SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback) class CrawlTestCase(TestCase): @@ -277,3 +277,15 @@ with multiples lines self._assert_retried(log) self.assertIn("Got response 200", str(log)) + + @defer.inlineCallbacks + def test_crawlspider_with_errback(self): + self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver) + + with LogCapture() as log: + yield self.runner.join() + + self.assertIn("[callback] status 200", str(log)) + self.assertIn("[callback] status 201", str(log)) + self.assertIn("[errback] status 404", str(log)) + self.assertIn("[errback] status 500", str(log)) From a5de2c64e6f1233a98b1b180606fca0ae4dd0871 Mon Sep 17 00:00:00 2001 From: Marc Hernandez Cabot <noviluni@gmail.com> Date: Wed, 18 Dec 2019 16:24:48 +0100 Subject: [PATCH 81/85] fix W291, W292, W293 (whitespaces) --- pytest.ini | 27 ++++++++++----------- scrapy/http/response/__init__.py | 4 +-- scrapy/http/response/text.py | 4 +-- scrapy/logformatter.py | 4 +-- scrapy/utils/markup.py | 2 +- scrapy/utils/multipart.py | 2 +- tests/test_contracts.py | 2 +- tests/test_downloadermiddleware_retry.py | 8 +++--- tests/test_dupefilters.py | 6 ++--- tests/test_pipeline_files.py | 1 - tests/test_robotstxt_interface.py | 2 +- tests/test_spidermiddleware_offsite.py | 2 +- tests/test_spidermiddleware_output_chain.py | 8 +++--- 13 files changed, 35 insertions(+), 37 deletions(-) diff --git a/pytest.ini b/pytest.ini index f088e10ef..ac3c8cfb5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -85,8 +85,8 @@ flake8-ignore = scrapy/http/request/__init__.py E501 scrapy/http/request/form.py E501 E123 scrapy/http/request/json_request.py E501 - scrapy/http/response/__init__.py E501 E128 W293 W291 - scrapy/http/response/text.py E501 W293 E128 E124 + scrapy/http/response/__init__.py E501 E128 + scrapy/http/response/text.py E501 E128 E124 # scrapy/linkextractors scrapy/linkextractors/__init__.py E731 E501 E402 scrapy/linkextractors/lxmlhtml.py E501 E731 E226 @@ -127,9 +127,9 @@ flake8-ignore = scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 E701 scrapy/utils/log.py E128 W503 - scrapy/utils/markup.py F403 W292 + scrapy/utils/markup.py F403 scrapy/utils/misc.py E501 E226 - scrapy/utils/multipart.py F403 W292 + scrapy/utils/multipart.py F403 scrapy/utils/project.py E501 scrapy/utils/python.py E501 scrapy/utils/reactor.py E226 @@ -144,7 +144,6 @@ flake8-ignore = scrapy/utils/url.py E501 F403 E128 F405 # scrapy scrapy/__init__.py E402 E501 - scrapy/_monkeypatches.py W293 scrapy/cmdline.py E501 scrapy/crawler.py E501 scrapy/dupefilters.py E501 E202 @@ -153,7 +152,7 @@ flake8-ignore = scrapy/interfaces.py E501 scrapy/item.py E501 E128 scrapy/link.py E501 - scrapy/logformatter.py E501 W293 + scrapy/logformatter.py E501 scrapy/mail.py E402 E128 E501 E502 scrapy/middleware.py E128 E501 scrapy/pqueues.py E501 @@ -174,7 +173,7 @@ flake8-ignore = tests/test_command_parse.py E501 E128 E303 E226 tests/test_command_shell.py E501 E128 tests/test_commands.py E128 E501 - tests/test_contracts.py E501 E128 W293 + tests/test_contracts.py E501 E128 tests/test_crawl.py E501 E741 E265 tests/test_crawler.py F841 E306 E501 tests/test_dependencies.py F841 E501 E305 @@ -189,17 +188,17 @@ flake8-ignore = tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123 tests/test_downloadermiddleware_httpproxy.py E501 E128 tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305 - tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E303 E126 + tests/test_downloadermiddleware_retry.py E501 E128 E251 E303 E126 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124 + tests/test_dupefilters.py E221 E501 E741 E128 E124 tests/test_engine.py E401 E501 E128 tests/test_exporters.py E501 E731 E306 E128 E124 tests/test_extension_telnet.py F841 tests/test_feedexport.py E501 F841 E241 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 - tests/test_http_request.py E402 E501 E127 E128 W293 E128 E126 E123 + tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123 tests/test_http_response.py E501 E301 E128 E265 tests/test_item.py E701 E128 F841 E306 tests/test_link.py E501 @@ -209,20 +208,20 @@ flake8-ignore = tests/test_mail.py E128 E501 E305 tests/test_middleware.py E501 E128 tests/test_pipeline_crawl.py E131 E501 E128 E126 - tests/test_pipeline_files.py E501 W293 E303 E272 E226 + tests/test_pipeline_files.py E501 E303 E272 E226 tests/test_pipeline_images.py F841 E501 E303 tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502 tests/test_proxy_connect.py E501 E741 tests/test_request_cb_kwargs.py E501 tests/test_responsetypes.py E501 E305 - tests/test_robotstxt_interface.py E501 W291 E501 + tests/test_robotstxt_interface.py E501 E501 tests/test_scheduler.py E501 E126 E123 tests/test_selector.py E501 E127 tests/test_spider.py E501 tests/test_spidermiddleware.py E501 E226 tests/test_spidermiddleware_httperror.py E128 E501 E127 E121 - tests/test_spidermiddleware_offsite.py E501 E128 E111 W293 - tests/test_spidermiddleware_output_chain.py E501 W293 E226 + tests/test_spidermiddleware_offsite.py E501 E128 E111 + tests/test_spidermiddleware_output_chain.py E501 E226 tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121 tests/test_squeues.py E501 E701 E741 tests/test_utils_conf.py E501 E303 E128 diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 64e9c6c20..e79ce9acc 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -113,8 +113,8 @@ class Response(object_ref): It accepts the same arguments as ``Request.__init__`` method, but ``url`` can be a relative URL or a ``scrapy.link.Link`` object, not only an absolute URL. - - :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` + + :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` method which supports selectors in addition to absolute/relative URLs and Link objects. """ diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 1079fd6e8..4f9afde87 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -125,7 +125,7 @@ class TextResponse(Response): Return a :class:`~.Request` instance to follow a link ``url``. It accepts the same arguments as ``Request.__init__`` method, but ``url`` can be not only an absolute URL, but also - + * a relative URL; * a scrapy.link.Link object (e.g. a link extractor result); * an attribute Selector (not SelectorList) - e.g. @@ -133,7 +133,7 @@ class TextResponse(Response): ``response.xpath('//img/@src')[0]``. * a Selector for ``<a>`` or ``<link>`` element, e.g. ``response.css('a.my_link')[0]``. - + See :ref:`response-follow-example` for usage examples. """ if isinstance(url, parsel.Selector): diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 5189d7cfa..4e5963e99 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -13,7 +13,7 @@ ERRORMSG = u"'Error processing %(item)s'" class LogFormatter(object): """Class for generating log messages for different actions. - + All methods must return a dictionary listing the parameters ``level``, ``msg`` and ``args`` which are going to be used for constructing the log message when calling ``logging.log``. @@ -48,7 +48,7 @@ class LogFormatter(object): } } """ - + def crawled(self, request, response, spider): """Logs a message when the crawler finds a webpage.""" request_flags = ' %s' % str(request.flags) if request.flags else '' diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py index 2455fcc16..9728c542a 100644 --- a/scrapy/utils/markup.py +++ b/scrapy/utils/markup.py @@ -11,4 +11,4 @@ from w3lib.html import * # noqa: F401 warnings.warn("Module `scrapy.utils.markup` is deprecated. " "Please import from `w3lib.html` instead.", - ScrapyDeprecationWarning, stacklevel=2) \ No newline at end of file + ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py index e81f63152..5dcf791b8 100644 --- a/scrapy/utils/multipart.py +++ b/scrapy/utils/multipart.py @@ -12,4 +12,4 @@ from w3lib.form import * # noqa: F401 warnings.warn("Module `scrapy.utils.multipart` is deprecated. " "If you're using `encode_multipart` function, please use " "`urllib3.filepost.encode_multipart_formdata` instead", - ScrapyDeprecationWarning, stacklevel=2) \ No newline at end of file + ScrapyDeprecationWarning, stacklevel=2) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 582e3d052..11d41c1fe 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -252,7 +252,7 @@ class ContractsManagerTest(unittest.TestCase): self.assertEqual(len(contracts), 3) self.assertEqual(frozenset(type(x) for x in contracts), frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract])) - + contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs) self.assertEqual(len(contracts), 3) self.assertEqual(frozenset(type(x) for x in contracts), diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index e09d66086..9c989977e 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -124,7 +124,7 @@ class MaxRetryTimesTest(unittest.TestCase): # SETTINGS: meta(max_retry_times) = 0 meta_max_retry_times = 0 - + req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times) @@ -137,7 +137,7 @@ class MaxRetryTimesTest(unittest.TestCase): self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times) def test_with_metakey_greater(self): - + # SETINGS: RETRY_TIMES < meta(max_retry_times) self.mw.max_retry_times = 2 meta_max_retry_times = 3 @@ -149,7 +149,7 @@ class MaxRetryTimesTest(unittest.TestCase): self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) def test_with_metakey_lesser(self): - + # SETINGS: RETRY_TIMES > meta(max_retry_times) self.mw.max_retry_times = 5 meta_max_retry_times = 4 @@ -172,7 +172,7 @@ class MaxRetryTimesTest(unittest.TestCase): self._test_retry(req, DNSLookupError('foo'), 0) def _test_retry(self, req, exception, max_retry_times): - + for i in range(0, max_retry_times): req = self.mw.process_exception(req, exception, self.spider) assert isinstance(req, Request) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index e4b0bdf83..0546558bc 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -142,12 +142,12 @@ class RFPDupeFilterTest(unittest.TestCase): r1 = Request('http://scrapytest.org/index.html') r2 = Request('http://scrapytest.org/index.html') - + dupefilter.log(r1, spider) dupefilter.log(r2, spider) assert crawler.stats.get_value('dupefilter/filtered') == 2 - l.check_present(('scrapy.dupefilters', 'DEBUG', + l.check_present(('scrapy.dupefilters', 'DEBUG', ('Filtered duplicate request: <GET http://scrapytest.org/index.html>' ' - no more duplicates will be shown' ' (see DUPEFILTER_DEBUG to show all duplicates)'))) @@ -169,7 +169,7 @@ class RFPDupeFilterTest(unittest.TestCase): r2 = Request('http://scrapytest.org/index.html', headers={'Referer': 'http://scrapytest.org/INDEX.html'} ) - + dupefilter.log(r1, spider) dupefilter.log(r2, spider) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 52f2b554e..141141671 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -58,7 +58,6 @@ class FilesPipelineTestCase(unittest.TestCase): self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\ //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")), 'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png') - def test_fs_store(self): assert isinstance(self.pipeline.store, FSFilesStore) diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 27d79437b..24aaaf7ec 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -44,7 +44,7 @@ class BaseRobotParserTest: def test_allowed_wildcards(self): robotstxt_robotstxt_body = """User-agent: first - Disallow: /disallowed/*/end$ + Disallow: /disallowed/*/end$ User-agent: second Allow: /*allowed diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 992e60be2..7511aa568 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -73,7 +73,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3): class TestOffsiteMiddleware5(TestOffsiteMiddleware4): - + def test_get_host_regex(self): self.spider.allowed_domains = ['http://scrapytest.org', 'scrapy.org', 'scrapy.test.org'] with warnings.catch_warnings(record=True) as w: diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 5b7b5e7aa..739cf1c2d 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -156,7 +156,7 @@ class GeneratorFailMiddleware: r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) yield r raise LookupError() - + def process_spider_exception(self, response, exception, spider): method = '{}.process_spider_exception'.format(self.__class__.__name__) spider.logger.info('%s: %s caught', method, exception.__class__.__name__) @@ -264,7 +264,7 @@ class TestSpiderMiddleware(TestCase): @classmethod def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - + @defer.inlineCallbacks def crawl_log(self, spider): crawler = get_crawler(spider) @@ -308,7 +308,7 @@ class TestSpiderMiddleware(TestCase): self.assertIn("{'from': 'errback'}", str(log1)) self.assertNotIn("{'from': 'callback'}", str(log1)) self.assertIn("'item_scraped_count': 1", str(log1)) - + @defer.inlineCallbacks def test_generator_callback(self): """ @@ -319,7 +319,7 @@ class TestSpiderMiddleware(TestCase): log2 = yield self.crawl_log(GeneratorCallbackSpider) self.assertIn("Middleware: ImportError exception caught", str(log2)) self.assertIn("'item_scraped_count': 2", str(log2)) - + @defer.inlineCallbacks def test_not_a_generator_callback(self): """ From c0d84f0962a4c269441db562e8cbc10298c53b72 Mon Sep 17 00:00:00 2001 From: Marc Hernandez Cabot <noviluni@gmail.com> Date: Wed, 18 Dec 2019 19:39:21 +0100 Subject: [PATCH 82/85] fix typos --- docs/contributing.rst | 2 +- docs/faq.rst | 2 +- docs/news.rst | 69 +++++++++++++++++----------------- docs/topics/jobs.rst | 2 +- docs/topics/leaks.rst | 2 +- docs/topics/media-pipeline.rst | 2 +- docs/topics/settings.rst | 2 +- docs/topics/telnetconsole.rst | 2 +- sep/sep-001.rst | 4 +- sep/sep-019.rst | 2 +- 10 files changed, 44 insertions(+), 45 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 3aebb3d50..b56295027 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -217,7 +217,7 @@ the tests with Python 3.6 use:: tox -e py36 -You can also specify a comma-separated list of environmets, and use :ref:`tox’s +You can also specify a comma-separated list of environments, and use :ref:`tox’s parallel mode <tox:parallel_mode>` to run the tests on multiple environments in parallel:: diff --git a/docs/faq.rst b/docs/faq.rst index 080d81981..aae2411e0 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -338,7 +338,7 @@ How to split an item into multiple items in an item pipeline? input item. :ref:`Create a spider middleware <custom-spider-middleware>` instead, and use its :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` -method for this puspose. For example:: +method for this purpose. For example:: from copy import deepcopy diff --git a/docs/news.rst b/docs/news.rst index 9dfd28508..28db40e3c 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -678,7 +678,7 @@ Usability improvements * a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`) * better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`) * non-zero exit code is returned from Scrapy commands when error happens - on spider inititalization (:issue:`3226`) + on spider initialization (:issue:`3226`) * Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`); "flv" is added to common video extensions (:issue:`3165`) * better error message when an exporter is disabled (:issue:`3358`); @@ -1156,7 +1156,7 @@ Bug fixes - Fix :command:`view` command ; it was a regression in v1.3.0 (:issue:`2503`). - Fix tests regarding ``*_EXPIRES settings`` with Files/Images pipelines (:issue:`2460`). - Fix name of generated pipeline class when using basic project template (:issue:`2466`). -- Fix compatiblity with Twisted 17+ (:issue:`2496`, :issue:`2528`). +- Fix compatibility with Twisted 17+ (:issue:`2496`, :issue:`2528`). - Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`). - Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``, ``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`). @@ -1164,7 +1164,7 @@ Bug fixes Documentation ~~~~~~~~~~~~~ -- Reword Code of Coduct section and upgrade to Contributor Covenant v1.4 +- Reword Code of Conduct section and upgrade to Contributor Covenant v1.4 (:issue:`2469`). - Clarify that passing spider arguments converts them to spider attributes (:issue:`2483`). @@ -1178,7 +1178,7 @@ Documentation Cleanups ~~~~~~~~ -- Remove reduntant check in ``MetaRefreshMiddleware`` (:issue:`2542`). +- Remove redundant check in ``MetaRefreshMiddleware`` (:issue:`2542`). - Faster checks in ``LinkExtractor`` for allow/deny patterns (:issue:`2538`). - Remove dead code supporting old Twisted versions (:issue:`2544`). @@ -1204,7 +1204,7 @@ New Features - ``MailSender`` now accepts single strings as values for ``to`` and ``cc`` arguments (:issue:`2272`) - ``scrapy fetch url``, ``scrapy shell url`` and ``fetch(url)`` inside - scrapy shell now follow HTTP redirections by default (:issue:`2290`); + Scrapy shell now follow HTTP redirections by default (:issue:`2290`); See :command:`fetch` and :command:`shell` for details. - ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``; this is technically **backward incompatible** so please check your log parsers. @@ -1705,7 +1705,7 @@ Scrapy 1.0.4 (2015-12-30) - fix ValueError: Invalid XPath: //div/[id="not-exists"]/text() on selectors.rst (:commit:`ca8d60f`) - Typos corrections (:commit:`7067117`) - fix typos in downloader-middleware.rst and exceptions.rst, middlware -> middleware (:commit:`32f115c`) -- Add note to ubuntu install section about debian compatibility (:commit:`23fda69`) +- Add note to Ubuntu install section about Debian compatibility (:commit:`23fda69`) - Replace alternative OSX install workaround with virtualenv (:commit:`98b63ee`) - Reference Homebrew's homepage for installation instructions (:commit:`1925db1`) - Add oldest supported tox version to contributing docs (:commit:`5d10d6d`) @@ -1758,7 +1758,7 @@ Scrapy 1.0.1 (2015-07-01) - include tests/ to source distribution in MANIFEST.in (:commit:`eca227e`) - DOC Fix SelectJmes documentation (:commit:`b8567bc`) - DOC Bring Ubuntu and Archlinux outside of Windows subsection (:commit:`392233f`) -- DOC remove version suffix from ubuntu package (:commit:`5303c66`) +- DOC remove version suffix from Ubuntu package (:commit:`5303c66`) - DOC Update release date for 1.0 (:commit:`c89fa29`) .. _release-1.0.0: @@ -2211,7 +2211,7 @@ Scrapy 0.24.2 (2014-07-08) - Use a mutable mapping to proxy deprecated settings.overrides and settings.defaults attribute (:commit:`e5e8133`) - there is not support for python3 yet (:commit:`3cd6146`) -- Update python compatible version set to debian packages (:commit:`fa5d76b`) +- Update python compatible version set to Debian packages (:commit:`fa5d76b`) - DOC fix formatting in release notes (:commit:`c6a9e20`) Scrapy 0.24.1 (2014-06-27) @@ -2229,12 +2229,12 @@ Enhancements - Improve Scrapy top-level namespace (:issue:`494`, :issue:`684`) - Add selector shortcuts to responses (:issue:`554`, :issue:`690`) -- Add new lxml based LinkExtractor to replace unmantained SgmlLinkExtractor +- Add new lxml based LinkExtractor to replace unmaintained SgmlLinkExtractor (:issue:`559`, :issue:`761`, :issue:`763`) - Cleanup settings API - part of per-spider settings **GSoC project** (:issue:`737`) - Add UTF8 encoding header to templates (:issue:`688`, :issue:`762`) - Telnet console now binds to 127.0.0.1 by default (:issue:`699`) -- Update debian/ubuntu install instructions (:issue:`509`, :issue:`549`) +- Update Debian/Ubuntu install instructions (:issue:`509`, :issue:`549`) - Disable smart strings in lxml XPath evaluations (:issue:`535`) - Restore filesystem based cache as default for http cache middleware (:issue:`541`, :issue:`500`, :issue:`571`) @@ -2267,7 +2267,7 @@ Enhancements - Tests and docs for ``request_fingerprint`` function (:issue:`597`) - Update SEP-19 for GSoC project ``per-spider settings`` (:issue:`705`) - Set exit code to non-zero when contracts fails (:issue:`727`) -- Add a setting to control what class is instanciated as Downloader component +- Add a setting to control what class is instantiated as Downloader component (:issue:`738`) - Pass response in ``item_dropped`` signal (:issue:`724`) - Improve ``scrapy check`` contracts command (:issue:`733`, :issue:`752`) @@ -2276,7 +2276,7 @@ Enhancements - Add a note about reporting security issues (:issue:`697`) - Add LevelDB http cache storage backend (:issue:`626`, :issue:`500`) - Sort spider list output of ``scrapy list`` command (:issue:`742`) -- Multiple documentation enhancemens and fixes +- Multiple documentation enhancements and fixes (:issue:`575`, :issue:`587`, :issue:`590`, :issue:`596`, :issue:`610`, :issue:`617`, :issue:`618`, :issue:`627`, :issue:`613`, :issue:`643`, :issue:`654`, :issue:`675`, :issue:`663`, :issue:`711`, :issue:`714`) @@ -2321,19 +2321,19 @@ Scrapy 0.22.1 (released 2014-02-08) - BaseSgmlLinkExtractor: Added unit test of a link with an inner tag (:commit:`c1cb418`) - BaseSgmlLinkExtractor: Fixed unknown_endtag() so that it only set current_link=None when the end tag match the opening tag (:commit:`7e4d627`) - Fix tests for Travis-CI build (:commit:`76c7e20`) -- replace unencodeable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`) +- replace unencodable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`) - RegexLinkExtractor: encode URL unicode value when creating Links (:commit:`d0ee545`) - Updated the tutorial crawl output with latest output. (:commit:`8da65de`) - Updated shell docs with the crawler reference and fixed the actual shell output. (:commit:`875b9ab`) - PEP8 minor edits. (:commit:`f89efaf`) -- Expose current crawler in the scrapy shell. (:commit:`5349cec`) +- Expose current crawler in the Scrapy shell. (:commit:`5349cec`) - Unused re import and PEP8 minor edits. (:commit:`387f414`) - Ignore None's values when using the ItemLoader. (:commit:`0632546`) - DOC Fixed HTTPCACHE_STORAGE typo in the default value which is now Filesystem instead Dbm. (:commit:`cde9a8c`) -- show ubuntu setup instructions as literal code (:commit:`fb5c9c5`) +- show Ubuntu setup instructions as literal code (:commit:`fb5c9c5`) - Update Ubuntu installation instructions (:commit:`70fb105`) - Merge pull request #550 from stray-leone/patch-1 (:commit:`6f70b6a`) -- modify the version of scrapy ubuntu package (:commit:`725900d`) +- modify the version of Scrapy Ubuntu package (:commit:`725900d`) - fix 0.22.0 release date (:commit:`af0219a`) - fix typos in news.rst and remove (not released yet) header (:commit:`b7f58f4`) @@ -2354,7 +2354,7 @@ Enhancements - Improve test coverage and forthcoming Python 3 support (:issue:`525`) - Promote startup info on settings and middleware to INFO level (:issue:`520`) - Support partials in ``get_func_args`` util (:issue:`506`, issue:`504`) -- Allow running indiviual tests via tox (:issue:`503`) +- Allow running individual tests via tox (:issue:`503`) - Update extensions ignored by link extractors (:issue:`498`) - Add middleware methods to get files/images/thumbs paths (:issue:`490`) - Improve offsite middleware tests (:issue:`478`) @@ -2411,7 +2411,7 @@ Enhancements - scrapy.mail.MailSender now can connect over TLS or upgrade using STARTTLS (:issue:`327`) - New FilesPipeline with functionality factored out from ImagesPipeline (:issue:`370`, :issue:`409`) - Recommend Pillow instead of PIL for image handling (:issue:`317`) -- Added debian packages for Ubuntu quantal and raring (:commit:`86230c0`) +- Added Debian packages for Ubuntu Quantal and raring (:commit:`86230c0`) - Mock server (used for tests) can listen for HTTPS requests (:issue:`410`) - Remove multi spider support from multiple core components (:issue:`422`, :issue:`421`, :issue:`420`, :issue:`419`, :issue:`423`, :issue:`418`) @@ -2430,7 +2430,7 @@ Bugfixes - Fix tests under Django 1.6 (:commit:`b6bed44c`) - Lot of bugfixes to retry middleware under disconnections using HTTP 1.1 download handler - Fix inconsistencies among Twisted releases (:issue:`406`) -- Fix scrapy shell bugs (:issue:`418`, :issue:`407`) +- Fix Scrapy shell bugs (:issue:`418`, :issue:`407`) - Fix invalid variable name in setup.py (:issue:`429`) - Fix tutorial references (:issue:`387`) - Improve request-response docs (:issue:`391`) @@ -2512,15 +2512,15 @@ Scrapy 0.18.1 (released 2013-08-27) - test PotentiaDataLoss errors on unbound responses (:commit:`b15470d`) - Treat responses without content-length or Transfer-Encoding as good responses (:commit:`c4bf324`) - do no include ResponseFailed if http11 handler is not enabled (:commit:`6cbe684`) -- New HTTP client wraps connection losts in ResponseFailed exception. fix #373 (:commit:`1a20bba`) +- New HTTP client wraps connection lost in ResponseFailed exception. fix #373 (:commit:`1a20bba`) - limit travis-ci build matrix (:commit:`3b01bb8`) - Merge pull request #375 from peterarenot/patch-1 (:commit:`fa766d7`) - Fixed so it refers to the correct folder (:commit:`3283809`) -- added quantal & raring to support ubuntu releases (:commit:`1411923`) +- added Quantal & raring to support Ubuntu releases (:commit:`1411923`) - fix retry middleware which didn't retry certain connection errors after the upgrade to http1 client, closes GH-373 (:commit:`bb35ed0`) - fix XmlItemExporter in Python 2.7.4 and 2.7.5 (:commit:`de3e451`) - minor updates to 0.18 release notes (:commit:`c45e5f1`) -- fix contributters list format (:commit:`0b60031`) +- fix contributors list format (:commit:`0b60031`) Scrapy 0.18.0 (released 2013-08-09) ----------------------------------- @@ -2617,7 +2617,7 @@ contributors sorted by number of commits:: Scrapy 0.16.5 (released 2013-05-30) ----------------------------------- -- obey request method when scrapy deploy is redirected to a new endpoint (:commit:`8c4fcee`) +- obey request method when Scrapy deploy is redirected to a new endpoint (:commit:`8c4fcee`) - fix inaccurate downloader middleware documentation. refs #280 (:commit:`40667cb`) - doc: remove links to diveintopython.org, which is no longer available. closes #246 (:commit:`bd58bfa`) - Find form nodes in invalid html5 documents (:commit:`e3d6945`) @@ -2631,8 +2631,8 @@ Scrapy 0.16.4 (released 2013-01-23) - Fixed error message formatting. log.err() doesn't support cool formatting and when error occurred, the message was: "ERROR: Error processing %(item)s" (:commit:`c16150c`) - lint and improve images pipeline error logging (:commit:`56b45fc`) - fixed doc typos (:commit:`243be84`) -- add documentation topics: Broad Crawls & Common Practies (:commit:`1fbb715`) -- fix bug in scrapy parse command when spider is not specified explicitly. closes #209 (:commit:`c72e682`) +- add documentation topics: Broad Crawls & Common Practices (:commit:`1fbb715`) +- fix bug in Scrapy parse command when spider is not specified explicitly. closes #209 (:commit:`c72e682`) - Update docs/topics/commands.rst (:commit:`28eac7a`) Scrapy 0.16.3 (released 2012-12-07) @@ -2651,11 +2651,11 @@ Scrapy 0.16.3 (released 2012-12-07) Scrapy 0.16.2 (released 2012-11-09) ----------------------------------- -- scrapy contracts: python2.6 compat (:commit:`a4a9199`) -- scrapy contracts verbose option (:commit:`ec41673`) -- proper unittest-like output for scrapy contracts (:commit:`86635e4`) +- Scrapy contracts: python2.6 compat (:commit:`a4a9199`) +- Scrapy contracts verbose option (:commit:`ec41673`) +- proper unittest-like output for Scrapy contracts (:commit:`86635e4`) - added open_in_browser to debugging doc (:commit:`c9b690d`) -- removed reference to global scrapy stats from settings doc (:commit:`dd55067`) +- removed reference to global Scrapy stats from settings doc (:commit:`dd55067`) - Fix SpiderState bug in Windows platforms (:commit:`58998f4`) @@ -2665,7 +2665,7 @@ Scrapy 0.16.1 (released 2012-10-26) - fixed LogStats extension, which got broken after a wrong merge before the 0.16 release (:commit:`8c780fd`) - better backward compatibility for scrapy.conf.settings (:commit:`3403089`) - extended documentation on how to access crawler stats from extensions (:commit:`c4da0b5`) -- removed .hgtags (no longer needed now that scrapy uses git) (:commit:`d52c188`) +- removed .hgtags (no longer needed now that Scrapy uses git) (:commit:`d52c188`) - fix dashes under rst headers (:commit:`fa4f7f9`) - set release date for 0.16.0 in news (:commit:`e292246`) @@ -2680,8 +2680,7 @@ Scrapy changes: - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. - added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares -- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info. -- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info. +- dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info. - documented :ref:`topics-api` - ``lxml`` is now the default selectors backend instead of ``libxml2`` @@ -2715,7 +2714,7 @@ Scrapy changes: Scrapy 0.14.4 ------------- -- added precise to supported ubuntu distros (:commit:`b7e46df`) +- added precise to supported Ubuntu distros (:commit:`b7e46df`) - fixed bug in json-rpc webservice reported in https://groups.google.com/forum/#!topic/scrapy-users/qgVBmFybNAQ/discussion. also removed no longer supported 'run' command from extras/scrapy-ws.py (:commit:`340fbdb`) - meta tag attributes for content-type http equiv can be in any order. #123 (:commit:`0cb68af`) - replace "import Image" by more standard "from PIL import Image". closes #88 (:commit:`4d17048`) @@ -2728,11 +2727,11 @@ Scrapy 0.14.3 - include egg files used by testsuite in source distribution. #118 (:commit:`c897793`) - update docstring in project template to avoid confusion with genspider command, which may be considered as an advanced feature. refs #107 (:commit:`2548dcc`) - added note to docs/topics/firebug.rst about google directory being shut down (:commit:`668e352`) -- dont discard slot when empty, just save in another dict in order to recycle if needed again. (:commit:`8e9f607`) +- don't discard slot when empty, just save in another dict in order to recycle if needed again. (:commit:`8e9f607`) - do not fail handling unicode xpaths in libxml2 backed selectors (:commit:`b830e95`) - fixed minor mistake in Request objects documentation (:commit:`bf3c9ee`) - fixed minor defect in link extractors documentation (:commit:`ba14f38`) -- removed some obsolete remaining code related to sqlite support in scrapy (:commit:`0665175`) +- removed some obsolete remaining code related to sqlite support in Scrapy (:commit:`0665175`) Scrapy 0.14.2 ------------- diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index f5542495b..8816a028c 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -74,7 +74,7 @@ Request serialization For persistence to work, :class:`~scrapy.http.Request` objects must be serializable with :mod:`pickle`, except for the ``callback`` and ``errback`` values passed to their ``__init__`` method, which must be methods of the -runnning :class:`~scrapy.spiders.Spider` class. +running :class:`~scrapy.spiders.Spider` class. If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 793636f59..83b4d8154 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -110,7 +110,7 @@ ties the response lifetime to the requests' one, and that would definitely cause memory leaks. Let's see how we can discover the cause (without knowing it -a-priori, of course) by using the ``trackref`` tool. +a priori, of course) by using the ``trackref`` tool. After the crawler is running for a few minutes and we notice its memory usage has grown a lot, we can enter its telnet console and check the live diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 206e7cfa5..a40682e5b 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -558,7 +558,7 @@ See here the methods that you can override in your custom Images Pipeline: Custom Images pipeline example ============================== -Here is a full example of the Images Pipeline whose methods are examplified +Here is a full example of the Images Pipeline whose methods are exemplified above:: import scrapy diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a1d15a760..f138b7064 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -884,7 +884,7 @@ LOG_FORMAT Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'`` -String for formatting log messsages. Refer to the `Python logging documentation`_ for the whole list of available +String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available placeholders. .. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 7db7e4f6b..ecdc5423c 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -48,7 +48,7 @@ autogenerated Password can be seen on scrapy logs like the example below:: 2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326 -Default Username and Password can be overriden by the settings +Default Username and Password can be overridden by the settings :setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`. .. warning:: diff --git a/sep/sep-001.rst b/sep/sep-001.rst index 2a66f9802..00226283f 100644 --- a/sep/sep-001.rst +++ b/sep/sep-001.rst @@ -254,8 +254,8 @@ ItemForm #!python class MySiteForm(ItemForm): - witdth = adaptor(ItemForm.witdh, default_unit='cm') - volume = adaptor(ItemForm.witdh, default_unit='lt') + width = adaptor(ItemForm.width, default_unit='cm') + volume = adaptor(ItemForm.width, default_unit='lt') ia['width'] = x.x('//p[@class="width"]') ia['volume'] = x.x('//p[@class="volume"]') diff --git a/sep/sep-019.rst b/sep/sep-019.rst index 9fbf6a223..84f3a96c3 100644 --- a/sep/sep-019.rst +++ b/sep/sep-019.rst @@ -185,7 +185,7 @@ These ideas translate to the following changes on the ``SpiderManager`` class: will return a spider class, not an instance. It's basically a ``__get__`` to ``self._spiders``. -- All remaining functions should be deprecated or remove accordantly, since a +- All remaining functions should be deprecated or remove accordingly, since a crawler reference is no longer needed. - New helper ``get_spider_manager_class_from_scrapycfg`` in From 23a67cec271c849ad9b9c07a99531163dc3789fc Mon Sep 17 00:00:00 2001 From: Marc Hernandez Cabot <noviluni@gmail.com> Date: Thu, 19 Dec 2019 09:57:17 +0100 Subject: [PATCH 83/85] fix first letter capitalization for Raring and Scrapy --- docs/contributing.rst | 2 +- docs/index.rst | 2 +- docs/intro/install.rst | 16 ++++++++-------- docs/news.rst | 20 ++++++++++---------- docs/topics/autothrottle.rst | 2 +- docs/topics/commands.rst | 2 +- docs/topics/contracts.rst | 2 +- docs/topics/debug.rst | 2 +- docs/topics/developer-tools.rst | 4 ++-- docs/topics/logging.rst | 4 ++-- docs/topics/settings.rst | 4 ++-- docs/topics/shell.rst | 2 +- docs/topics/telnetconsole.rst | 2 +- sep/sep-004.rst | 2 +- 14 files changed, 33 insertions(+), 33 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index b56295027..f40a6bba2 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -44,7 +44,7 @@ guidelines when you're going to report a new bug. * check the :ref:`FAQ <faq>` first to see if your issue is addressed in a well-known question -* if you have a general question about scrapy usage, please ask it at +* if you have a general question about Scrapy usage, please ask it at `Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__ (use "scrapy" tag). diff --git a/docs/index.rst b/docs/index.rst index 6d5f9e77d..a4343b7e0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -170,7 +170,7 @@ Solving specific problems Get answers to most frequently asked questions. :doc:`topics/debug` - Learn how to debug common problems of your scrapy spider. + Learn how to debug common problems of your Scrapy spider. :doc:`topics/contracts` Learn how to use contracts for testing your spiders. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index e924b5303..0d6171884 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -78,9 +78,9 @@ TL;DR: We recommend installing Scrapy inside a virtual environment on all platforms. Python packages can be installed either globally (a.k.a system wide), -or in user-space. We do not recommend installing scrapy system wide. +or in user-space. We do not recommend installing Scrapy system wide. -Instead, we recommend that you install scrapy within a so-called +Instead, we recommend that you install Scrapy within a so-called "virtual environment" (`virtualenv`_). Virtualenvs allow you to not conflict with already-installed Python system packages (which could break some of your system tools and scripts), @@ -97,7 +97,7 @@ Check this `user guide`_ on how to create your virtualenv. .. note:: If you use Linux or OS X, `virtualenvwrapper`_ is a handy tool to create virtualenvs. -Once you have created a virtualenv, you can install scrapy inside it with ``pip``, +Once you have created a virtualenv, you can install Scrapy inside it with ``pip``, just like any other Python package. (See :ref:`platform-specific guides <intro-install-platform-notes>` below for non-Python dependencies that you may need to install beforehand). @@ -144,7 +144,7 @@ albeit with potential issues with TLS connections. typically too old and slow to catch up with latest Scrapy. -To install scrapy on Ubuntu (or Ubuntu-based) systems, you need to install +To install Scrapy on Ubuntu (or Ubuntu-based) systems, you need to install these dependencies:: sudo apt-get install python3 python3-dev python3-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev @@ -225,17 +225,17 @@ PyPy We recommend using the latest PyPy version. The version tested is 5.9.0. For PyPy3, only Linux installation was tested. -Most scrapy dependencides now have binary wheels for CPython, but not for PyPy. +Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy. This means that these dependecies will be built during installation. On OS X, you are likely to face an issue with building Cryptography dependency, solution to this problem is described `here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_, that is to ``brew install openssl`` and then export the flags that this command -recommends (only needed when installing scrapy). Installing on Linux has no special +recommends (only needed when installing Scrapy). Installing on Linux has no special issues besides installing build dependencies. -Installing scrapy with PyPy on Windows is not tested. +Installing Scrapy with PyPy on Windows is not tested. -You can check that scrapy is installed correctly by running ``scrapy bench``. +You can check that Scrapy is installed correctly by running ``scrapy bench``. If this command gives errors such as ``TypeError: ... got 2 unexpected keyword arguments``, this means that setuptools was unable to pick up one PyPy-specific dependency. diff --git a/docs/news.rst b/docs/news.rst index 28db40e3c..406e889f5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1360,7 +1360,7 @@ Documentation - Grammar fixes: :issue:`2128`, :issue:`1566`. - Download stats badge removed from README (:issue:`2160`). -- New scrapy :ref:`architecture diagram <topics-architecture>` (:issue:`2165`). +- New Scrapy :ref:`architecture diagram <topics-architecture>` (:issue:`2165`). - Updated ``Response`` parameters documentation (:issue:`2197`). - Reworded misleading :setting:`RANDOMIZE_DOWNLOAD_DELAY` description (:issue:`2190`). - Add StackOverflow as a support channel (:issue:`2257`). @@ -1450,7 +1450,7 @@ Documentation - Use "url" variable in downloader middleware example (:issue:`2015`) - Grammar fixes (:issue:`2054`, :issue:`2120`) - New FAQ entry on using BeautifulSoup in spider callbacks (:issue:`2048`) -- Add notes about scrapy not working on Windows with Python 3 (:issue:`2060`) +- Add notes about Scrapy not working on Windows with Python 3 (:issue:`2060`) - Encourage complete titles in pull requests (:issue:`2026`) Tests @@ -1509,7 +1509,7 @@ This 1.1 release brings a lot of interesting features and bug fixes: You can use :setting:`FILES_STORE_S3_ACL` to change it. - We've reimplemented ``canonicalize_url()`` for more correct output, especially for URLs with non-ASCII characters (:issue:`1947`). - This could change link extractors output compared to previous scrapy versions. + This could change link extractors output compared to previous Scrapy versions. This may also invalidate some cache entries you could still have from pre-1.1 runs. **Warning: backward incompatible!**. @@ -1722,7 +1722,7 @@ Scrapy 1.0.4 (2015-12-30) - Merge pull request #1513 from mgedmin/patch-2 (:commit:`5d4daf8`) - Typo (:commit:`f8d0682`) - Fix list formatting (:commit:`5f83a93`) -- fix scrapy squeue tests after recent changes to queuelib (:commit:`3365c01`) +- fix Scrapy squeue tests after recent changes to queuelib (:commit:`3365c01`) - Merge pull request #1475 from rweindl/patch-1 (:commit:`2d688cd`) - Update tutorial.rst (:commit:`fbc1f25`) - Merge pull request #1449 from rhoekman/patch-1 (:commit:`7d6538c`) @@ -1734,7 +1734,7 @@ Scrapy 1.0.4 (2015-12-30) Scrapy 1.0.3 (2015-08-11) ------------------------- -- add service_identity to scrapy install_requires (:commit:`cbc2501`) +- add service_identity to Scrapy install_requires (:commit:`cbc2501`) - Workaround for travis#296 (:commit:`66af9cd`) .. _release-1.0.2: @@ -2411,7 +2411,7 @@ Enhancements - scrapy.mail.MailSender now can connect over TLS or upgrade using STARTTLS (:issue:`327`) - New FilesPipeline with functionality factored out from ImagesPipeline (:issue:`370`, :issue:`409`) - Recommend Pillow instead of PIL for image handling (:issue:`317`) -- Added Debian packages for Ubuntu Quantal and raring (:commit:`86230c0`) +- Added Debian packages for Ubuntu Quantal and Raring (:commit:`86230c0`) - Mock server (used for tests) can listen for HTTPS requests (:issue:`410`) - Remove multi spider support from multiple core components (:issue:`422`, :issue:`421`, :issue:`420`, :issue:`419`, :issue:`423`, :issue:`418`) @@ -2516,7 +2516,7 @@ Scrapy 0.18.1 (released 2013-08-27) - limit travis-ci build matrix (:commit:`3b01bb8`) - Merge pull request #375 from peterarenot/patch-1 (:commit:`fa766d7`) - Fixed so it refers to the correct folder (:commit:`3283809`) -- added Quantal & raring to support Ubuntu releases (:commit:`1411923`) +- added Quantal & Raring to support Ubuntu releases (:commit:`1411923`) - fix retry middleware which didn't retry certain connection errors after the upgrade to http1 client, closes GH-373 (:commit:`bb35ed0`) - fix XmlItemExporter in Python 2.7.4 and 2.7.5 (:commit:`de3e451`) - minor updates to 0.18 release notes (:commit:`c45e5f1`) @@ -2555,8 +2555,8 @@ Scrapy 0.18.0 (released 2013-08-09) - Collect idle downloader slots (:issue:`297`) - Add ``ftp://`` scheme downloader handler (:issue:`329`) - Added downloader benchmark webserver and spider tools :ref:`benchmarking` -- Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on -- Add scrapy commands using external libraries (:issue:`260`) +- Moved persistent (on disk) queues to a separate project (queuelib_) which Scrapy now depends on +- Add Scrapy commands using external libraries (:issue:`260`) - Added ``--pdb`` option to ``scrapy`` command line tool - Added :meth:`XPathSelector.remove_namespaces <scrapy.selector.Selector.remove_namespaces>` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`. - Several improvements to spider contracts @@ -2568,7 +2568,7 @@ Scrapy 0.18.0 (released 2013-08-09) - several more cleanups to singletons and multi-spider support (thanks Nicolas Ramirez) - support custom download slots - added --spider option to "shell" command. -- log overridden settings when scrapy starts +- log overridden settings when Scrapy starts Thanks to everyone who contribute to this release. Here is a list of contributors sorted by number of commits:: diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index c9bece753..4317019fc 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -11,7 +11,7 @@ Design goals ============ 1. be nicer to sites instead of using default download delay of zero -2. automatically adjust scrapy to the optimum crawling speed, so the user +2. automatically adjust Scrapy to the optimum crawling speed, so the user doesn't have to tune the download delays to find the optimum one. The user only needs to specify the maximum concurrent requests it allows, and the extension does the rest. diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 5b3cd7e75..a0dcba90d 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -29,7 +29,7 @@ in standard locations: 1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide), 2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``) for global (user-wide) settings, and -3. ``scrapy.cfg`` inside a scrapy project's root (see next section). +3. ``scrapy.cfg`` inside a Scrapy project's root (see next section). Settings from these files are merged in the listed order of preference: user-defined values have higher priority than system-wide defaults diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 371ae62d5..43db8f101 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -64,7 +64,7 @@ Use the :command:`check` command to run the contract checks. Custom Contracts ================ -If you find you need more power than the built-in scrapy contracts you can +If you find you need more power than the built-in Scrapy contracts you can create and load your own contracts in the project by using the :setting:`SPIDER_CONTRACTS` setting:: diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 4b2588518..d75f17301 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -5,7 +5,7 @@ Debugging Spiders ================= This document explains the most common techniques for debugging spiders. -Consider the following scrapy spider below:: +Consider the following Scrapy spider below:: import scrapy from myproject.items import MyItem diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index bf14643be..d1d4ebf5d 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -80,7 +80,7 @@ expand and collapse a tag by clicking on the arrow in front of it or by double clicking directly on the tag. If we expand the ``span`` tag with the ``class= "text"`` we will see the quote-text we clicked on. The `Inspector` lets you copy XPaths to selected elements. Let's try it out: Right-click on the ``span`` -tag, select ``Copy > XPath`` and paste it in the scrapy shell like so:: +tag, select ``Copy > XPath`` and paste it in the Scrapy shell like so:: $ scrapy shell "http://quotes.toscrape.com/" (...) @@ -159,7 +159,7 @@ The page is quite similar to the basic `quotes.toscrape.com`_-page, but instead of the above-mentioned ``Next`` button, the page automatically loads new quotes when you scroll to the bottom. We could go ahead and try out different XPaths directly, but instead -we'll check another quite useful command from the scrapy shell:: +we'll check another quite useful command from the Scrapy shell:: $ scrapy shell "quotes.toscrape.com/scroll" (...) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index dd09477b8..d4d22d889 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -171,9 +171,9 @@ listed in `logging's logrecord attributes docs <https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_ respectively. -If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the scrapy +If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy component that prints the log. It is unset by default, hence logs contain the -scrapy component responsible for that log output. +Scrapy component responsible for that log output. Command-line options -------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f138b7064..afe4fade1 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1264,7 +1264,7 @@ Default:: 'scrapy.contracts.default.ScrapesContract': 3, } -A dict containing the scrapy contracts enabled by default in Scrapy. You should +A dict containing the Scrapy contracts enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS` instead. For more info see :ref:`topics-contracts`. @@ -1295,7 +1295,7 @@ SPIDER_LOADER_WARN_ONLY Default: ``False`` -By default, when scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, +By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, it will fail loudly if there is any ``ImportError`` exception. But you can choose to silence this exception and turn it into a simple warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 68a0b19b5..4fe0dea06 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -31,7 +31,7 @@ for more info. Scrapy also has support for `bpython`_, and will try to use it where `IPython`_ is unavailable. -Through scrapy's settings you can configure it to use any one of +Through Scrapy's settings you can configure it to use any one of ``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`:: diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index ecdc5423c..47d8d393c 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -44,7 +44,7 @@ the console you need to type:: >>> By default Username is ``scrapy`` and Password is autogenerated. The -autogenerated Password can be seen on scrapy logs like the example below:: +autogenerated Password can be seen on Scrapy logs like the example below:: 2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326 diff --git a/sep/sep-004.rst b/sep/sep-004.rst index 69edfa136..05b0eb99c 100644 --- a/sep/sep-004.rst +++ b/sep/sep-004.rst @@ -53,7 +53,7 @@ Here's a simple proof-of-concept code of such script: # ... do something more interesting with scraped_items ... The behaviour of the Scrapy crawler would be controller by the Scrapy settings, -naturally, just like any typical scrapy project. But the default settings +naturally, just like any typical Scrapy project. But the default settings should be sufficient so as to not require adding any specific setting. But, at the same time, you could do it if you need to, say, for specifying a custom middleware. From f6bc1940a3394f74d8aa088faf2912a43690b260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 19 Dec 2019 12:06:15 +0100 Subject: [PATCH 84/85] Use Python 3.7 to build the documentation --- .readthedocs.yml | 4 +++- .travis.yml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 3c1c3e8be..563add75f 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -2,6 +2,8 @@ version: 2 sphinx: configuration: docs/conf.py python: - version: 3.8 + # For available versions, see: + # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image + version: 3.7 # Keep in sync with .travis.yml install: - requirements: docs/requirements.txt diff --git a/.travis.yml b/.travis.yml index c870934e1..f9d0dc8be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ matrix: - env: TOXENV=extra-deps python: 3.8 - env: TOXENV=docs - python: 3.8 + python: 3.7 # Keep in sync with .readthedocs.yml install: - | if [ "$TOXENV" = "pypy3" ]; then From e22c0c27d9d33383f4ac18a4142ccffe1d9a0d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io> Date: Thu, 19 Dec 2019 12:15:54 +0100 Subject: [PATCH 85/85] Revert "Improve FilteringLinkExtractor.__new__" This reverts commit ee9881d2704798c9cd61b6da503bb0694227c58c. --- scrapy/linkextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index d0d34035d..bdeab3a75 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -60,7 +60,7 @@ class FilteringLinkExtractor(object): warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' 'please use scrapy.linkextractors.LinkExtractor instead', ScrapyDeprecationWarning, stacklevel=2) - return super().__new__(cls, *args, **kwargs) + return super(FilteringLinkExtractor, cls).__new__(cls) def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):