From 18d0affc015cd1be02cc9b227c4d89007dd8c816 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Mon, 5 Aug 2019 16:53:35 +0530 Subject: [PATCH 01/70] Update reactor.py, updated 'if' sequencing , possibly eliminating a bug if portrange=None This should be the proper ordering.This is the explanation. If 'not portrange' is True ,it is guaranteed that `not hasattr(portrange, '__iter__')` is also True the converse of this is not always true.(for example, consider portrange=None, for such case we were executing the logic for `not hasattr(portrange, '__iter__')` . ).Such case is eliminated by this PR. --- scrapy/utils/reactor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 83186a372..eda7867e3 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -3,10 +3,10 @@ from twisted.internet import reactor, error def listen_tcp(portrange, host, factory): """Like reactor.listenTCP but tries different ports in a range.""" assert len(portrange) <= 2, "invalid portrange: %s" % portrange - if not hasattr(portrange, '__iter__'): - return reactor.listenTCP(portrange, factory, interface=host) if not portrange: return reactor.listenTCP(0, factory, interface=host) + if not hasattr(portrange, '__iter__'): + return reactor.listenTCP(portrange, factory, interface=host) if len(portrange) == 1: return reactor.listenTCP(portrange[0], factory, interface=host) for x in range(portrange[0], portrange[1]+1): From 50c4cafe0ccc53064f38654fd7426ded876469f7 Mon Sep 17 00:00:00 2001 From: Tobias Hernstig Date: Sun, 11 Mar 2018 11:46:07 +0100 Subject: [PATCH 02/70] Update documentation for logging manually Usage of basicConfig() together with crawlerRunner is not recommended. Update documentation to highlight this fact. Closes #2149, #2352, #3146 --- docs/topics/logging.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 87ea43c7d..2fd85196d 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -254,18 +254,18 @@ scrapy.utils.log module when running custom scripts using :class:`~scrapy.crawler.CrawlerRunner`. In that case, its usage is not required but it's recommended. - If you plan on configuring the handlers yourself is still recommended you - call this function, passing ``install_root_handler=False``. Bear in mind - there won't be any log output set by default in that case. + Another option when running custom scripts is to manually configure the logging. + To do this you can use `logging.basicConfig()`_ to set a basic root handler. - To get you started on manually configuring logging's output, you can use - `logging.basicConfig()`_ to set a basic root handler. This is an example - on how to redirect ``INFO`` or higher messages to a file:: + Note that ``scrapy.crawler.CrawlerProcess`` automatically calls ``configure_logging``, + so it is recommended to only use `logging.basicConfig()`_ together with + ``scrapy.crawler.CrawlerRunner`` + + This is an example on how to redirect ``INFO`` or higher messages to a file:: import logging from scrapy.utils.log import configure_logging - configure_logging(install_root_handler=False) logging.basicConfig( filename='log.txt', format='%(levelname)s: %(message)s', From 2b0de0606c3b1a379722e916e9ce350e8de08a20 Mon Sep 17 00:00:00 2001 From: Tobias Hernstig Date: Thu, 15 Aug 2019 18:54:28 +0200 Subject: [PATCH 03/70] Fix merge conflicts --- docs/topics/logging.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 2fd85196d..87ab6e19a 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -257,9 +257,9 @@ scrapy.utils.log module Another option when running custom scripts is to manually configure the logging. To do this you can use `logging.basicConfig()`_ to set a basic root handler. - Note that ``scrapy.crawler.CrawlerProcess`` automatically calls ``configure_logging``, + Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``, so it is recommended to only use `logging.basicConfig()`_ together with - ``scrapy.crawler.CrawlerRunner`` + :class:`~scrapy.crawler.CrawlerRunner`. This is an example on how to redirect ``INFO`` or higher messages to a file:: From f6872189b96595625e428331ddd4d1c620047642 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 29 Aug 2019 10:38:49 -0300 Subject: [PATCH 04/70] Add LogFormatter.error method --- scrapy/core/scraper.py | 6 +++--- scrapy/logformatter.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 1f389cf2e..3273a1506 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -231,9 +231,9 @@ class Scraper(object): signal=signals.item_dropped, item=item, response=response, spider=spider, exception=output.value) else: - logger.error('Error processing %(item)s', {'item': item}, - exc_info=failure_to_exc_info(output), - extra={'spider': spider}) + logkws = self.logformatter.error(item, ex, response, spider) + logger.log(*logformatter_adapter(logkws), extra={'spider': spider}, + exc_info=failure_to_exc_info(output)) return self.signals.send_catch_log_deferred( signal=signals.item_error, item=item, response=response, spider=spider, failure=output) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index f15940ed1..4437d1106 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -8,6 +8,7 @@ from scrapy.utils.request import referer_str SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s" DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s" CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s" +ERRORMSG = u"'Error processing %(item)s'" class LogFormatter(object): @@ -92,6 +93,16 @@ class LogFormatter(object): } } + def error(self, item, exception, response, spider): + """Logs a message when an item causes an error while it is passing through the item pipeline.""" + return { + 'level': logging.ERROR, + 'msg': ERRORMSG, + 'args': { + 'item': item, + } + } + @classmethod def from_crawler(cls, crawler): return cls() From 27436cbbc9e7331d18d4b63f3c894e6621226efb Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 29 Aug 2019 13:51:42 -0300 Subject: [PATCH 05/70] [test] LogFormatter.error --- tests/test_logformatter.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index eb9c4a561..502bc4ccc 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -23,7 +23,7 @@ class CustomItem(Item): return "name: %s" % self['name'] -class LoggingContribTest(unittest.TestCase): +class LoggingFormatterTest(unittest.TestCase): def setUp(self): self.formatter = LogFormatter() @@ -61,6 +61,16 @@ class LoggingContribTest(unittest.TestCase): lines = logline.splitlines() assert all(isinstance(x, six.text_type) for x in lines) self.assertEqual(lines, [u"Dropped: \u2018", '{}']) + + def test_error(self): + # In practice, the complete traceback is shown by passing the + # 'exc_info' argument to the logging function + item = {'key': 'value'} + exception = Exception() + response = Response("http://www.example.com") + logkws = self.formatter.error(item, exception, response, self.spider) + logline = logkws['msg'] % logkws['args'] + self.assertEqual(logline, u"'Error processing {'key': 'value'}'") def test_scraped(self): item = CustomItem() @@ -75,26 +85,30 @@ class LoggingContribTest(unittest.TestCase): class LogFormatterSubclass(LogFormatter): def crawled(self, request, response, spider): - kwargs = super(LogFormatterSubclass, self).crawled( - request, response, spider) + kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider) CRAWLEDMSG = ( - u"Crawled (%(status)s) %(request)s (referer: " - u"%(referer)s)%(flags)s" + u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" ) + log_args = kwargs['args'] + log_args['flags'] = str(request.flags) return { 'level': kwargs['level'], 'msg': CRAWLEDMSG, - 'args': kwargs['args'] + 'args': log_args, } -class LogformatterSubclassTest(LoggingContribTest): +class LogformatterSubclassTest(unittest.TestCase): def setUp(self): self.formatter = LogFormatterSubclass() self.spider = Spider('default') def test_flags_in_request(self): - pass + req = Request("http://www.example.com", flags=['test','flag']) + res = Response("http://www.example.com") + logkws = self.formatter.crawled(req, res, self.spider) + logline = logkws['msg'] % logkws['args'] + self.assertEqual(logline, "Crawled (200) (referer: None) ['test', 'flag']") class SkipMessagesLogFormatter(LogFormatter): From 07a31b13db96b2aef3074157808f7ad300780c3d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Oct 2019 17:55:57 -0300 Subject: [PATCH 06/70] Update LogFormatter tests --- tests/test_logformatter.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 502bc4ccc..f12ffc11b 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -23,13 +23,13 @@ class CustomItem(Item): return "name: %s" % self['name'] -class LoggingFormatterTest(unittest.TestCase): +class LogFormatterTestCase(unittest.TestCase): def setUp(self): self.formatter = LogFormatter() self.spider = Spider('default') - def test_crawled(self): + def test_crawled_with_referer(self): req = Request("http://www.example.com") res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) @@ -37,6 +37,7 @@ class LoggingFormatterTest(unittest.TestCase): self.assertEqual(logline, "Crawled (200) (referer: None)") + def test_crawled_without_referer(self): req = Request("http://www.example.com", headers={'referer': 'http://example.com'}) res = Response("http://www.example.com", flags=['cached']) logkws = self.formatter.crawled(req, res, self.spider) @@ -98,11 +99,27 @@ class LogFormatterSubclass(LogFormatter): } -class LogformatterSubclassTest(unittest.TestCase): +class LogformatterSubclassTest(LogFormatterTestCase): def setUp(self): self.formatter = LogFormatterSubclass() self.spider = Spider('default') + def test_crawled_with_referer(self): + req = Request("http://www.example.com") + res = Response("http://www.example.com") + logkws = self.formatter.crawled(req, res, self.spider) + logline = logkws['msg'] % logkws['args'] + self.assertEqual(logline, + "Crawled (200) (referer: None) []") + + def test_crawled_without_referer(self): + req = Request("http://www.example.com", headers={'referer': 'http://example.com'}, flags=['cached']) + res = Response("http://www.example.com") + logkws = self.formatter.crawled(req, res, self.spider) + logline = logkws['msg'] % logkws['args'] + self.assertEqual(logline, + "Crawled (200) (referer: http://example.com) ['cached']") + def test_flags_in_request(self): req = Request("http://www.example.com", flags=['test','flag']) res = Response("http://www.example.com") From f52148143be614779ccfdb34cac995b16f8264ff Mon Sep 17 00:00:00 2001 From: akhter wahab Date: Mon, 7 Oct 2019 23:28:33 +0500 Subject: [PATCH 07/70] Add dmg, iso & apk to ignored other extensions --- scrapy/linkextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index ebf3cd7d8..6049c312c 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -35,7 +35,7 @@ IGNORED_EXTENSIONS = [ 'odp', # other - 'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', + 'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', 'dmg', 'iso', 'apk' ] From 877ef4269e5683bf87832e816cf20a3cac352440 Mon Sep 17 00:00:00 2001 From: akhter wahab Date: Wed, 9 Oct 2019 16:03:44 +0500 Subject: [PATCH 08/70] Add .webm to ignored video extensions --- scrapy/linkextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 6049c312c..5f6df9c73 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -28,7 +28,7 @@ IGNORED_EXTENSIONS = [ # video '3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv', - 'm4a', 'm4v', 'flv', + 'm4a', 'm4v', 'flv', 'webm', # office suites 'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg', From a25a2d5ee4755046d736efa0a03898d9d18671f9 Mon Sep 17 00:00:00 2001 From: akhter wahab Date: Wed, 9 Oct 2019 16:05:39 +0500 Subject: [PATCH 09/70] Add .tar.xz to ignored other extensions --- scrapy/linkextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 5f6df9c73..0405462d6 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -35,7 +35,7 @@ IGNORED_EXTENSIONS = [ 'odp', # other - 'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', 'dmg', 'iso', 'apk' + 'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', 'dmg', 'iso', 'apk', 'tar.xz' ] From 532770df5226757498a941746cf94ae47043e101 Mon Sep 17 00:00:00 2001 From: akhter wahab Date: Wed, 9 Oct 2019 22:53:14 +0500 Subject: [PATCH 10/70] instead of .tar.xz adding .xz in others extensions --- scrapy/linkextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 0405462d6..3c75e683d 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -35,7 +35,7 @@ IGNORED_EXTENSIONS = [ 'odp', # other - 'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', 'dmg', 'iso', 'apk', 'tar.xz' + 'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', 'dmg', 'iso', 'apk', 'xz' ] From 12f1e468e9f071e8f5c00921d0fbada460e41c57 Mon Sep 17 00:00:00 2001 From: Purva Udai Date: Sun, 13 Oct 2019 15:55:27 +0530 Subject: [PATCH 11/70] Issue #3731 --- scrapy/extensions/feedexport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index ce2846eba..981efee55 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -199,9 +199,9 @@ class FeedExporter(object): def __init__(self, settings): self.settings = settings - self.urifmt = settings['FEED_URI'] - if not self.urifmt: + if not settings['FEED_URI']: raise NotConfigured + self.urifmt=str(settings['FEED_URI']) self.format = settings['FEED_FORMAT'].lower() self.export_encoding = settings['FEED_EXPORT_ENCODING'] self.storages = self._load_components('FEED_STORAGES') From 865d58fd1b5676dcca44b64970b5cd48fc1ab715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Alberto=20Orejuela=20Garc=C3=ADa?= Date: Thu, 3 Oct 2019 18:06:19 +0200 Subject: [PATCH 12/70] Make punctuation consistent --- CODE_OF_CONDUCT.md | 2 +- README.rst | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index d477168eb..d1cd3e517 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -68,7 +68,7 @@ members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at [http://contributor-covenant.org/version/1/4][version] +available at [http://contributor-covenant.org/version/1/4][version]. [homepage]: http://contributor-covenant.org [version]: http://contributor-covenant.org/version/1/4/ diff --git a/README.rst b/README.rst index bd82bff06..87eaac2af 100644 --- a/README.rst +++ b/README.rst @@ -34,8 +34,8 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages. It can be used for a wide range of purposes, from data mining to monitoring and automated testing. -For more information including a list of features check the Scrapy homepage at: -https://scrapy.org +Check the Scrapy homepage at https://scrapy.org for more information, +including a list of features. Requirements ============ @@ -50,8 +50,8 @@ The quick way:: pip install scrapy -For more details see the install section in the documentation: -https://docs.scrapy.org/en/latest/intro/install.html +See the install section in the documentation at +https://docs.scrapy.org/en/latest/intro/install.html for more details. Documentation ============= @@ -62,17 +62,17 @@ directory. Releases ======== -You can find release notes at https://docs.scrapy.org/en/latest/news.html +You can check https://docs.scrapy.org/en/latest/news.html for release notes. Community (blog, twitter, mail list, IRC) ========================================= -See https://scrapy.org/community/ +See https://scrapy.org/community/ for details. Contributing ============ -See https://docs.scrapy.org/en/master/contributing.html +See https://docs.scrapy.org/en/master/contributing.html for details. Code of Conduct --------------- @@ -86,9 +86,9 @@ Please report unacceptable behavior to opensource@scrapinghub.com. Companies using Scrapy ====================== -See https://scrapy.org/companies/ +See https://scrapy.org/companies/ for a list. Commercial Support ================== -See https://scrapy.org/support/ +See https://scrapy.org/support/ for details. From 68a7d05ed86331ae70778c83a131e6dcc8b4ffc8 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Mon, 21 Oct 2019 15:42:24 +0200 Subject: [PATCH 13/70] docs: use __init__ method instead of constructor Issue #4086 --- docs/conf.py | 2 +- docs/news.rst | 10 ++++---- docs/topics/email.rst | 4 ++-- docs/topics/exporters.rst | 40 +++++++++++++++---------------- docs/topics/extensions.rst | 2 +- docs/topics/items.rst | 8 +++---- docs/topics/loaders.rst | 26 ++++++++++---------- docs/topics/request-response.rst | 14 +++++------ scrapy/exporters.py | 2 +- scrapy/extensions/feedexport.py | 2 +- scrapy/utils/datatypes.py | 2 +- scrapy/utils/misc.py | 6 ++--- scrapy/utils/python.py | 2 +- sep/sep-009.rst | 12 +++++----- tests/test_http_request.py | 6 ++--- tests/test_http_response.py | 2 +- tests/test_loader.py | 12 +++++----- tests/test_spider.py | 4 ++-- tests/test_utils_misc/__init__.py | 10 ++++---- 19 files changed, 83 insertions(+), 83 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 34dd5bcb7..6ab5959d5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -237,7 +237,7 @@ coverage_ignore_pyobjects = [ r'\bContractsManager\b$', # For default contracts we only want to document their general purpose in - # their constructor, the methods they reimplement to achieve that purpose + # their __init__ method, the methods they reimplement to achieve that purpose # should be irrelevant to developers using those contracts. r'\w+Contract\.(adjust_request_args|(pre|post)_process)$', diff --git a/docs/news.rst b/docs/news.rst index 59317f5eb..c1f548060 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -84,12 +84,12 @@ New features convenient way to build JSON requests (:issue:`3504`, :issue:`3505`) * A ``process_request`` callback passed to the :class:`~scrapy.spiders.Rule` - constructor now receives the :class:`~scrapy.http.Response` object that + ``__init__`` method now receives the :class:`~scrapy.http.Response` object that originated the request as its second argument (:issue:`3682`) * A new ``restrict_text`` parameter for the :attr:`LinkExtractor ` - constructor allows filtering links by linking text (:issue:`3622`, + ``__init__`` method allows filtering links by linking text (:issue:`3622`, :issue:`3635`) * A new :setting:`FEED_STORAGE_S3_ACL` setting allows defining a custom ACL @@ -255,7 +255,7 @@ The following deprecated APIs have been removed (:issue:`3578`): * From :class:`~scrapy.selector.Selector`: - * ``_root`` (both the constructor argument and the object property, use + * ``_root`` (both the ``__init__`` method argument and the object property, use ``root``) * ``extract_unquoted`` (use ``getall``) @@ -2479,7 +2479,7 @@ Scrapy changes: - removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library - promoted :ref:`topics-djangoitem` to main contrib - LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`) -- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the constructor +- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method - replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module - removed signal: ``scrapy.mail.mail_sent`` - removed ``TRACK_REFS`` setting, now :ref:`trackrefs ` is always enabled @@ -2693,7 +2693,7 @@ API changes - ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231) - Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default) - Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225) -- Removed Spider Manager ``load()`` method. Now spiders are loaded in the constructor itself. +- Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself. - Changes to Scrapy Manager (now called "Crawler"): - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` - ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler`` diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 949cdc638..12eedf2cd 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -21,7 +21,7 @@ Quick example ============= There are two ways to instantiate the mail sender. You can instantiate it using -the standard constructor:: +the standard ``__init__`` method:: from scrapy.mail import MailSender mailer = MailSender() @@ -111,7 +111,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework. Mail settings ============= -These settings define the default constructor values of the :class:`MailSender` +These settings define the default ``__init__`` method values of the :class:`MailSender` class, and can be used to configure e-mail notifications in your project without writing any code (for those extensions and code that uses :class:`MailSender`). diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index a698a6a4e..d7ab7a5cc 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -87,8 +87,8 @@ described next. 1. Declaring a serializer in the field -------------------------------------- -If you use :class:`~.Item` you can declare a serializer in the -:ref:`field metadata `. The serializer must be +If you use :class:`~.Item` you can declare a serializer in the +:ref:`field metadata `. The serializer must be a callable which receives a value and returns its serialized form. Example:: @@ -144,7 +144,7 @@ BaseItemExporter defining what fields to export, whether to export empty fields, or which encoding to use. - These features can be configured through the constructor arguments which + These features can be configured through the `__init__` method arguments which populate their respective instance attributes: :attr:`fields_to_export`, :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`. @@ -246,8 +246,8 @@ XmlItemExporter :param item_element: The name of each item element in the exported XML. :type item_element: str - The additional keyword arguments of this constructor are passed to the - :class:`BaseItemExporter` constructor. + The additional keyword arguments of this `__init__` method are passed to the + :class:`BaseItemExporter` `__init__` method A typical output of this exporter would be:: @@ -306,9 +306,9 @@ CsvItemExporter multi-valued fields, if found. :type include_headers_line: str - The additional keyword arguments of this constructor are passed to the - :class:`BaseItemExporter` constructor, and the leftover arguments to the - `csv.writer`_ constructor, so you can use any ``csv.writer`` constructor + The additional keyword arguments of this `__init__` method are passed to the + :class:`BaseItemExporter` `__init__` method, and the leftover arguments to the + `csv.writer`_ `__init__` method, so you can use any ``csv.writer`` `__init__` method argument to customize this exporter. A typical output of this exporter would be:: @@ -334,8 +334,8 @@ PickleItemExporter For more information, refer to the `pickle module documentation`_. - The additional keyword arguments of this constructor are passed to the - :class:`BaseItemExporter` constructor. + The additional keyword arguments of this `__init__` method are passed to the + :class:`BaseItemExporter` `__init__` method. Pickle isn't a human readable format, so no output examples are provided. @@ -351,8 +351,8 @@ PprintItemExporter :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) - The additional keyword arguments of this constructor are passed to the - :class:`BaseItemExporter` constructor. + The additional keyword arguments of this `__init__` method are passed to the + :class:`BaseItemExporter` `__init__` method A typical output of this exporter would be:: @@ -367,10 +367,10 @@ JsonItemExporter .. class:: JsonItemExporter(file, \**kwargs) Exports Items in JSON format to the specified file-like object, writing all - objects as a list of objects. The additional constructor arguments are - passed to the :class:`BaseItemExporter` constructor, and the leftover - arguments to the `JSONEncoder`_ constructor, so you can use any - `JSONEncoder`_ constructor argument to customize this exporter. + objects as a list of objects. The additional `__init__` method arguments are + passed to the :class:`BaseItemExporter` `__init__` method, and the leftover + arguments to the `JSONEncoder`_ `__init__` method, so you can use any + `JSONEncoder`_ `__init__` method argument to customize this exporter. :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) @@ -398,10 +398,10 @@ JsonLinesItemExporter .. class:: JsonLinesItemExporter(file, \**kwargs) Exports Items in JSON format to the specified file-like object, writing one - JSON-encoded item per line. The additional constructor arguments are passed - to the :class:`BaseItemExporter` constructor, and the leftover arguments to - the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_ - constructor argument to customize this exporter. + JSON-encoded item per line. The additional `__init__` method arguments are passed + to the :class:`BaseItemExporter` `__init__` method and the leftover arguments to + the `JSONEncoder`_ `__init__` method, so you can use any `JSONEncoder`_ + `__init__` method argument to customize this exporter. :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 72c2290b5..0a7455ec9 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -28,7 +28,7 @@ Loading & activating extensions Extensions are loaded and activated at startup by instantiating a single instance of the extension class. Therefore, all the extension initialization -code must be performed in the class constructor (``__init__`` method). +code must be performed in the class ``__init__`` method. To make an extension available, add it to the :setting:`EXTENSIONS` setting in your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented diff --git a/docs/topics/items.rst b/docs/topics/items.rst index d70e7428b..8ea2635cc 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -16,12 +16,12 @@ especially in a larger project with many spiders. To define common output data format Scrapy provides the :class:`Item` class. :class:`Item` objects are simple containers used to collect the scraped data. They provide a `dictionary-like`_ API with a convenient syntax for declaring -their available fields. +their available fields. -Various Scrapy components use extra information provided by Items: +Various Scrapy components use extra information provided by Items: exporters look at declared fields to figure out columns to export, serialization can be customized using Item fields metadata, :mod:`trackref` -tracks Item instances to help find memory leaks +tracks Item instances to help find memory leaks (see :ref:`topics-leaks-trackrefs`), etc. .. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict @@ -237,7 +237,7 @@ Item objects Return a new Item optionally initialized from the given argument. - Items replicate the standard `dict API`_, including its constructor, and + Items replicate the standard `dict API`_, including its `__init__` method and also provide the following additional API members: .. automethod:: copy diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 1c2f1da4d..db1175a13 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -26,7 +26,7 @@ Using Item Loaders to populate items To use an Item Loader, you must first instantiate it. You can either instantiate it with a dict-like object (e.g. Item or dict) or without one, in -which case an Item is automatically instantiated in the Item Loader constructor +which case an Item is automatically instantiated in the Item Loader ``__init__`` method using the Item class specified in the :attr:`ItemLoader.default_item_class` attribute. @@ -265,7 +265,7 @@ There are several ways to modify Item Loader context values: loader.context['unit'] = 'cm' 2. On Item Loader instantiation (the keyword arguments of Item Loader - constructor are stored in the Item Loader context):: + ``__init__`` methodare stored in the Item Loader context):: loader = ItemLoader(product, unit='cm') @@ -494,7 +494,7 @@ ItemLoader objects .. attribute:: default_item_class An Item class (or factory), used to instantiate items when not given in - the constructor. + the `__init__` method .. attribute:: default_input_processor @@ -509,15 +509,15 @@ ItemLoader objects .. attribute:: default_selector_class The class used to construct the :attr:`selector` of this - :class:`ItemLoader`, if only a response is given in the constructor. - If a selector is given in the constructor this attribute is ignored. + :class:`ItemLoader`, if only a response is given in the `__init__` method + If a selector is given in the `__init__` method this attribute is ignored. This attribute is sometimes overridden in subclasses. .. attribute:: selector The :class:`~scrapy.selector.Selector` object to extract data from. - It's either the selector given in the constructor or one created from - the response given in the constructor using the + It's either the selector given in the `__init__` methodor one created from + the response given in the `__init__` methodusing the :attr:`default_selector_class`. This attribute is meant to be read-only. @@ -642,7 +642,7 @@ Here is a list of all built-in processors: .. class:: Identity The simplest processor, which doesn't do anything. It returns the original - values unchanged. It doesn't receive any constructor arguments, nor does it + values unchanged. It doesn't receive any `__init__` method arguments, nor does it accept Loader contexts. Example:: @@ -656,7 +656,7 @@ Here is a list of all built-in processors: Returns the first non-null/non-empty value from the values received, so it's typically used as an output processor to single-valued fields. - It doesn't receive any constructor arguments, nor does it accept Loader contexts. + It doesn't receive any `__init__` methodarguments, nor does it accept Loader contexts. Example:: @@ -667,7 +667,7 @@ Here is a list of all built-in processors: .. class:: Join(separator=u' ') - Returns the values joined with the separator given in the constructor, which + Returns the values joined with the separator given in the `__init__` method which defaults to ``u' '``. It doesn't accept Loader contexts. When using the default separator, this processor is equivalent to the @@ -705,7 +705,7 @@ Here is a list of all built-in processors: those which do, this processor will pass the currently active :ref:`Loader context ` through that parameter. - The keyword arguments passed in the constructor are used as the default + The keyword arguments passed in the `__init__` methodare used as the default Loader context values passed to each function call. However, the final Loader context values passed to functions are overridden with the currently active Loader context accessible through the :meth:`ItemLoader.context` @@ -749,12 +749,12 @@ Here is a list of all built-in processors: ['HELLO, 'THIS', 'IS', 'SCRAPY'] As with the Compose processor, functions can receive Loader contexts, and - constructor keyword arguments are used as default context values. See + `__init__` method keyword arguments are used as default context values. See :class:`Compose` processor for more info. .. class:: SelectJmes(json_path) - Queries the value using the json path provided to the constructor and returns the output. + Queries the value using the json path provided to the `__init__` methodand returns the output. Requires jmespath (https://github.com/jmespath/jmespath.py) to run. This processor takes only one input at a time. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 727c67482..d253064f2 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -137,7 +137,7 @@ Request objects A string containing the URL of this request. Keep in mind that this attribute contains the escaped URL, so it can differ from the URL passed in - the constructor. + the `__init__` method This attribute is read-only. To change the URL of a Request use :meth:`replace`. @@ -400,7 +400,7 @@ fields with form data from :class:`Response` objects. .. class:: FormRequest(url, [formdata, ...]) - The :class:`FormRequest` class adds a new keyword parameter to the constructor. The + The :class:`FormRequest` class adds a new keyword parameter to the `__init__` method The remaining arguments are the same as for the :class:`Request` class and are not documented here. @@ -473,7 +473,7 @@ fields with form data from :class:`Response` objects. :type dont_click: boolean The other parameters of this class method are passed directly to the - :class:`FormRequest` constructor. + :class:`FormRequest` `__init__` method .. versionadded:: 0.10.3 The ``formname`` parameter. @@ -547,7 +547,7 @@ dealing with JSON requests. .. class:: JsonRequest(url, [... data, dumps_kwargs]) - The :class:`JsonRequest` class adds two new keyword parameters to the constructor. The + The :class:`JsonRequest` class adds two new keyword parameters to the `__init__` method The remaining arguments are the same as for the :class:`Request` class and are not documented here. @@ -556,7 +556,7 @@ dealing with JSON requests. :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body. if :attr:`Request.body` argument is provided this parameter will be ignored. - if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be + if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be set to ``'POST'`` automatically. :type data: JSON serializable object @@ -721,7 +721,7 @@ TextResponse objects :class:`Response` class, which is meant to be used only for binary data, such as images, sounds or any media file. - :class:`TextResponse` objects support a new constructor argument, in + :class:`TextResponse` objects support a new `__init__` method argument, in addition to the base :class:`Response` objects. The remaining functionality is the same as for the :class:`Response` class and is not documented here. @@ -755,7 +755,7 @@ TextResponse objects A string with the encoding of this response. The encoding is resolved by trying the following mechanisms, in order: - 1. the encoding passed in the constructor ``encoding`` argument + 1. the encoding passed in the `__init__` method`` ``encoding`` argument 2. the encoding declared in the Content-Type HTTP header. If this encoding is not valid (ie. unknown), it is ignored and the next diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 6fc87ed18..2fdc86b63 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -31,7 +31,7 @@ class BaseItemExporter(object): def _configure(self, options, dont_fail=False): """Configure the exporter by poping options from the ``options`` dict. If dont_fail is set, it won't raise an exception on unexpected options - (useful for using with keyword arguments in subclasses constructors) + (useful for using with keyword arguments in subclasses __init__ methods) """ self.encoding = options.pop('encoding', None) self.fields_to_export = options.pop('fields_to_export', None) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index ce2846eba..655c10482 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -106,7 +106,7 @@ class S3FeedStorage(BlockingFeedStorage): warnings.warn( "Initialising `scrapy.extensions.feedexport.S3FeedStorage` " "without AWS keys is deprecated. Please supply credentials or " - "use the `from_crawler()` constructor.", + "use the `from_crawler()` __init__ method.", category=ScrapyDeprecationWarning, stacklevel=2 ) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index df2b99c28..0dfee1b27 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -246,7 +246,7 @@ class CaselessDict(dict): class MergeDict(object): """ A simple class for creating new "virtual" dictionaries that actually look - up values in more than one dictionary, passed in the constructor. + up values in more than one dictionary, passed in the __init__ method. If a key appears in more than one of the given dictionaries, only the first occurrence will be used. diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index f638adb25..c892bd438 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -123,14 +123,14 @@ def rel_has_nofollow(rel): def create_instance(objcls, settings, crawler, *args, **kwargs): """Construct a class instance using its ``from_crawler`` or - ``from_settings`` constructors, if available. + ``from_settings`` __init__ method, if available. At least one of ``settings`` and ``crawler`` needs to be different from ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used. - If ``crawler`` is ``None``, only the ``from_settings`` constructor will be + If ``crawler`` is ``None``, only the ``from_settings`` __init__ method will be tried. - ``*args`` and ``**kwargs`` are forwarded to the constructors. + ``*args`` and ``**kwargs`` are forwarded to the __init__ methods. Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``. """ diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index c6140f885..0d645543a 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -303,7 +303,7 @@ class WeakKeyCache(object): def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): """Return a (new) dict with unicode keys (and values when "keys_only" is False) of the given dict converted to strings. ``dct_or_tuples`` can be a - dict or a list of tuples, like any dict constructor supports. + dict or a list of tuples, like any dict __init__ method supports. """ d = {} for k, v in six.iteritems(dict(dct_or_tuples)): diff --git a/sep/sep-009.rst b/sep/sep-009.rst index 232a536a8..929cedc61 100644 --- a/sep/sep-009.rst +++ b/sep/sep-009.rst @@ -38,7 +38,7 @@ singletons members of that object, as explained below: ``scrapy.core.manager.ExecutionManager``) - instantiated with a ``Settings`` object - - **crawler.settings**: ``scrapy.conf.Settings`` instance (passed in the constructor) + - **crawler.settings**: ``scrapy.conf.Settings`` instance (passed in the ``__init__`` method) - **crawler.extensions**: ``scrapy.extension.ExtensionManager`` instance - **crawler.engine**: ``scrapy.core.engine.ExecutionEngine`` instance - ``crawler.engine.scheduler`` @@ -55,7 +55,7 @@ singletons members of that object, as explained below: ``STATS_CLASS`` setting) - **crawler.log**: Logger class with methods replacing the current ``scrapy.log`` functions. Logging would be started (if enabled) on - ``Crawler`` constructor, so no log starting functions are required. + ``Crawler`` __init__ method, so no log starting functions are required. - ``crawler.log.msg`` - **crawler.signals**: signal handling @@ -69,12 +69,12 @@ Required code changes after singletons removal ============================================== All components (extensions, middlewares, etc) will receive this ``Crawler`` -object in their constructors, and this will be the only mechanism for accessing +object in their ``__init__`` methods, and this will be the only mechanism for accessing any other components (as opposed to importing each singleton from their respective module). This will also serve to stabilize the core API, something which we haven't documented so far (partly because of this). -So, for a typical middleware constructor code, instead of this: +So, for a typical middleware ``__init__`` method code, instead of this: :: @@ -125,13 +125,13 @@ Open issues to resolve - Should we pass ``Settings`` object to ``ScrapyCommand.add_options()``? - How should spiders access settings? - - Option 1. Pass ``Crawler`` object to spider constructors too + - Option 1. Pass ``Crawler`` object to spider ``__init__`` methods too - pro: one way to access all components (settings and signals being the most relevant to spiders) - con?: spider code can access (and control) any crawler component - since we don't want to support spiders messing with the crawler (write an extension or spider middleware if you need that) - - Option 2. Pass ``Settings`` object to spider constructors, which would + - Option 2. Pass ``Settings`` object to spider ``__init__`` methods, which would then be accessed through ``self.settings``, like logging which is accessed through ``self.log`` diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 16d7a1cb8..96a4fb141 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -25,7 +25,7 @@ class RequestTest(unittest.TestCase): default_meta = {} def test_init(self): - # Request requires url in the constructor + # Request requires url in the __init__ method self.assertRaises(Exception, self.request_class) # url argument must be basestring @@ -500,7 +500,7 @@ class FormRequestTest(RequestTest): formdata=(('foo', 'bar'), ('foo', 'baz'))) self.assertEqual(urlparse(req.url).hostname, 'www.example.com') self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz') - + def test_from_response_override_duplicate_form_key(self): response = _buildresponse( """
@@ -657,7 +657,7 @@ class FormRequestTest(RequestTest): req = self.request_class.from_response(response, dont_click=True) fs = _qs(req) self.assertEqual(fs, {b'i1': [b'i1v'], b'i2': [b'i2v']}) - + def test_from_response_clickdata_does_not_ignore_image(self): response = _buildresponse( """ diff --git a/tests/test_http_response.py b/tests/test_http_response.py index cd5c3486e..dfc8562f3 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -534,7 +534,7 @@ class XmlResponseTest(TextResponseTest): r2 = self.response_class("http://www.example.com", body=body) self._assert_response_values(r2, 'iso-8859-1', body) - # make sure replace() preserves the explicit encoding passed in the constructor + # make sure replace() preserves the explicit encoding passed in the __init__ method body = b"""""" r3 = self.response_class("http://www.example.com", body=body, encoding='utf-8') body2 = b"New body" diff --git a/tests/test_loader.py b/tests/test_loader.py index 2725b001a..bcc1e6421 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -548,11 +548,11 @@ class SelectortemLoaderTest(unittest.TestCase): """) - def test_constructor(self): + def test_init_method(self): l = TestItemLoader() self.assertEqual(l.selector, None) - def test_constructor_errors(self): + def test_init_method_errors(self): l = TestItemLoader() self.assertRaises(RuntimeError, l.add_xpath, 'url', '//a/@href') self.assertRaises(RuntimeError, l.replace_xpath, 'url', '//a/@href') @@ -561,7 +561,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertRaises(RuntimeError, l.replace_css, 'name', '#name::text') self.assertRaises(RuntimeError, l.get_css, '#name::text') - def test_constructor_with_selector(self): + def test_init_method_with_selector(self): sel = Selector(text=u"
marta
") l = TestItemLoader(selector=sel) self.assertIs(l.selector, sel) @@ -569,7 +569,7 @@ class SelectortemLoaderTest(unittest.TestCase): l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) - def test_constructor_with_selector_css(self): + def test_init_method_with_selector_css(self): sel = Selector(text=u"
marta
") l = TestItemLoader(selector=sel) self.assertIs(l.selector, sel) @@ -577,14 +577,14 @@ class SelectortemLoaderTest(unittest.TestCase): l.add_css('name', 'div::text') self.assertEqual(l.get_output_value('name'), [u'Marta']) - def test_constructor_with_response(self): + def test_init_method_with_response(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) - def test_constructor_with_response_css(self): + def test_init_method_with_response_css(self): l = TestItemLoader(response=self.response) self.assertTrue(l.selector) diff --git a/tests/test_spider.py b/tests/test_spider.py index 2220b8ffc..64ff40b61 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -42,12 +42,12 @@ class SpiderTest(unittest.TestCase): self.assertEqual(list(start_requests), []) def test_spider_args(self): - """Constructor arguments are assigned to spider attributes""" + """__init__ method arguments are assigned to spider attributes""" spider = self.spider_class('example.com', foo='bar') self.assertEqual(spider.foo, 'bar') def test_spider_without_name(self): - """Constructor arguments are assigned to spider attributes""" + """__init__ method arguments are assigned to spider attributes""" self.assertRaises(ValueError, self.spider_class) self.assertRaises(ValueError, self.spider_class, somearg='foo') diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index e109d5343..457a2aa78 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -109,11 +109,11 @@ class UtilsMiscTestCase(unittest.TestCase): else: mock.assert_called_once_with(*args, **kwargs) - # Check usage of correct constructor using four mocks: - # 1. with no alternative constructors - # 2. with from_settings() constructor - # 3. with from_crawler() constructor - # 4. with from_settings() and from_crawler() constructor + # Check usage of correct __init__ method using four mocks: + # 1. with no alternative __init__ methods + # 2. with from_settings() __init__ method + # 3. with from_crawler() __init__ method + # 4. with from_settings() and from_crawler() __init__ method spec_sets = ([], ['from_settings'], ['from_crawler'], ['from_settings', 'from_crawler']) for specs in spec_sets: From da8cd9448de28bdeac7343bcd367e95237423789 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Mon, 21 Oct 2019 19:48:13 +0200 Subject: [PATCH 14/70] docs: always surround __init__ with `` in docs Issue #4086 --- docs/topics/exporters.rst | 36 ++++++++++++++++---------------- docs/topics/items.rst | 2 +- docs/topics/loaders.rst | 22 +++++++++---------- docs/topics/request-response.rst | 12 +++++------ scrapy/exporters.py | 2 +- scrapy/extensions/feedexport.py | 2 +- scrapy/utils/datatypes.py | 2 +- scrapy/utils/misc.py | 6 +++--- scrapy/utils/python.py | 2 +- sep/sep-009.rst | 2 +- tests/test_spider.py | 4 ++-- 11 files changed, 46 insertions(+), 46 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index d7ab7a5cc..1b8a69ca3 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -144,7 +144,7 @@ BaseItemExporter defining what fields to export, whether to export empty fields, or which encoding to use. - These features can be configured through the `__init__` method arguments which + These features can be configured through the ``__init__`` method arguments which populate their respective instance attributes: :attr:`fields_to_export`, :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`. @@ -246,8 +246,8 @@ XmlItemExporter :param item_element: The name of each item element in the exported XML. :type item_element: str - The additional keyword arguments of this `__init__` method are passed to the - :class:`BaseItemExporter` `__init__` method + The additional keyword arguments of this ``__init__`` method are passed to the + :class:`BaseItemExporter` ``__init__`` method A typical output of this exporter would be:: @@ -306,9 +306,9 @@ CsvItemExporter multi-valued fields, if found. :type include_headers_line: str - The additional keyword arguments of this `__init__` method are passed to the - :class:`BaseItemExporter` `__init__` method, and the leftover arguments to the - `csv.writer`_ `__init__` method, so you can use any ``csv.writer`` `__init__` method + The additional keyword arguments of this ``__init__`` method are passed to the + :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the + `csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method argument to customize this exporter. A typical output of this exporter would be:: @@ -334,8 +334,8 @@ PickleItemExporter For more information, refer to the `pickle module documentation`_. - The additional keyword arguments of this `__init__` method are passed to the - :class:`BaseItemExporter` `__init__` method. + The additional keyword arguments of this ``__init__`` method are passed to the + :class:`BaseItemExporter` ``__init__`` method. Pickle isn't a human readable format, so no output examples are provided. @@ -351,8 +351,8 @@ PprintItemExporter :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) - The additional keyword arguments of this `__init__` method are passed to the - :class:`BaseItemExporter` `__init__` method + The additional keyword arguments of this ``__init__`` method are passed to the + :class:`BaseItemExporter` ``__init__`` method A typical output of this exporter would be:: @@ -367,10 +367,10 @@ JsonItemExporter .. class:: JsonItemExporter(file, \**kwargs) Exports Items in JSON format to the specified file-like object, writing all - objects as a list of objects. The additional `__init__` method arguments are - passed to the :class:`BaseItemExporter` `__init__` method, and the leftover - arguments to the `JSONEncoder`_ `__init__` method, so you can use any - `JSONEncoder`_ `__init__` method argument to customize this exporter. + objects as a list of objects. The additional ``__init__`` method arguments are + passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover + arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any + `JSONEncoder`_ ``__init__`` method argument to customize this exporter. :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) @@ -398,10 +398,10 @@ JsonLinesItemExporter .. class:: JsonLinesItemExporter(file, \**kwargs) Exports Items in JSON format to the specified file-like object, writing one - JSON-encoded item per line. The additional `__init__` method arguments are passed - to the :class:`BaseItemExporter` `__init__` method and the leftover arguments to - the `JSONEncoder`_ `__init__` method, so you can use any `JSONEncoder`_ - `__init__` method argument to customize this exporter. + JSON-encoded item per line. The additional ``__init__`` method arguments are passed + to the :class:`BaseItemExporter` ``__init__`` method and the leftover arguments to + the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_ + ``__init__`` method argument to customize this exporter. :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 8ea2635cc..370409026 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -237,7 +237,7 @@ Item objects Return a new Item optionally initialized from the given argument. - Items replicate the standard `dict API`_, including its `__init__` method and + Items replicate the standard `dict API`_, including its ``__init__`` method and also provide the following additional API members: .. automethod:: copy diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index db1175a13..72610f645 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -494,7 +494,7 @@ ItemLoader objects .. attribute:: default_item_class An Item class (or factory), used to instantiate items when not given in - the `__init__` method + the ``__init__`` method .. attribute:: default_input_processor @@ -509,15 +509,15 @@ ItemLoader objects .. attribute:: default_selector_class The class used to construct the :attr:`selector` of this - :class:`ItemLoader`, if only a response is given in the `__init__` method - If a selector is given in the `__init__` method this attribute is ignored. + :class:`ItemLoader`, if only a response is given in the ``__init__`` method + If a selector is given in the ``__init__`` method this attribute is ignored. This attribute is sometimes overridden in subclasses. .. attribute:: selector The :class:`~scrapy.selector.Selector` object to extract data from. - It's either the selector given in the `__init__` methodor one created from - the response given in the `__init__` methodusing the + It's either the selector given in the ``__init__`` methodor one created from + the response given in the ``__init__`` methodusing the :attr:`default_selector_class`. This attribute is meant to be read-only. @@ -642,7 +642,7 @@ Here is a list of all built-in processors: .. class:: Identity The simplest processor, which doesn't do anything. It returns the original - values unchanged. It doesn't receive any `__init__` method arguments, nor does it + values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts. Example:: @@ -656,7 +656,7 @@ Here is a list of all built-in processors: Returns the first non-null/non-empty value from the values received, so it's typically used as an output processor to single-valued fields. - It doesn't receive any `__init__` methodarguments, nor does it accept Loader contexts. + It doesn't receive any ``__init__`` methodarguments, nor does it accept Loader contexts. Example:: @@ -667,7 +667,7 @@ Here is a list of all built-in processors: .. class:: Join(separator=u' ') - Returns the values joined with the separator given in the `__init__` method which + Returns the values joined with the separator given in the ``__init__`` method which defaults to ``u' '``. It doesn't accept Loader contexts. When using the default separator, this processor is equivalent to the @@ -705,7 +705,7 @@ Here is a list of all built-in processors: those which do, this processor will pass the currently active :ref:`Loader context ` through that parameter. - The keyword arguments passed in the `__init__` methodare used as the default + The keyword arguments passed in the ``__init__`` methodare used as the default Loader context values passed to each function call. However, the final Loader context values passed to functions are overridden with the currently active Loader context accessible through the :meth:`ItemLoader.context` @@ -749,12 +749,12 @@ Here is a list of all built-in processors: ['HELLO, 'THIS', 'IS', 'SCRAPY'] As with the Compose processor, functions can receive Loader contexts, and - `__init__` method keyword arguments are used as default context values. See + ``__init__`` method keyword arguments are used as default context values. See :class:`Compose` processor for more info. .. class:: SelectJmes(json_path) - Queries the value using the json path provided to the `__init__` methodand returns the output. + Queries the value using the json path provided to the ``__init__`` methodand returns the output. Requires jmespath (https://github.com/jmespath/jmespath.py) to run. This processor takes only one input at a time. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d253064f2..bf6a02a1d 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -137,7 +137,7 @@ Request objects A string containing the URL of this request. Keep in mind that this attribute contains the escaped URL, so it can differ from the URL passed in - the `__init__` method + the ``__init__`` method This attribute is read-only. To change the URL of a Request use :meth:`replace`. @@ -400,7 +400,7 @@ fields with form data from :class:`Response` objects. .. class:: FormRequest(url, [formdata, ...]) - The :class:`FormRequest` class adds a new keyword parameter to the `__init__` method The + The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method The remaining arguments are the same as for the :class:`Request` class and are not documented here. @@ -473,7 +473,7 @@ fields with form data from :class:`Response` objects. :type dont_click: boolean The other parameters of this class method are passed directly to the - :class:`FormRequest` `__init__` method + :class:`FormRequest` ``__init__`` method .. versionadded:: 0.10.3 The ``formname`` parameter. @@ -547,7 +547,7 @@ dealing with JSON requests. .. class:: JsonRequest(url, [... data, dumps_kwargs]) - The :class:`JsonRequest` class adds two new keyword parameters to the `__init__` method The + The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method The remaining arguments are the same as for the :class:`Request` class and are not documented here. @@ -721,7 +721,7 @@ TextResponse objects :class:`Response` class, which is meant to be used only for binary data, such as images, sounds or any media file. - :class:`TextResponse` objects support a new `__init__` method argument, in + :class:`TextResponse` objects support a new ``__init__`` method argument, in addition to the base :class:`Response` objects. The remaining functionality is the same as for the :class:`Response` class and is not documented here. @@ -755,7 +755,7 @@ TextResponse objects A string with the encoding of this response. The encoding is resolved by trying the following mechanisms, in order: - 1. the encoding passed in the `__init__` method`` ``encoding`` argument + 1. the encoding passed in the ``__init__`` method`` ``encoding`` argument 2. the encoding declared in the Content-Type HTTP header. If this encoding is not valid (ie. unknown), it is ignored and the next diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 2fdc86b63..8ed8d55f1 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -31,7 +31,7 @@ class BaseItemExporter(object): def _configure(self, options, dont_fail=False): """Configure the exporter by poping options from the ``options`` dict. If dont_fail is set, it won't raise an exception on unexpected options - (useful for using with keyword arguments in subclasses __init__ methods) + (useful for using with keyword arguments in subclasses ``__init__`` methods) """ self.encoding = options.pop('encoding', None) self.fields_to_export = options.pop('fields_to_export', None) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 655c10482..bceb648a0 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -106,7 +106,7 @@ class S3FeedStorage(BlockingFeedStorage): warnings.warn( "Initialising `scrapy.extensions.feedexport.S3FeedStorage` " "without AWS keys is deprecated. Please supply credentials or " - "use the `from_crawler()` __init__ method.", + "use the `from_crawler()` ``__init__`` method.", category=ScrapyDeprecationWarning, stacklevel=2 ) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 0dfee1b27..70c2aebc8 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -246,7 +246,7 @@ class CaselessDict(dict): class MergeDict(object): """ A simple class for creating new "virtual" dictionaries that actually look - up values in more than one dictionary, passed in the __init__ method. + up values in more than one dictionary, passed in the ``__init__`` method. If a key appears in more than one of the given dictionaries, only the first occurrence will be used. diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index c892bd438..b3ba2ccec 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -123,14 +123,14 @@ def rel_has_nofollow(rel): def create_instance(objcls, settings, crawler, *args, **kwargs): """Construct a class instance using its ``from_crawler`` or - ``from_settings`` __init__ method, if available. + ``from_settings`` ``__init__`` method, if available. At least one of ``settings`` and ``crawler`` needs to be different from ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used. - If ``crawler`` is ``None``, only the ``from_settings`` __init__ method will be + If ``crawler`` is ``None``, only the ``from_settings`` ``__init__`` method will be tried. - ``*args`` and ``**kwargs`` are forwarded to the __init__ methods. + ``*args`` and ``**kwargs`` are forwarded to the ``__init__`` methods. Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``. """ diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 0d645543a..ea5193f12 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -303,7 +303,7 @@ class WeakKeyCache(object): def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): """Return a (new) dict with unicode keys (and values when "keys_only" is False) of the given dict converted to strings. ``dct_or_tuples`` can be a - dict or a list of tuples, like any dict __init__ method supports. + dict or a list of tuples, like any dict ``__init__`` method supports. """ d = {} for k, v in six.iteritems(dict(dct_or_tuples)): diff --git a/sep/sep-009.rst b/sep/sep-009.rst index 929cedc61..e46479a74 100644 --- a/sep/sep-009.rst +++ b/sep/sep-009.rst @@ -55,7 +55,7 @@ singletons members of that object, as explained below: ``STATS_CLASS`` setting) - **crawler.log**: Logger class with methods replacing the current ``scrapy.log`` functions. Logging would be started (if enabled) on - ``Crawler`` __init__ method, so no log starting functions are required. + ``Crawler`` ``__init__`` method, so no log starting functions are required. - ``crawler.log.msg`` - **crawler.signals**: signal handling diff --git a/tests/test_spider.py b/tests/test_spider.py index 64ff40b61..6f6cdb8ff 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -42,12 +42,12 @@ class SpiderTest(unittest.TestCase): self.assertEqual(list(start_requests), []) def test_spider_args(self): - """__init__ method arguments are assigned to spider attributes""" + """``__init__`` method arguments are assigned to spider attributes""" spider = self.spider_class('example.com', foo='bar') self.assertEqual(spider.foo, 'bar') def test_spider_without_name(self): - """__init__ method arguments are assigned to spider attributes""" + """``__init__`` method arguments are assigned to spider attributes""" self.assertRaises(ValueError, self.spider_class) self.assertRaises(ValueError, self.spider_class, somearg='foo') From 2ee38e8ddbac6841c8d2bb067ddc0ddba813151f Mon Sep 17 00:00:00 2001 From: purvaudai Date: Tue, 22 Oct 2019 14:43:47 +0530 Subject: [PATCH 15/70] Added Pathlib.Path test --- tests/test_feedexport.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f32ac2a4b..842e8d8c0 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -29,6 +29,8 @@ from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get from scrapy.utils.python import to_native_str from scrapy.utils.project import get_project_settings +from Pathlib import Path + class FileFeedStorageTest(unittest.TestCase): @@ -843,3 +845,17 @@ class FeedExportTest(unittest.TestCase): yield self.exported_data({}, settings) self.assertTrue(FromCrawlerCsvItemExporter.init_with_crawler) self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler) + + @defer.inlineCallbacks + def test_pathlib_uri(self): + tmpdir = tempfile.mkdtemp() + feed_uri = Path(tmpdir) / 'res' + settings = { + 'FEED_FORMAT': 'csv', + 'FEED_STORE_EMPTY': True, + 'FEED_URI': feed_uri, + } + + data = yield self.exported_no_data(settings) + self.assertEqual(data, b'') + shutil.rmtree(tmpdir, ignore_errors=True) \ No newline at end of file From ad96d6ef594c29e8648c3c3b310c42e8fcb210d3 Mon Sep 17 00:00:00 2001 From: purvaudai Date: Tue, 22 Oct 2019 14:53:59 +0530 Subject: [PATCH 16/70] Added Pathlib.Path test correctly --- tests/test_feedexport.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 842e8d8c0..664cfd6de 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -856,6 +856,6 @@ class FeedExportTest(unittest.TestCase): 'FEED_URI': feed_uri, } - data = yield self.exported_no_data(settings) - self.assertEqual(data, b'') - shutil.rmtree(tmpdir, ignore_errors=True) \ No newline at end of file + data = yield self.exported_no_data(settings) + self.assertEqual(data, b'') + shutil.rmtree(tmpdir, ignore_errors=True) \ No newline at end of file From 4226791481bb440b9542dda063d1b5ac920a8411 Mon Sep 17 00:00:00 2001 From: purvaudai Date: Tue, 22 Oct 2019 15:07:13 +0530 Subject: [PATCH 17/70] Added Pathlib.Path test --- requirements-py2.txt | 1 + requirements-py3.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements-py2.txt b/requirements-py2.txt index dde8d1c9c..42e057417 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -16,3 +16,4 @@ service_identity>=16.0.0 six>=1.10.0 Twisted>=16.0.0 zope.interface>=4.1.3 +pathlib2>=2.0 diff --git a/requirements-py3.txt b/requirements-py3.txt index 2c98e6f6d..77296b91b 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -16,3 +16,4 @@ lxml>=3.5.0 service_identity>=16.0.0 six>=1.10.0 zope.interface>=4.1.3 +pathlib2>=2.0 From 0b7d8a51b4dabeb4097f5fc2de396904f308c56d Mon Sep 17 00:00:00 2001 From: purvaudai Date: Tue, 22 Oct 2019 15:12:53 +0530 Subject: [PATCH 18/70] Added Pathlib.Path test --- requirements-py2.txt | 2 +- requirements-py3.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index 42e057417..2a0bb49d3 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -16,4 +16,4 @@ service_identity>=16.0.0 six>=1.10.0 Twisted>=16.0.0 zope.interface>=4.1.3 -pathlib2>=2.0 +pathlib==1.0.1 diff --git a/requirements-py3.txt b/requirements-py3.txt index 77296b91b..c57cff5da 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -16,4 +16,4 @@ lxml>=3.5.0 service_identity>=16.0.0 six>=1.10.0 zope.interface>=4.1.3 -pathlib2>=2.0 +pathlib==1.0.1 From a776554282aadc651f9e142aea9bc436f9f587a7 Mon Sep 17 00:00:00 2001 From: purvaudai Date: Tue, 22 Oct 2019 15:31:55 +0530 Subject: [PATCH 19/70] Added Pathlib.Path test --- tests/test_feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 664cfd6de..fcada3a45 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -29,7 +29,7 @@ from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get from scrapy.utils.python import to_native_str from scrapy.utils.project import get_project_settings -from Pathlib import Path +from pathlib import Path class FileFeedStorageTest(unittest.TestCase): From cd4c211f4b5c9d5716d0e49472a978c11d7351f8 Mon Sep 17 00:00:00 2001 From: purvaudai Date: Tue, 22 Oct 2019 15:38:06 +0530 Subject: [PATCH 20/70] Added Pathlib.Path test --- tests/test_feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index fcada3a45..f497bb32e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -29,7 +29,7 @@ from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get from scrapy.utils.python import to_native_str from scrapy.utils.project import get_project_settings -from pathlib import Path +from pathlib2 import Path class FileFeedStorageTest(unittest.TestCase): From cd0964643879b5513807c8955c7e887c77624970 Mon Sep 17 00:00:00 2001 From: purvaudai Date: Tue, 22 Oct 2019 16:19:41 +0530 Subject: [PATCH 21/70] Added Pathlib.Path test --- requirements-py2.txt | 2 +- requirements-py3.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index 2a0bb49d3..42e057417 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -16,4 +16,4 @@ service_identity>=16.0.0 six>=1.10.0 Twisted>=16.0.0 zope.interface>=4.1.3 -pathlib==1.0.1 +pathlib2>=2.0 diff --git a/requirements-py3.txt b/requirements-py3.txt index c57cff5da..77296b91b 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -16,4 +16,4 @@ lxml>=3.5.0 service_identity>=16.0.0 six>=1.10.0 zope.interface>=4.1.3 -pathlib==1.0.1 +pathlib2>=2.0 From 7031e3a12422ae977448e7b338457f6c09af9b0e Mon Sep 17 00:00:00 2001 From: purvaudai Date: Tue, 22 Oct 2019 16:31:14 +0530 Subject: [PATCH 22/70] Added Pathlib.Path test --- tests/test_feedexport.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f497bb32e..c4bfdb4fe 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -850,10 +850,11 @@ class FeedExportTest(unittest.TestCase): def test_pathlib_uri(self): tmpdir = tempfile.mkdtemp() feed_uri = Path(tmpdir) / 'res' + res_uri = urljoin('file:', pathname2url(feed_uri)) settings = { 'FEED_FORMAT': 'csv', 'FEED_STORE_EMPTY': True, - 'FEED_URI': feed_uri, + 'FEED_URI': res_uri, } data = yield self.exported_no_data(settings) From 85f56a92f0c753dfa55012e647f425a6a3d23076 Mon Sep 17 00:00:00 2001 From: purvaudai Date: Tue, 22 Oct 2019 16:43:17 +0530 Subject: [PATCH 23/70] Added Pathlib.Path test --- tests/test_feedexport.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c4bfdb4fe..17526716b 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -850,6 +850,7 @@ class FeedExportTest(unittest.TestCase): def test_pathlib_uri(self): tmpdir = tempfile.mkdtemp() feed_uri = Path(tmpdir) / 'res' + feed_uri=str(feeduri) res_uri = urljoin('file:', pathname2url(feed_uri)) settings = { 'FEED_FORMAT': 'csv', From 4184bac0687d55a442f599ebcfc913231a7c98e2 Mon Sep 17 00:00:00 2001 From: purvaudai Date: Tue, 22 Oct 2019 16:57:14 +0530 Subject: [PATCH 24/70] Added Pathlib.Path test --- tests/test_feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 17526716b..8c0e5cd3d 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -850,7 +850,7 @@ class FeedExportTest(unittest.TestCase): def test_pathlib_uri(self): tmpdir = tempfile.mkdtemp() feed_uri = Path(tmpdir) / 'res' - feed_uri=str(feeduri) + feed_uri=str(feed_uri) res_uri = urljoin('file:', pathname2url(feed_uri)) settings = { 'FEED_FORMAT': 'csv', From d21e1034f046c7e8d778e698614c47ebd6875c87 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Tue, 22 Oct 2019 13:24:57 +0200 Subject: [PATCH 25/70] docs: correct point,comma and plural replacements Issue #4086 --- docs/topics/exporters.rst | 6 +++--- docs/topics/items.rst | 2 +- docs/topics/loaders.rst | 18 +++++++++--------- docs/topics/request-response.rst | 10 +++++----- scrapy/utils/misc.py | 2 +- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 1b8a69ca3..b8d898022 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -247,7 +247,7 @@ XmlItemExporter :type item_element: str The additional keyword arguments of this ``__init__`` method are passed to the - :class:`BaseItemExporter` ``__init__`` method + :class:`BaseItemExporter` ``__init__`` method. A typical output of this exporter would be:: @@ -352,7 +352,7 @@ PprintItemExporter accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) The additional keyword arguments of this ``__init__`` method are passed to the - :class:`BaseItemExporter` ``__init__`` method + :class:`BaseItemExporter` ``__init__`` method. A typical output of this exporter would be:: @@ -399,7 +399,7 @@ JsonLinesItemExporter Exports Items in JSON format to the specified file-like object, writing one JSON-encoded item per line. The additional ``__init__`` method arguments are passed - to the :class:`BaseItemExporter` ``__init__`` method and the leftover arguments to + to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_ ``__init__`` method argument to customize this exporter. diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 370409026..cdf60208e 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -237,7 +237,7 @@ Item objects Return a new Item optionally initialized from the given argument. - Items replicate the standard `dict API`_, including its ``__init__`` method and + Items replicate the standard `dict API`_, including its ``__init__`` method, and also provide the following additional API members: .. automethod:: copy diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 72610f645..a4465f88d 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -265,7 +265,7 @@ There are several ways to modify Item Loader context values: loader.context['unit'] = 'cm' 2. On Item Loader instantiation (the keyword arguments of Item Loader - ``__init__`` methodare stored in the Item Loader context):: + ``__init__`` method are stored in the Item Loader context):: loader = ItemLoader(product, unit='cm') @@ -494,7 +494,7 @@ ItemLoader objects .. attribute:: default_item_class An Item class (or factory), used to instantiate items when not given in - the ``__init__`` method + the ``__init__`` method. .. attribute:: default_input_processor @@ -509,15 +509,15 @@ ItemLoader objects .. attribute:: default_selector_class The class used to construct the :attr:`selector` of this - :class:`ItemLoader`, if only a response is given in the ``__init__`` method + :class:`ItemLoader`, if only a response is given in the ``__init__`` method. If a selector is given in the ``__init__`` method this attribute is ignored. This attribute is sometimes overridden in subclasses. .. attribute:: selector The :class:`~scrapy.selector.Selector` object to extract data from. - It's either the selector given in the ``__init__`` methodor one created from - the response given in the ``__init__`` methodusing the + It's either the selector given in the ``__init__`` method or one created from + the response given in the ``__init__`` method using the :attr:`default_selector_class`. This attribute is meant to be read-only. @@ -656,7 +656,7 @@ Here is a list of all built-in processors: Returns the first non-null/non-empty value from the values received, so it's typically used as an output processor to single-valued fields. - It doesn't receive any ``__init__`` methodarguments, nor does it accept Loader contexts. + It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts. Example:: @@ -667,7 +667,7 @@ Here is a list of all built-in processors: .. class:: Join(separator=u' ') - Returns the values joined with the separator given in the ``__init__`` method which + Returns the values joined with the separator given in the ``__init__`` method, which defaults to ``u' '``. It doesn't accept Loader contexts. When using the default separator, this processor is equivalent to the @@ -705,7 +705,7 @@ Here is a list of all built-in processors: those which do, this processor will pass the currently active :ref:`Loader context ` through that parameter. - The keyword arguments passed in the ``__init__`` methodare used as the default + The keyword arguments passed in the ``__init__`` method are used as the default Loader context values passed to each function call. However, the final Loader context values passed to functions are overridden with the currently active Loader context accessible through the :meth:`ItemLoader.context` @@ -754,7 +754,7 @@ Here is a list of all built-in processors: .. class:: SelectJmes(json_path) - Queries the value using the json path provided to the ``__init__`` methodand returns the output. + Queries the value using the json path provided to the ``__init__`` method and returns the output. Requires jmespath (https://github.com/jmespath/jmespath.py) to run. This processor takes only one input at a time. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index bf6a02a1d..123c2dde1 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -137,7 +137,7 @@ Request objects A string containing the URL of this request. Keep in mind that this attribute contains the escaped URL, so it can differ from the URL passed in - the ``__init__`` method + the ``__init__`` method. This attribute is read-only. To change the URL of a Request use :meth:`replace`. @@ -400,7 +400,7 @@ fields with form data from :class:`Response` objects. .. class:: FormRequest(url, [formdata, ...]) - The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method The + The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The remaining arguments are the same as for the :class:`Request` class and are not documented here. @@ -473,7 +473,7 @@ fields with form data from :class:`Response` objects. :type dont_click: boolean The other parameters of this class method are passed directly to the - :class:`FormRequest` ``__init__`` method + :class:`FormRequest` ``__init__`` method. .. versionadded:: 0.10.3 The ``formname`` parameter. @@ -547,7 +547,7 @@ dealing with JSON requests. .. class:: JsonRequest(url, [... data, dumps_kwargs]) - The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method The + The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method. The remaining arguments are the same as for the :class:`Request` class and are not documented here. @@ -755,7 +755,7 @@ TextResponse objects A string with the encoding of this response. The encoding is resolved by trying the following mechanisms, in order: - 1. the encoding passed in the ``__init__`` method`` ``encoding`` argument + 1. the encoding passed in the ``__init__`` method ``encoding`` argument 2. the encoding declared in the Content-Type HTTP header. If this encoding is not valid (ie. unknown), it is ignored and the next diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b3ba2ccec..8060553ad 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -123,7 +123,7 @@ def rel_has_nofollow(rel): def create_instance(objcls, settings, crawler, *args, **kwargs): """Construct a class instance using its ``from_crawler`` or - ``from_settings`` ``__init__`` method, if available. + ``from_settings`` ``__init__`` methods, if available. At least one of ``settings`` and ``crawler`` needs to be different from ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used. From 7a84a4bdba45bab56cee0daed9d9c3a04d8d61c0 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Tue, 22 Oct 2019 15:31:34 +0200 Subject: [PATCH 26/70] docs: use "constructor" for from_settings() & rom_crawler() factory methods Issue #4086 --- scrapy/utils/misc.py | 6 +++--- tests/test_utils_misc/__init__.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 8060553ad..f638adb25 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -123,14 +123,14 @@ def rel_has_nofollow(rel): def create_instance(objcls, settings, crawler, *args, **kwargs): """Construct a class instance using its ``from_crawler`` or - ``from_settings`` ``__init__`` methods, if available. + ``from_settings`` constructors, if available. At least one of ``settings`` and ``crawler`` needs to be different from ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used. - If ``crawler`` is ``None``, only the ``from_settings`` ``__init__`` method will be + If ``crawler`` is ``None``, only the ``from_settings`` constructor will be tried. - ``*args`` and ``**kwargs`` are forwarded to the ``__init__`` methods. + ``*args`` and ``**kwargs`` are forwarded to the constructors. Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``. """ diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 457a2aa78..e109d5343 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -109,11 +109,11 @@ class UtilsMiscTestCase(unittest.TestCase): else: mock.assert_called_once_with(*args, **kwargs) - # Check usage of correct __init__ method using four mocks: - # 1. with no alternative __init__ methods - # 2. with from_settings() __init__ method - # 3. with from_crawler() __init__ method - # 4. with from_settings() and from_crawler() __init__ method + # Check usage of correct constructor using four mocks: + # 1. with no alternative constructors + # 2. with from_settings() constructor + # 3. with from_crawler() constructor + # 4. with from_settings() and from_crawler() constructor spec_sets = ([], ['from_settings'], ['from_crawler'], ['from_settings', 'from_crawler']) for specs in spec_sets: From f701f5b0db10faef08e4ed9a21b98fd72f9cfc9a Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Tue, 22 Oct 2019 10:48:02 -0300 Subject: [PATCH 27/70] fix #2552 by improving request schema check on its initialization --- scrapy/http/request/__init__.py | 2 +- tests/test_http_request.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index d09eaf849..76a428199 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -66,7 +66,7 @@ class Request(object_ref): s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) - if ':' not in self._url: + if ('://' not in self._url) and (not self._url.startswith('data:')): raise ValueError('Missing scheme in request url: %s' % self._url) url = property(_get_url, obsolete_setter(_set_url, 'url')) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 16d7a1cb8..64f1184c3 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -52,6 +52,8 @@ class RequestTest(unittest.TestCase): def test_url_no_scheme(self): self.assertRaises(ValueError, self.request_class, 'foo') + self.assertRaises(ValueError, self.request_class, '/foo/') + self.assertRaises(ValueError, self.request_class, '/foo:bar') def test_headers(self): # Different ways of setting headers attribute From 7fba8434f3997197f98d2b3e73099ad85429e377 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Tue, 22 Oct 2019 15:55:52 +0200 Subject: [PATCH 28/70] use instantiation for "Crawler" Issue #4086 Co-Authored-By: Mikhail Korobov --- sep/sep-009.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sep/sep-009.rst b/sep/sep-009.rst index e46479a74..da87fa9aa 100644 --- a/sep/sep-009.rst +++ b/sep/sep-009.rst @@ -55,7 +55,7 @@ singletons members of that object, as explained below: ``STATS_CLASS`` setting) - **crawler.log**: Logger class with methods replacing the current ``scrapy.log`` functions. Logging would be started (if enabled) on - ``Crawler`` ``__init__`` method, so no log starting functions are required. + ``Crawler`` instantiation, so no log starting functions are required. - ``crawler.log.msg`` - **crawler.signals**: signal handling From bf5c1a3dec02165b81dffb98fc560c43fb065f38 Mon Sep 17 00:00:00 2001 From: Ammar Najjar Date: Tue, 22 Oct 2019 15:56:46 +0200 Subject: [PATCH 29/70] docs: use "constructor" for "from_crawler" Issue #4086 --- scrapy/extensions/feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index bceb648a0..ce2846eba 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -106,7 +106,7 @@ class S3FeedStorage(BlockingFeedStorage): warnings.warn( "Initialising `scrapy.extensions.feedexport.S3FeedStorage` " "without AWS keys is deprecated. Please supply credentials or " - "use the `from_crawler()` ``__init__`` method.", + "use the `from_crawler()` constructor.", category=ScrapyDeprecationWarning, stacklevel=2 ) From c623a16a223df31669d9f204b8af7d89062ee56c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 22 Oct 2019 17:52:34 +0200 Subject: [PATCH 30/70] Remove unused method from scrapy.pqueues._SlotPriorityQueues --- scrapy/pqueues.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 6ecd1b51a..717ed4d27 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -86,9 +86,6 @@ class _SlotPriorityQueues(object): def __len__(self): return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 - def __contains__(self, slot): - return slot in self.pqueues - class ScrapyPriorityQueue(PriorityQueue): """ From 439a3e59b8e858441f8d97dbc32f398db392330d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 4 Nov 2019 10:35:58 -0300 Subject: [PATCH 31/70] Fix scrapy.utils.datatypes.LocalCache limit issue --- scrapy/utils/datatypes.py | 5 +++-- tests/test_utils_datatypes.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index df2b99c28..f7e3240c1 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -315,8 +315,9 @@ class LocalCache(collections.OrderedDict): self.limit = limit def __setitem__(self, key, value): - while len(self) >= self.limit: - self.popitem(last=False) + if self.limit: + while len(self) >= self.limit: + self.popitem(last=False) super(LocalCache, self).__setitem__(key, value) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 535095b8d..6ffd7c73c 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -7,7 +7,7 @@ if six.PY2: else: from collections.abc import Mapping, MutableMapping -from scrapy.utils.datatypes import CaselessDict, SequenceExclude +from scrapy.utils.datatypes import CaselessDict, SequenceExclude, LocalCache __doctests__ = ['scrapy.utils.datatypes'] @@ -242,6 +242,31 @@ class SequenceExcludeTest(unittest.TestCase): for v in [-3, "test", 1.1]: self.assertNotIn(v, d) + +class LocalCacheTest(unittest.TestCase): + + def test_cache_with_limit(self): + cache = LocalCache(limit=2) + cache['a'] = 1 + cache['b'] = 2 + cache['c'] = 3 + self.assertEqual(len(cache), 2) + self.assertNotIn('a', cache) + self.assertIn('b', cache) + self.assertIn('c', cache) + self.assertEqual(cache['b'], 2) + self.assertEqual(cache['c'], 3) + + def test_cache_without_limit(self): + max = 10**4 + cache = LocalCache() + for x in range(max): + cache[str(x)] = x + self.assertEqual(len(cache), max) + for x in range(max): + self.assertIn(str(x), cache) + self.assertEqual(cache[str(x)], x) + + if __name__ == "__main__": unittest.main() - From fed9fbe62d54175d70660498f2fba6b5b7f68a92 Mon Sep 17 00:00:00 2001 From: elacuesta Date: Mon, 4 Nov 2019 15:34:27 -0300 Subject: [PATCH 32/70] Update tests/test_utils_datatypes.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- tests/test_utils_datatypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 6ffd7c73c..fb2362829 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -7,7 +7,7 @@ if six.PY2: else: from collections.abc import Mapping, MutableMapping -from scrapy.utils.datatypes import CaselessDict, SequenceExclude, LocalCache +from scrapy.utils.datatypes import CaselessDict, LocalCache, SequenceExclude __doctests__ = ['scrapy.utils.datatypes'] From 613c66a034cebc885b90c8ba2a76aea2109ef72e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 5 Nov 2019 09:45:51 -0300 Subject: [PATCH 33/70] Do not override built-in max function --- tests/test_utils_datatypes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index fb2362829..7e671f627 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -258,12 +258,12 @@ class LocalCacheTest(unittest.TestCase): self.assertEqual(cache['c'], 3) def test_cache_without_limit(self): - max = 10**4 + maximum = 10**4 cache = LocalCache() - for x in range(max): + for x in range(maximum): cache[str(x)] = x - self.assertEqual(len(cache), max) - for x in range(max): + self.assertEqual(len(cache), maximum) + for x in range(maximum): self.assertIn(str(x), cache) self.assertEqual(cache[str(x)], x) From aef98188facfc79dc574d8a86200b4e95b96b880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 7 Nov 2019 18:06:55 +0100 Subject: [PATCH 34/70] Improve the details about request serialization requirements for JOBDIR --- docs/topics/jobs.rst | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 9fd311c69..f5542495b 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -71,34 +71,11 @@ on cookies. Request serialization --------------------- -Requests must be serializable by the ``pickle`` module, in order for persistence -to work, so you should make sure that your requests are serializable. - -The most common issue here is to use ``lambda`` functions on request callbacks that -can't be persisted. - -So, for example, this won't work:: - - def some_callback(self, response): - somearg = 'test' - return scrapy.Request('http://www.example.com', - callback=lambda r: self.other_callback(r, somearg)) - - def other_callback(self, response, somearg): - print("the argument passed is: %s" % somearg) - -But this will:: - - def some_callback(self, response): - somearg = 'test' - return scrapy.Request('http://www.example.com', - callback=self.other_callback, cb_kwargs={'somearg': somearg}) - - def other_callback(self, response, somearg): - print("the argument passed is: %s" % somearg) +For persistence to work, :class:`~scrapy.http.Request` objects must be +serializable with :mod:`pickle`, except for the ``callback`` and ``errback`` +values passed to their ``__init__`` method, which must be methods of the +runnning :class:`~scrapy.spiders.Spider` class. If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. It is ``False`` by default. - -.. _pickle: https://docs.python.org/library/pickle.html From 1df5755699eac5a98239ae73dfb82908706bf03b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 8 Nov 2019 16:00:10 +0100 Subject: [PATCH 35/70] Set the bases for testing code examples from the documentation --- docs/conftest.py | 16 ++++++++++++++++ pytest.ini | 20 +++++++++++++++++++- tests/requirements-py3.txt | 1 + tox.ini | 6 +++--- 4 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 docs/conftest.py diff --git a/docs/conftest.py b/docs/conftest.py new file mode 100644 index 000000000..91c1d4428 --- /dev/null +++ b/docs/conftest.py @@ -0,0 +1,16 @@ +from doctest import ELLIPSIS + +from sybil import Sybil +from sybil.parsers.codeblock import CodeBlockParser +from sybil.parsers.doctest import DocTestParser +from sybil.parsers.skip import skip + + +pytest_collect_file = Sybil( + parsers=[ + DocTestParser(optionflags=ELLIPSIS), + CodeBlockParser(future_imports=['print_function']), + skip, + ], + pattern='*.rst', +).pytest() diff --git a/pytest.ini b/pytest.ini index db5bee228..8c5a2cd54 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,7 +2,25 @@ usefixtures = chdir python_files=test_*.py __init__.py python_classes= -addopts = --doctest-modules --assert=plain +addopts = + --assert=plain + --doctest-modules + --ignore=docs/_ext + --ignore=docs/conf.py + --ignore=docs/intro/tutorial.rst + --ignore=docs/news.rst + --ignore=docs/topics/commands.rst + --ignore=docs/topics/debug.rst + --ignore=docs/topics/developer-tools.rst + --ignore=docs/topics/dynamic-content.rst + --ignore=docs/topics/items.rst + --ignore=docs/topics/leaks.rst + --ignore=docs/topics/loaders.rst + --ignore=docs/topics/selectors.rst + --ignore=docs/topics/shell.rst + --ignore=docs/topics/stats.rst + --ignore=docs/topics/telnetconsole.rst + --ignore=docs/utils twisted = 1 flake8-ignore = # extras diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index dd5b23cc3..2e8d319d2 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -4,6 +4,7 @@ pytest pytest-cov pytest-twisted pytest-xdist +sybil testfixtures # optional for shell wrapper tests diff --git a/tox.ini b/tox.ini index cc3463a4d..3668058c3 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,7 @@ passenv = GCS_TEST_FILE_URI GCS_PROJECT_ID commands = - py.test --cov=scrapy --cov-report= {posargs:scrapy tests} + py.test --cov=scrapy --cov-report= {posargs:docs scrapy tests} [testenv:py35] basepython = python3.5 @@ -60,7 +60,7 @@ basepython = python3.8 [testenv:pypy3] basepython = pypy3 commands = - py.test {posargs:scrapy tests} + py.test {posargs:docs scrapy tests} [testenv:flake8] basepython = python3.8 @@ -68,7 +68,7 @@ deps = {[testenv]deps} pytest-flake8 commands = - py.test --flake8 {posargs:scrapy tests} + py.test --flake8 {posargs:docs scrapy tests} [docs] changedir = docs From 084a1cda6dfd94a3671d49c362db2ee6ea88a10d Mon Sep 17 00:00:00 2001 From: purvaudai Date: Mon, 11 Nov 2019 15:41:00 +0530 Subject: [PATCH 36/70] Adding test --- tests/test_feedexport.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8c0e5cd3d..16916f728 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -415,7 +415,7 @@ class FeedExportTest(unittest.TestCase): spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) - with open(res_path, 'rb') as f: + with open(res_uri, 'rb') as f: content = f.read() finally: @@ -850,12 +850,12 @@ class FeedExportTest(unittest.TestCase): def test_pathlib_uri(self): tmpdir = tempfile.mkdtemp() feed_uri = Path(tmpdir) / 'res' - feed_uri=str(feed_uri) res_uri = urljoin('file:', pathname2url(feed_uri)) settings = { 'FEED_FORMAT': 'csv', 'FEED_STORE_EMPTY': True, - 'FEED_URI': res_uri, + 'FEED_URI': feed_uri, + 'FEED_URI_ISPATH' : True } data = yield self.exported_no_data(settings) From 0042c389eb2d44d017bc8af069c7dd7ebd9319bd Mon Sep 17 00:00:00 2001 From: purvaudai Date: Mon, 11 Nov 2019 15:57:58 +0530 Subject: [PATCH 37/70] Adding test --- tests/test_feedexport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 16916f728..275579959 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -855,7 +855,6 @@ class FeedExportTest(unittest.TestCase): 'FEED_FORMAT': 'csv', 'FEED_STORE_EMPTY': True, 'FEED_URI': feed_uri, - 'FEED_URI_ISPATH' : True } data = yield self.exported_no_data(settings) From 9e6e2dde2b7736278075d6a8268511a3bc44b8b5 Mon Sep 17 00:00:00 2001 From: purvaudai Date: Mon, 11 Nov 2019 16:10:37 +0530 Subject: [PATCH 38/70] Adding test --- tests/test_feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 275579959..b6e4a5449 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -415,7 +415,7 @@ class FeedExportTest(unittest.TestCase): spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) - with open(res_uri, 'rb') as f: + with open(defaults['FEED_URI'], 'rb') as f: content = f.read() finally: From 970c3be1603483a61637b94afdb965eb24342744 Mon Sep 17 00:00:00 2001 From: purvaudai Date: Mon, 11 Nov 2019 18:34:15 +0530 Subject: [PATCH 39/70] Added Test --- scrapy/extensions/feedexport.py | 2 +- tests/test_feedexport.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 981efee55..8dacce459 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -201,7 +201,7 @@ class FeedExporter(object): self.settings = settings if not settings['FEED_URI']: raise NotConfigured - self.urifmt=str(settings['FEED_URI']) + self.urifmt = str(settings['FEED_URI']) self.format = settings['FEED_FORMAT'].lower() self.export_encoding = settings['FEED_EXPORT_ENCODING'] self.storages = self._load_components('FEED_STORAGES') diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index b6e4a5449..11d32bd14 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -407,6 +407,7 @@ class FeedExportTest(unittest.TestCase): defaults = { 'FEED_URI': res_uri, 'FEED_FORMAT': 'csv', + 'FEED_PATH': res_path } defaults.update(settings or {}) try: @@ -415,7 +416,7 @@ class FeedExportTest(unittest.TestCase): spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) - with open(defaults['FEED_URI'], 'rb') as f: + with open(defaults['FEED_PATH'], 'rb') as f: content = f.read() finally: @@ -855,8 +856,8 @@ class FeedExportTest(unittest.TestCase): 'FEED_FORMAT': 'csv', 'FEED_STORE_EMPTY': True, 'FEED_URI': feed_uri, + 'FEED_PATH': feed_uri } - data = yield self.exported_no_data(settings) self.assertEqual(data, b'') - shutil.rmtree(tmpdir, ignore_errors=True) \ No newline at end of file + shutil.rmtree(tmpdir, ignore_errors=True) From 0c2dcd5092eccf08399f0838d86d598329cc3a28 Mon Sep 17 00:00:00 2001 From: purvaudai Date: Mon, 11 Nov 2019 18:35:50 +0530 Subject: [PATCH 40/70] Added Test --- requirements-py2.txt | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 requirements-py2.txt diff --git a/requirements-py2.txt b/requirements-py2.txt deleted file mode 100644 index 42e057417..000000000 --- a/requirements-py2.txt +++ /dev/null @@ -1,19 +0,0 @@ -parsel>=1.5.0 -PyDispatcher>=2.0.5 -w3lib>=1.17.0 -protego>=0.1.15 - -pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" -queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" -cryptography>=2.0 # Earlier versions would fail to install - -# Reference versions taken from -# https://packages.ubuntu.com/xenial/python/ -# https://packages.ubuntu.com/xenial/zope/ -cssselect>=0.9.1 -lxml>=3.5.0 -service_identity>=16.0.0 -six>=1.10.0 -Twisted>=16.0.0 -zope.interface>=4.1.3 -pathlib2>=2.0 From f39ff4945854fb5d98389fabfa2d2e5a059c0643 Mon Sep 17 00:00:00 2001 From: purvaudai Date: Mon, 11 Nov 2019 18:54:21 +0530 Subject: [PATCH 41/70] Added Test --- tests/test_feedexport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 11d32bd14..9b07c2051 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -851,7 +851,6 @@ class FeedExportTest(unittest.TestCase): def test_pathlib_uri(self): tmpdir = tempfile.mkdtemp() feed_uri = Path(tmpdir) / 'res' - res_uri = urljoin('file:', pathname2url(feed_uri)) settings = { 'FEED_FORMAT': 'csv', 'FEED_STORE_EMPTY': True, From 50eaabe1fc540218f9f04197810367154eb3e102 Mon Sep 17 00:00:00 2001 From: purvaudai Date: Mon, 11 Nov 2019 20:00:26 +0530 Subject: [PATCH 42/70] Added Test --- tests/test_feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 9b07c2051..2819f8f0b 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -416,7 +416,7 @@ class FeedExportTest(unittest.TestCase): spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) - with open(defaults['FEED_PATH'], 'rb') as f: + with open(str(defaults['FEED_PATH']), 'rb') as f: content = f.read() finally: From 79d2f99995a12ffc19ab7cd2fda10112cf5e9b65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 12 Nov 2019 08:08:50 +0100 Subject: [PATCH 43/70] Make tutorial doctests pass --- docs/_tests/quotes1.html | 281 +++++++++++++++++++++++++++++++++++++++ docs/conftest.py | 17 ++- docs/intro/tutorial.rst | 23 ++-- pytest.ini | 1 - 4 files changed, 310 insertions(+), 12 deletions(-) create mode 100644 docs/_tests/quotes1.html diff --git a/docs/_tests/quotes1.html b/docs/_tests/quotes1.html new file mode 100644 index 000000000..71aff8847 --- /dev/null +++ b/docs/_tests/quotes1.html @@ -0,0 +1,281 @@ + + + + + Quotes to Scrape + + + + +
+
+
+

+ Quotes to Scrape +

+
+
+

+ + Login + +

+
+
+ + +
+
+ +
+ “The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.” + by + (about) + +
+ Tags: + + + change + + deep-thoughts + + thinking + + world + +
+
+ +
+ “It is our choices, Harry, that show what we truly are, far more than our abilities.” + by + (about) + +
+ Tags: + + + abilities + + choices + +
+
+ +
+ “There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.” + by + (about) + +
+ Tags: + + + inspirational + + life + + live + + miracle + + miracles + +
+
+ +
+ “The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.” + by + (about) + +
+ Tags: + + + aliteracy + + books + + classic + + humor + +
+
+ +
+ “Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.” + by + (about) + +
+ Tags: + + + be-yourself + + inspirational + +
+
+ +
+ “Try not to become a man of success. Rather become a man of value.” + by + (about) + +
+ Tags: + + + adulthood + + success + + value + +
+
+ +
+ “It is better to be hated for what you are than to be loved for what you are not.” + by + (about) + +
+ Tags: + + + life + + love + +
+
+ +
+ “I have not failed. I've just found 10,000 ways that won't work.” + by + (about) + +
+ Tags: + + + edison + + failure + + inspirational + + paraphrased + +
+
+ +
+ “A woman is like a tea bag; you never know how strong it is until it's in hot water.” + by + (about) + +
+ Tags: + + + misattributed-eleanor-roosevelt + +
+
+ +
+ “A day without sunshine is like, you know, night.” + by + (about) + +
+ Tags: + + + humor + + obvious + + simile + +
+
+ + +
+
+ +

Top Ten tags

+ + + love + + + + inspirational + + + + life + + + + humor + + + + books + + + + reading + + + + friendship + + + + friends + + + + truth + + + + simile + + + +
+
+ +
+ + + \ No newline at end of file diff --git a/docs/conftest.py b/docs/conftest.py index 91c1d4428..8c735e838 100644 --- a/docs/conftest.py +++ b/docs/conftest.py @@ -1,16 +1,29 @@ -from doctest import ELLIPSIS +import os +from doctest import ELLIPSIS, NORMALIZE_WHITESPACE +from scrapy.http.response.html import HtmlResponse from sybil import Sybil from sybil.parsers.codeblock import CodeBlockParser from sybil.parsers.doctest import DocTestParser from sybil.parsers.skip import skip +def load_response(url, filename): + input_path = os.path.join(os.path.dirname(__file__), '_tests', filename) + with open(input_path, 'rb') as input_file: + return HtmlResponse(url, body=input_file.read()) + + +def setup(namespace): + namespace['load_response'] = load_response + + pytest_collect_file = Sybil( parsers=[ - DocTestParser(optionflags=ELLIPSIS), + DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE), CodeBlockParser(future_imports=['print_function']), skip, ], pattern='*.rst', + setup=setup, ).pytest() diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 0629b9e19..996e3b475 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -235,13 +235,16 @@ You will see something like:: [s] shelp() Shell help (print this help) [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser - >>> Using the shell, you can try selecting elements using `CSS`_ with the response -object:: +object: - >>> response.css('title') - [] +.. invisible-code-block: python + + response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html') + +>>> response.css('title') +[] The result of running ``response.css('title')`` is a list-like object called :class:`~scrapy.selector.SelectorList`, which represents a list of @@ -372,6 +375,9 @@ we want:: We get a list of selectors for the quote HTML elements with:: >>> response.css("div.quote") + [, + , + ...] Each of the selectors returned by the query above allows us to run further queries over their sub-elements. Let's assign the first selector to a @@ -404,10 +410,9 @@ quotes elements and put them together into a Python dictionary:: ... author = quote.css("small.author::text").get() ... tags = quote.css("div.tags a.tag::text").getall() ... print(dict(text=text, author=author, tags=tags)) - {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'} - {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'} - ... a few more of these, omitted for brevity - >>> + {'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']} + {'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']} + ... Extracting data in our spider ----------------------------- @@ -521,7 +526,7 @@ There is also an ``attrib`` property available (see :ref:`selecting-attributes` for more):: >>> response.css('li.next a').attrib['href'] - '/page/2' + '/page/2/' Let's see now our spider modified to recursively follow the link to the next page, extracting data from it:: diff --git a/pytest.ini b/pytest.ini index 8c5a2cd54..3f1cc5800 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,7 +7,6 @@ addopts = --doctest-modules --ignore=docs/_ext --ignore=docs/conf.py - --ignore=docs/intro/tutorial.rst --ignore=docs/news.rst --ignore=docs/topics/commands.rst --ignore=docs/topics/debug.rst From 7b7bb028f45a06b3444950521ea582d0e83691ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 12 Nov 2019 08:49:06 +0100 Subject: [PATCH 44/70] Use intersphinx for links to the Sphinx documentation --- docs/conf.py | 1 + docs/contributing.rst | 17 ++++++++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 34dd5bcb7..0cc1dc22b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -273,4 +273,5 @@ coverage_ignore_pyobjects = [ intersphinx_mapping = { 'python': ('https://docs.python.org/3', None), + 'sphinx': ('https://www.sphinx-doc.org/en/stable', None), } diff --git a/docs/contributing.rst b/docs/contributing.rst index 28dea74de..f084bd23d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -177,20 +177,19 @@ Documentation policies ====================== For reference documentation of API members (classes, methods, etc.) use -docstrings and make sure that the Sphinx documentation uses the autodoc_ -extension to pull the docstrings. API reference documentation should follow -docstring conventions (`PEP 257`_) and be IDE-friendly: short, to the point, -and it may provide short examples. +docstrings and make sure that the Sphinx documentation uses the +:mod:`~sphinx.ext.autodoc` extension to pull the docstrings. API reference +documentation should follow docstring conventions (`PEP 257`_) and be +IDE-friendly: short, to the point, and it may provide short examples. Other types of documentation, such as tutorials or topics, should be covered in files within the ``docs/`` directory. This includes documentation that is specific to an API member, but goes beyond API reference documentation. -In any case, if something is covered in a docstring, use the autodoc_ -extension to pull the docstring into the documentation instead of duplicating -the docstring in files within the ``docs/`` directory. - -.. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html +In any case, if something is covered in a docstring, use the +:mod:`~sphinx.ext.autodoc` extension to pull the docstring into the +documentation instead of duplicating the docstring in files within the +``docs/`` directory. Tests ===== From 8a6a063778d45e8ba68a59558d2930332c1e9a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 12 Nov 2019 10:23:19 +0100 Subject: [PATCH 45/70] Allow opening the source code from the API documentation --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 34dd5bcb7..b09000de0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,6 +31,7 @@ extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.coverage', 'sphinx.ext.intersphinx', + 'sphinx.ext.viewcode', ] # Add any paths that contain templates here, relative to this directory. From 4b8b0345e58ee5990bd0c28205b5eb0b892680d1 Mon Sep 17 00:00:00 2001 From: purvaudai Date: Tue, 12 Nov 2019 18:17:15 +0530 Subject: [PATCH 46/70] Mades Changes as per review --- requirements-py3.txt | 1 - tests/test_feedexport.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements-py3.txt b/requirements-py3.txt index 77296b91b..2c98e6f6d 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -16,4 +16,3 @@ lxml>=3.5.0 service_identity>=16.0.0 six>=1.10.0 zope.interface>=4.1.3 -pathlib2>=2.0 diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2819f8f0b..1f46ac04a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -29,7 +29,7 @@ from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get from scrapy.utils.python import to_native_str from scrapy.utils.project import get_project_settings -from pathlib2 import Path +from pathlib import Path class FileFeedStorageTest(unittest.TestCase): From 414e6e2fd568e0dfae699873d3c1ccd865261d09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 13 Nov 2019 07:56:45 +0100 Subject: [PATCH 47/70] Skip a doctest in Python 3.5- because of dictionary changes --- docs/intro/tutorial.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 996e3b475..30b1ddeab 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -402,6 +402,12 @@ to get all of them:: >>> tags ['change', 'deep-thoughts', 'thinking', 'world'] +.. invisible-code-block: python + + from sys import version_info + +.. skip: next if(version_info <= (3, 5), reason="Only Python 3.6+ dictionaries match the output") + Having figured out how to extract each bit, we can now iterate over all the quotes elements and put them together into a Python dictionary:: From b642a1fca29852adf0ba3ddd62c2ecfdbaf9610e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 13 Nov 2019 09:14:20 +0100 Subject: [PATCH 48/70] Fix doctest skipping based on the running Python version --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 30b1ddeab..6b15a5fbd 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -406,7 +406,7 @@ to get all of them:: from sys import version_info -.. skip: next if(version_info <= (3, 5), reason="Only Python 3.6+ dictionaries match the output") +.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output") Having figured out how to extract each bit, we can now iterate over all the quotes elements and put them together into a Python dictionary:: From 76c31094dff2920778b42ed746c6a9cd5b08f4e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 13 Nov 2019 09:28:48 +0100 Subject: [PATCH 49/70] Install the sphinx-notfound-page Sphinx extension --- docs/conf.py | 1 + docs/requirements.txt | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 6ab5959d5..935c3c9a1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,6 +27,7 @@ sys.path.insert(0, path.dirname(path.dirname(__file__))) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ + 'notfound.extension', 'scrapydocs', 'sphinx.ext.autodoc', 'sphinx.ext.coverage', diff --git a/docs/requirements.txt b/docs/requirements.txt index 379da9994..f9db85146 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ Sphinx>=2.1 -sphinx_rtd_theme \ No newline at end of file +sphinx-notfound-page +sphinx_rtd_theme From 33ef24c757c797c804c8fd242b8ab5705219b452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 13 Nov 2019 10:52:05 +0100 Subject: [PATCH 50/70] =?UTF-8?q?Add=20missing=20whitespace=20after=20?= =?UTF-8?q?=E2=80=98,=E2=80=99,=20=E2=80=98;=E2=80=99=20or=20=E2=80=98:?= =?UTF-8?q?=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pytest.ini | 16 ++++++++-------- scrapy/core/spidermw.py | 2 +- tests/test_http_request.py | 4 ++-- tests/test_item.py | 2 +- tests/test_linkextractors.py | 4 ++-- tests/test_logformatter.py | 2 +- tests/test_utils_conf.py | 2 +- tests/test_utils_console.py | 2 +- tests/test_utils_misc/__init__.py | 2 +- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pytest.ini b/pytest.ini index 8c5a2cd54..984930459 100644 --- a/pytest.ini +++ b/pytest.ini @@ -48,7 +48,7 @@ flake8-ignore = scrapy/core/engine.py E261 E501 E128 E127 E306 E502 scrapy/core/scheduler.py E501 scrapy/core/scraper.py E501 E306 E261 E128 W504 - scrapy/core/spidermw.py E501 E731 E502 E231 E126 E226 + scrapy/core/spidermw.py E501 E731 E502 E126 E226 scrapy/core/downloader/__init__.py F401 E501 scrapy/core/downloader/contextfactory.py E501 E128 E126 scrapy/core/downloader/middleware.py E501 E502 @@ -214,13 +214,13 @@ flake8-ignore = tests/test_feedexport.py E501 F401 F841 E241 tests/test_http_cookies.py E501 tests/test_http_headers.py E302 E501 - tests/test_http_request.py F401 E402 E501 E231 E261 E127 E128 W293 E502 E128 E502 E126 E123 + tests/test_http_request.py F401 E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123 tests/test_http_response.py E501 E301 E502 E128 E265 - tests/test_item.py E701 E128 E231 F841 E306 + tests/test_item.py E701 E128 F841 E306 tests/test_link.py E501 - tests/test_linkextractors.py E501 E128 E231 E124 + tests/test_linkextractors.py E501 E128 E124 tests/test_loader.py E302 E501 E731 E303 E741 E128 E117 E241 - tests/test_logformatter.py E128 E501 E231 E122 E302 + tests/test_logformatter.py E128 E501 E122 E302 tests/test_mail.py E302 E128 E501 E305 tests/test_middleware.py E302 E501 E128 tests/test_pipeline_crawl.py E131 E501 E128 E126 @@ -239,8 +239,8 @@ flake8-ignore = tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226 tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 E241 E121 tests/test_squeues.py E501 E302 E701 E741 - tests/test_utils_conf.py E501 E231 E303 E128 - tests/test_utils_console.py E302 E231 + tests/test_utils_conf.py E501 E303 E128 + tests/test_utils_console.py E302 tests/test_utils_curl.py E501 tests/test_utils_datatypes.py E402 E501 E305 tests/test_utils_defer.py E306 E261 E501 E302 F841 E226 @@ -269,4 +269,4 @@ flake8-ignore = tests/test_spiderloader/test_spiders/spider2.py E302 tests/test_spiderloader/test_spiders/spider3.py E302 tests/test_spiderloader/test_spiders/nested/spider4.py E302 - tests/test_utils_misc/__init__.py E501 E231 + tests/test_utils_misc/__init__.py E501 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index b5f9837ff..00cee3ada 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -36,7 +36,7 @@ class SpiderMiddlewareManager(MiddlewareManager): self.methods['process_spider_exception'].appendleft(getattr(mw, 'process_spider_exception', None)) def scrape_response(self, scrape_func, response, request, spider): - fname = lambda f:'%s.%s' % ( + fname = lambda f: '%s.%s' % ( six.get_method_self(f).__class__.__name__, six.get_method_function(f).__name__) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 96a4fb141..45a547f40 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -56,7 +56,7 @@ class RequestTest(unittest.TestCase): def test_headers(self): # Different ways of setting headers attribute url = 'http://www.scrapy.org' - headers = {b'Accept':'gzip', b'Custom-Header':'nothing to tell you'} + headers = {b'Accept': 'gzip', b'Custom-Header': 'nothing to tell you'} r = self.request_class(url=url, headers=headers) p = self.request_class(url=url, headers=r.headers) @@ -816,7 +816,7 @@ class FormRequestTest(RequestTest): """) - r1 = self.request_class.from_response(response, formdata={'two':'3'}) + r1 = self.request_class.from_response(response, formdata={'two': '3'}) self.assertEqual(r1.method, 'POST') self.assertEqual(r1.headers['Content-type'], b'application/x-www-form-urlencoded') fs = _qs(r1) diff --git a/tests/test_item.py b/tests/test_item.py index 947566686..7c9468f65 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -245,7 +245,7 @@ class ItemTest(unittest.TestCase): def test_copy(self): class TestItem(Item): name = Field() - item = TestItem({'name':'lower'}) + item = TestItem({'name': 'lower'}) copied_item = item.copy() self.assertNotEqual(id(item), id(copied_item)) copied_item['name'] = copied_item['name'].upper() diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index d96e259f6..57ef1694a 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -322,7 +322,7 @@ class Base: Link(url=page4_url, text=u'href with whitespaces'), ]) - lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=()) + lx = self.extractor_cls(attrs=("href", "src"), tags=("a", "area", "img"), deny_extensions=()) self.assertEqual(lx.extract_links(self.response), [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), @@ -360,7 +360,7 @@ class Base: Link(url='http://example.com/sample2.html', text=u'sample 2'), ]) - lx = self.extractor_cls(tags=("a","img"), attrs=("href", "src"), deny_extensions=()) + lx = self.extractor_cls(tags=("a", "img"), attrs=("href", "src"), deny_extensions=()) self.assertEqual(lx.extract_links(response), [ Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample2.jpg', text=u''), diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index eb9c4a561..b4ea30bb7 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -45,7 +45,7 @@ class LoggingContribTest(unittest.TestCase): "Crawled (200) (referer: http://example.com) ['cached']") def test_flags_in_request(self): - req = Request("http://www.example.com", flags=['test','flag']) + req = Request("http://www.example.com", flags=['test', 'flag']) res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 29937c189..02d8ba51e 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -79,7 +79,7 @@ class BuildComponentListTest(unittest.TestCase): self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) d = {'one': {'a': 'a', 'b': 2}} self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - d = {'one': 'lorem ipsum',} + d = {'one': 'lorem ipsum'} self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index 65782747b..c2211848c 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -21,7 +21,7 @@ class UtilsConsoleTestCase(unittest.TestCase): shell = get_shell_embed_func(['invalid']) self.assertEqual(shell, None) - shell = get_shell_embed_func(['invalid','python']) + shell = get_shell_embed_func(['invalid', 'python']) self.assertTrue(callable(shell)) self.assertEqual(shell.__name__, '_embed_standard_shell') diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index e109d5343..de6f173e0 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -74,7 +74,7 @@ class UtilsMiscTestCase(unittest.TestCase): self.assertEqual(list(arg_to_iter(100)), [100]) self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c']) self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3]) - self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}]) + self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}]) self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")]) def test_create_instance(self): From 1d7c8cb0b1d3aa225fd396f263938fd2f171fb73 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 22 Jul 2019 22:27:29 +0500 Subject: [PATCH 51/70] Remove six.PY2 and six.PY3 conditionals. --- docs/topics/downloader-middleware.rst | 6 +++--- scrapy/_monkeypatches.py | 10 ---------- scrapy/commands/fetch.py | 5 ++--- scrapy/crawler.py | 11 ----------- scrapy/exporters.py | 2 +- scrapy/extensions/feedexport.py | 3 +-- scrapy/http/request/form.py | 3 +-- scrapy/http/response/text.py | 3 --- scrapy/item.py | 8 +------- scrapy/link.py | 15 ++------------- scrapy/mail.py | 9 ++------- scrapy/settings/__init__.py | 8 +------- scrapy/settings/default_settings.py | 4 +--- scrapy/utils/boto.py | 10 +--------- scrapy/utils/conf.py | 5 +---- scrapy/utils/datatypes.py | 20 +++++++------------- scrapy/utils/gz.py | 13 +++---------- scrapy/utils/iterators.py | 6 +----- scrapy/utils/python.py | 10 ++-------- tests/__init__.py | 15 --------------- tests/test_http_request.py | 11 +++-------- tests/test_http_response.py | 3 +-- tests/test_item.py | 8 ++------ tests/test_link.py | 13 ++----------- tests/test_middleware.py | 21 ++++++--------------- tests/test_request_cb_kwargs.py | 8 +------- tests/test_settings/__init__.py | 8 +------- tests/test_utils_datatypes.py | 7 +------ tests/test_utils_python.py | 7 +++---- 29 files changed, 50 insertions(+), 202 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 8048e1c86..366b95510 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -474,7 +474,7 @@ DBM storage backend A DBM_ storage backend is also available for the HTTP cache middleware. - By default, it uses the anydbm_ module, but you can change it with the + By default, it uses the dbm_ module, but you can change it with the :setting:`HTTPCACHE_DBM_MODULE` setting. .. _httpcache-storage-custom: @@ -626,7 +626,7 @@ HTTPCACHE_DBM_MODULE .. versionadded:: 0.13 -Default: ``'anydbm'`` +Default: ``'dbm'`` The database module to use in the :ref:`DBM storage backend `. This setting is specific to the DBM backend. @@ -1202,4 +1202,4 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. .. _DBM: https://en.wikipedia.org/wiki/Dbm -.. _anydbm: https://docs.python.org/2/library/anydbm.html +.. _dbm: https://docs.python.org/3/library/dbm.html diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index b68099cad..1f8067b35 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -1,16 +1,6 @@ -import six from six.moves import copyreg -if six.PY2: - from urlparse import urlparse - - # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4 - if urlparse('s3://bucket/key?key=value').query != 'key=value': - from urlparse import uses_query - uses_query.append('s3') - - # Undo what Twisted's perspective broker adds to pickle register # to prevent bugs like Twisted#7989 while serializing requests import twisted.persisted.styles # NOQA diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 7d4840529..d45133e0e 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,5 +1,5 @@ from __future__ import print_function -import sys, six +import sys from w3lib.url import is_url from scrapy.commands import ScrapyCommand @@ -45,8 +45,7 @@ class Command(ScrapyCommand): self._print_bytes(response.body) def _print_bytes(self, bytes_): - bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer - bytes_writer.write(bytes_ + b'\n') + sys.stdout.buffer.write(bytes_ + b'\n') def run(self, args, opts): if len(args) != 1 or not is_url(args[0]): diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ded3c082b..19b998e0d 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -88,20 +88,9 @@ class Crawler(object): yield self.engine.open_spider(self.spider, start_requests) yield defer.maybeDeferred(self.engine.start) except Exception: - # In Python 2 reraising an exception after yield discards - # the original traceback (see https://bugs.python.org/issue7563), - # so sys.exc_info() workaround is used. - # This workaround also works in Python 3, but it is not needed, - # and it is slower, so in Python 3 we use native `raise`. - if six.PY2: - exc_info = sys.exc_info() - self.crawling = False if self.engine is not None: yield self.engine.close() - - if six.PY2: - six.reraise(*exc_info) raise def _create_spider(self, *args, **kwargs): diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 8ed8d55f1..0d9c35654 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -216,7 +216,7 @@ class CsvItemExporter(BaseItemExporter): write_through=True, encoding=self.encoding, newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 - ) if six.PY3 else file + ) self.csv_writer = csv.writer(self.stream, **kwargs) self._headers_not_written = True self._join_multivalued = join_multivalued diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 41d68fb14..e2492d506 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -10,7 +10,6 @@ import logging import posixpath from tempfile import NamedTemporaryFile from datetime import datetime -import six from six.moves.urllib.parse import urlparse, unquote from ftplib import FTP @@ -65,7 +64,7 @@ class StdoutFeedStorage(object): def __init__(self, uri, _stdout=None): if not _stdout: - _stdout = sys.stdout if six.PY2 else sys.stdout.buffer + _stdout = sys.stdout.buffer self._stdout = _stdout def open(self, spider): diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 3ce8fc48e..b6feede07 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -104,8 +104,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): el = el.getparent() if el is None: break - encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape') - raise ValueError('No
element found with %s' % encoded) + raise ValueError('No element found with %s' % formxpath) # If we get here, it means that either formname was None # or invalid diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 339913d4e..a8010877c 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -32,9 +32,6 @@ class TextResponse(Response): def _set_url(self, url): if isinstance(url, six.text_type): - if six.PY2 and self.encoding is None: - raise TypeError("Cannot convert unicode url - %s " - "has no encoding" % type(self).__name__) self._url = to_native_str(url, self.encoding) else: super(TextResponse, self)._set_url(url) diff --git a/scrapy/item.py b/scrapy/item.py index 73b8f54b0..32f9b2ebb 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -4,8 +4,8 @@ Scrapy Item See documentation in docs/topics/item.rst """ -import collections from abc import ABCMeta +from collections.abc import MutableMapping from copy import deepcopy from pprint import pformat from warnings import warn @@ -16,12 +16,6 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref -if six.PY2: - MutableMapping = collections.MutableMapping -else: - MutableMapping = collections.abc.MutableMapping - - class BaseItem(object_ref): """Base class for all scraped items. diff --git a/scrapy/link.py b/scrapy/link.py index f0638ced2..be1888ef0 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,12 +4,6 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ -import warnings -import six - -from scrapy.utils.python import to_bytes - - class Link(object): """Link objects represent an extracted link by the LinkExtractor.""" @@ -17,13 +11,8 @@ class Link(object): def __init__(self, url, text='', fragment='', nofollow=False): if not isinstance(url, str): - if six.PY2: - warnings.warn("Link urls must be str objects. " - "Assuming utf-8 encoding (which could be wrong)") - url = to_bytes(url, encoding='utf8') - else: - got = url.__class__.__name__ - raise TypeError("Link urls must be str objects, got %s" % got) + got = url.__class__.__name__ + raise TypeError("Link urls must be str objects, got %s" % got) self.url = url self.text = text self.fragment = fragment diff --git a/scrapy/mail.py b/scrapy/mail.py index 5b944e1c4..746468e25 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -9,18 +9,13 @@ try: from cStringIO import StringIO as BytesIO except ImportError: from io import BytesIO -import six from email.utils import COMMASPACE, formatdate from six.moves.email_mime_multipart import MIMEMultipart from six.moves.email_mime_text import MIMEText from six.moves.email_mime_base import MIMEBase -if six.PY2: - from email.MIMENonMultipart import MIMENonMultipart - from email import Encoders -else: - from email.mime.nonmultipart import MIMENonMultipart - from email import encoders as Encoders +from email.mime.nonmultipart import MIMENonMultipart +from email import encoders as Encoders from twisted.internet import defer, reactor, ssl diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index f28c7940d..c871e86e0 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,19 +1,13 @@ import six import json import copy -import collections +from collections.abc import MutableMapping from importlib import import_module from pprint import pformat from scrapy.settings import default_settings -if six.PY2: - MutableMapping = collections.MutableMapping -else: - MutableMapping = collections.abc.MutableMapping - - SETTINGS_PRIORITIES = { 'default': 0, 'command': 10, diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9c22999cb..5c9678c01 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -17,8 +17,6 @@ import sys from importlib import import_module from os.path import join, abspath, dirname -import six - AJAXCRAWL_ENABLED = False AUTOTHROTTLE_ENABLED = False @@ -179,7 +177,7 @@ HTTPCACHE_ALWAYS_STORE = False HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = [] -HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm' +HTTPCACHE_DBM_MODULE = 'dbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 421ab2f7e..c8fc911bb 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,7 +1,6 @@ """Boto/botocore helpers""" from __future__ import absolute_import -import six from scrapy.exceptions import NotConfigured @@ -11,11 +10,4 @@ def is_botocore(): import botocore return True except ImportError: - if six.PY2: - try: - import boto - return False - except ImportError: - raise NotConfigured('missing botocore or boto library') - else: - raise NotConfigured('missing botocore library') + raise NotConfigured('missing botocore library') diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index fb7ca3310..561bb72fc 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,13 +1,10 @@ +from configparser import ConfigParser import os import sys import numbers from operator import itemgetter import six -if six.PY2: - from ConfigParser import SafeConfigParser as ConfigParser -else: - from configparser import ConfigParser from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 87536e9d7..39d389fa6 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -7,6 +7,7 @@ This module must not depend on any module outside the Standard Library. import copy import collections +from collections.abc import Mapping import warnings import six @@ -14,12 +15,6 @@ import six from scrapy.exceptions import ScrapyDeprecationWarning -if six.PY2: - Mapping = collections.Mapping -else: - Mapping = collections.abc.Mapping - - class MultiValueDictKeyError(KeyError): def __init__(self, *args, **kwargs): warnings.warn( @@ -252,13 +247,12 @@ class MergeDict(object): first occurrence will be used. """ def __init__(self, *dicts): - if not six.PY2: - warnings.warn( - "scrapy.utils.datatypes.MergeDict is deprecated in favor " - "of collections.ChainMap (introduced in Python 3.3)", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) + warnings.warn( + "scrapy.utils.datatypes.MergeDict is deprecated in favor " + "of collections.ChainMap (introduced in Python 3.3)", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) self.dicts = dicts def __getitem__(self, key): diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index b3fb16b1e..9984492f0 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -6,7 +6,6 @@ except ImportError: from io import BytesIO from gzip import GzipFile -import six import re from scrapy.utils.decorators import deprecated @@ -17,14 +16,8 @@ from scrapy.utils.decorators import deprecated # (regression or bug-fix compared to Python 3.4) # - read1(), which fetches data before raising EOFError on next call # works here but is only available from Python>=3.3 -# - scrapy does not support Python 3.2 -# - Python 2.7 GzipFile works fine with standard read() + extrabuf -if six.PY2: - def read1(gzf, size=-1): - return gzf.read(size) -else: - def read1(gzf, size=-1): - return gzf.read1(size) +def read1(gzf, size=-1): + return gzf.read1(size) def gunzip(data): @@ -37,7 +30,7 @@ def gunzip(data): chunk = b'.' while chunk: try: - chunk = read1(f, 8196) + chunk = f.read1(8196) output_list.append(chunk) except (IOError, EOFError, struct.error): # complete only if there is some data, otherwise re-raise diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index a12e14005..dbc1e0d20 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -102,11 +102,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def row_to_unicode(row_): return [to_unicode(field, encoding) for field in row_] - # Python 3 csv reader input object needs to return strings - if six.PY3: - lines = StringIO(_body_or_str(obj, unicode=True)) - else: - lines = BytesIO(_body_or_str(obj, unicode=False)) + lines = StringIO(_body_or_str(obj, unicode=True)) kwargs = {} if delimiter: kwargs["delimiter"] = delimiter diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index ea5193f12..37e6be868 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -113,10 +113,7 @@ def to_bytes(text, encoding=None, errors='strict'): def to_native_str(text, encoding=None, errors='strict'): """ Return str representation of ``text`` (bytes in Python 2.x and unicode in Python 3.x). """ - if six.PY2: - return to_bytes(text, encoding, errors) - else: - return to_unicode(text, encoding, errors) + return to_unicode(text, encoding, errors) def re_rsearch(pattern, text, chunk_size=1024): @@ -189,7 +186,7 @@ def _getargspec_py23(func): """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords, defaults) - Identical to inspect.getargspec() in python2, but uses + Was identical to inspect.getargspec() in python2, but uses inspect.getfullargspec() for python3 behind the scenes to avoid DeprecationWarning. @@ -199,9 +196,6 @@ def _getargspec_py23(func): >>> _getargspec_py23(f) ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,)) """ - if six.PY2: - return inspect.getargspec(func) - return inspect.ArgSpec(*inspect.getfullargspec(func)[:4]) diff --git a/tests/__init__.py b/tests/__init__.py index 9c9e35c35..a54367f8c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -35,18 +35,3 @@ def get_testdata(*paths): path = os.path.join(tests_datadir, *paths) with open(path, 'rb') as f: return f.read() - - -# FIXME: delete after dropping py2 support -# Monkey patch the unittest module to prevent the -# DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex -import six -if six.PY2: - import unittest - import twisted.trial.unittest - if not getattr(unittest.TestCase, 'assertRegex', None): - unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches - if not getattr(unittest.TestCase, 'assertRaisesRegex', None): - unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp - if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None): - twisted.trial.unittest.TestCase.assertRaisesRegex = twisted.trial.unittest.TestCase.assertRaisesRegexp diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 96a4fb141..bb451b5f4 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -3,13 +3,12 @@ import cgi import unittest import re import json +from urllib.parse import unquote_to_bytes import warnings import six from six.moves import xmlrpc_client as xmlrpclib from six.moves.urllib.parse import urlparse, parse_qs, unquote -if six.PY3: - from urllib.parse import unquote_to_bytes from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_native_str @@ -1064,8 +1063,7 @@ class FormRequestTest(RequestTest): self.assertEqual(fs, {}) xpath = u"//form[@name='\u03b1']" - encoded = xpath if six.PY3 else xpath.encode('unicode_escape') - self.assertRaisesRegex(ValueError, re.escape(encoded), + self.assertRaisesRegex(ValueError, re.escape(xpath), self.request_class.from_response, response, formxpath=xpath) @@ -1208,10 +1206,7 @@ def _qs(req, encoding='utf-8', to_unicode=False): qs = req.body else: qs = req.url.partition('?')[2] - if six.PY2: - uqs = unquote(to_native_str(qs, encoding)) - elif six.PY3: - uqs = unquote_to_bytes(qs) + uqs = unquote_to_bytes(qs) if to_unicode: uqs = uqs.decode(encoding) return parse_qs(uqs, True) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index dfc8562f3..ec3b51086 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -21,8 +21,7 @@ class BaseResponseTest(unittest.TestCase): # Response requires url in the consturctor self.assertRaises(Exception, self.response_class) self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class)) - if not six.PY2: - self.assertRaises(TypeError, self.response_class, b"http://example.com") + self.assertRaises(TypeError, self.response_class, b"http://example.com") # body can be str or None self.assertTrue(isinstance(self.response_class('http://example.com/', body=b''), self.response_class)) self.assertTrue(isinstance(self.response_class('http://example.com/', body=b'body'), self.response_class)) diff --git a/tests/test_item.py b/tests/test_item.py index 947566686..0ad278701 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -62,12 +62,8 @@ class ItemTest(unittest.TestCase): i['number'] = 123 itemrepr = repr(i) - if six.PY2: - self.assertEqual(itemrepr, - "{'name': u'John Doe', 'number': 123}") - else: - self.assertEqual(itemrepr, - "{'name': 'John Doe', 'number': 123}") + self.assertEqual(itemrepr, + "{'name': 'John Doe', 'number': 123}") i2 = eval(itemrepr) self.assertEqual(i2['name'], 'John Doe') diff --git a/tests/test_link.py b/tests/test_link.py index 955430b37..5e2ce5eeb 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -1,6 +1,4 @@ import unittest -import warnings -import six from scrapy.link import Link @@ -46,12 +44,5 @@ class LinkTest(unittest.TestCase): self._assert_same_links(l1, l2) def test_non_str_url_py2(self): - if six.PY2: - with warnings.catch_warnings(record=True) as w: - link = Link(u"http://www.example.com/\xa3") - self.assertIsInstance(link.url, str) - self.assertEqual(link.url, b'http://www.example.com/\xc2\xa3') - assert len(w) == 1, "warning not issued" - else: - with self.assertRaises(TypeError): - Link(b"http://www.example.com/\xc2\xa3") + with self.assertRaises(TypeError): + Link(b"http://www.example.com/\xc2\xa3") diff --git a/tests/test_middleware.py b/tests/test_middleware.py index aea0be825..af9b43d61 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -3,7 +3,6 @@ from twisted.trial import unittest from scrapy.settings import Settings from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager -import six class M1(object): @@ -66,20 +65,12 @@ class MiddlewareManagerTest(unittest.TestCase): def test_methods(self): mwman = TestMiddlewareManager(M1(), M2(), M3()) - if six.PY2: - self.assertEqual([x.im_class for x in mwman.methods['open_spider']], - [M1, M2]) - self.assertEqual([x.im_class for x in mwman.methods['close_spider']], - [M2, M1]) - self.assertEqual([x.im_class for x in mwman.methods['process']], - [M1, M3]) - else: - self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']], - [M1, M2]) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']], - [M2, M1]) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']], - [M1, M3]) + self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']], + [M1, M2]) + self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']], + [M2, M1]) + self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']], + [M1, M3]) def test_enabled(self): m1, m2, m3 = M1(), M2(), M3() diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index c9943faa8..a5cdc0de0 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -1,7 +1,6 @@ from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase -import six from scrapy.http import Request from scrapy.crawler import CrawlerRunner @@ -161,9 +160,4 @@ class CallbackKeywordArgumentsTestCase(TestCase): self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError) self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'") self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError) - # py2 and py3 messages are different - exc_message = str(exceptions['takes_more'].exc_info[1]) - if six.PY2: - self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)") - elif six.PY3: - self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'") + self.assertEqual(str(exceptions['takes_more'].exc_info[1]), "parse_takes_more() missing 1 required positional argument: 'other'") diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 1dbacbea3..08286ff02 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -60,9 +60,6 @@ class SettingsAttributeTest(unittest.TestCase): class BaseSettingsTest(unittest.TestCase): - if six.PY3: - assertItemsEqual = unittest.TestCase.assertCountEqual - def setUp(self): self.settings = BaseSettings() @@ -152,7 +149,7 @@ class BaseSettingsTest(unittest.TestCase): self.settings.setmodule( 'tests.test_settings.default_settings', 10) - self.assertItemsEqual(six.iterkeys(self.settings.attributes), + self.assertCountEqual(six.iterkeys(self.settings.attributes), six.iterkeys(ctrl_attributes)) for key in six.iterkeys(ctrl_attributes): @@ -343,9 +340,6 @@ class BaseSettingsTest(unittest.TestCase): class SettingsTest(unittest.TestCase): - if six.PY3: - assertItemsEqual = unittest.TestCase.assertCountEqual - def setUp(self): self.settings = Settings() diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 7e671f627..53228fc6e 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,11 +1,6 @@ import copy import unittest - -import six -if six.PY2: - from collections import Mapping, MutableMapping -else: - from collections.abc import Mapping, MutableMapping +from collections.abc import Mapping, MutableMapping from scrapy.utils.datatypes import CaselessDict, LocalCache, SequenceExclude diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 3e1148354..096aa50b7 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -231,12 +231,11 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), []) else: - stripself = not six.PY2 # PyPy3 exposes them as methods self.assertEqual( - get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit']) - self.assertEqual(get_func_args(" ".join, stripself), ['list']) + get_func_args(six.text_type.split, True), ['sep', 'maxsplit']) + self.assertEqual(get_func_args(" ".join, True), ['list']) self.assertEqual( - get_func_args(operator.itemgetter(2), stripself), ['obj']) + get_func_args(operator.itemgetter(2), True), ['obj']) def test_without_none_values(self): From 0e696ed06d2975ee86e7ce3a2d3892588c420a04 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 20 Aug 2019 20:56:26 +0500 Subject: [PATCH 52/70] Remove unneeded and unused code from XmlItemExporter. --- scrapy/exporters.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 0d9c35654..1aa195b0b 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -143,11 +143,11 @@ class XmlItemExporter(BaseItemExporter): def _beautify_newline(self, new_item=False): if self.indent is not None and (self.indent > 0 or new_item): - self._xg_characters('\n') + self.xg.characters('\n') def _beautify_indent(self, depth=1): if self.indent: - self._xg_characters(' ' * self.indent * depth) + self.xg.characters(' ' * self.indent * depth) def start_exporting(self): self.xg.startDocument() @@ -182,26 +182,12 @@ class XmlItemExporter(BaseItemExporter): self._export_xml_field('value', value, depth=depth+1) self._beautify_indent(depth=depth) elif isinstance(serialized_value, six.text_type): - self._xg_characters(serialized_value) + self.xg.characters(serialized_value) else: - self._xg_characters(str(serialized_value)) + self.xg.characters(str(serialized_value)) self.xg.endElement(name) self._beautify_newline() - # Workaround for https://bugs.python.org/issue17606 - # Before Python 2.7.4 xml.sax.saxutils required bytes; - # since 2.7.4 it requires unicode. The bug is likely to be - # fixed in 2.7.6, but 2.7.6 will still support unicode, - # and Python 3.x will require unicode, so ">= 2.7.4" should be fine. - if sys.version_info[:3] >= (2, 7, 4): - def _xg_characters(self, serialized_value): - if not isinstance(serialized_value, six.text_type): - serialized_value = serialized_value.decode(self.encoding) - return self.xg.characters(serialized_value) - else: # pragma: no cover - def _xg_characters(self, serialized_value): - return self.xg.characters(serialized_value) - class CsvItemExporter(BaseItemExporter): From 065fe29d3cc6634893dfead320779498fb061cd4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 20 Aug 2019 21:06:52 +0500 Subject: [PATCH 53/70] Deprecate scrapy.utils.gz.read1. --- scrapy/utils/gz.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 9984492f0..dc8316d8c 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -16,6 +16,7 @@ from scrapy.utils.decorators import deprecated # (regression or bug-fix compared to Python 3.4) # - read1(), which fetches data before raising EOFError on next call # works here but is only available from Python>=3.3 +@deprecated('GzipFile.read1') def read1(gzf, size=-1): return gzf.read1(size) From 85e79ae792752353ea60bff3d93f9e77bea300fb Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 20 Aug 2019 21:09:49 +0500 Subject: [PATCH 54/70] Remove cStringIO imports. --- scrapy/downloadermiddlewares/decompression.py | 6 +----- scrapy/mail.py | 6 +----- scrapy/pipelines/files.py | 7 +------ scrapy/pipelines/images.py | 6 +----- scrapy/utils/gz.py | 9 ++------- scrapy/utils/iterators.py | 6 +----- tests/test_cmdline/__init__.py | 5 +---- tests/test_pipeline_media.py | 6 ++---- 8 files changed, 10 insertions(+), 41 deletions(-) diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index 49313cc04..e2d73f347 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -4,6 +4,7 @@ and extract the potentially compressed responses that may arrive. import bz2 import gzip +from io import BytesIO import zipfile import tarfile import logging @@ -11,11 +12,6 @@ from tempfile import mktemp import six -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO - from scrapy.responsetypes import responsetypes logger = logging.getLogger(__name__) diff --git a/scrapy/mail.py b/scrapy/mail.py index 746468e25..d24de2212 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -3,13 +3,9 @@ Mail sending helpers See documentation in docs/topics/email.rst """ +from io import BytesIO import logging -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO - from email.utils import COMMASPACE, formatdate from six.moves.email_mime_multipart import MIMEMultipart from six.moves.email_mime_text import MIMEText diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index cc3d10b63..8d74c5011 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -5,6 +5,7 @@ See documentation in topics/media-pipeline.rst """ import functools import hashlib +from io import BytesIO import mimetypes import os import os.path @@ -15,12 +16,6 @@ from six.moves.urllib.parse import urlparse from collections import defaultdict import six - -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO - from twisted.internet import defer, threads from scrapy.pipelines.media import MediaPipeline diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index fa4d12ad1..e77cef4ff 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -5,13 +5,9 @@ See documentation in topics/media-pipeline.rst """ import functools import hashlib +from io import BytesIO import six -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO - from PIL import Image from scrapy.utils.misc import md5sum diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index dc8316d8c..f41e62fe3 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,12 +1,7 @@ -import struct - -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO from gzip import GzipFile - +from io import BytesIO import re +import struct from scrapy.utils.decorators import deprecated diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index dbc1e0d20..9693ba768 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,11 +1,7 @@ import re import csv -import logging -try: - from cStringIO import StringIO as BytesIO -except ImportError: - from io import BytesIO from io import StringIO +import logging import six from scrapy.http import TextResponse, Response diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 68dfb1cca..56cfe642a 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -1,3 +1,4 @@ +from io import StringIO import json import os import pstats @@ -7,10 +8,6 @@ from subprocess import Popen, PIPE import sys import tempfile import unittest -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO from scrapy.utils.test import get_testenv diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 28e39cefa..ad2618ec9 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -144,10 +144,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase): # The Failure should encapsulate a FileException ... self.assertEqual(failure.value, file_exc) - # ... and if we're running on Python 3 ... - if sys.version_info.major >= 3: - # ... it should have the returnValue exception set as its context - self.assertEqual(failure.value.__context__, def_gen_return_exc) + # ... and it should have the returnValue exception set as its context + self.assertEqual(failure.value.__context__, def_gen_return_exc) # Let's calculate the request fingerprint and fake some runtime data... fp = request_fingerprint(request) From cfa633f5e865fa491257215cc912d7321cc6da78 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 20 Aug 2019 21:13:01 +0500 Subject: [PATCH 55/70] Some text function messages cleanup, deprecate to_native_str. --- scrapy/http/response/__init__.py | 2 +- scrapy/utils/python.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index b0a526b72..a81404afb 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -88,7 +88,7 @@ class Response(object_ref): @property def text(self): """For subclasses of TextResponse, this will return the body - as text (unicode object in Python 2 and str in Python 3) + as str """ raise AttributeError("Response content isn't text") diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 37e6be868..663a8ebaa 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -90,7 +90,7 @@ def to_unicode(text, encoding=None, errors='strict'): if isinstance(text, six.text_type): return text if not isinstance(text, (bytes, six.text_type)): - raise TypeError('to_unicode must receive a bytes, str or unicode ' + raise TypeError('to_unicode must receive a bytes or str ' 'object, got %s' % type(text).__name__) if encoding is None: encoding = 'utf-8' @@ -103,16 +103,16 @@ def to_bytes(text, encoding=None, errors='strict'): if isinstance(text, bytes): return text if not isinstance(text, six.string_types): - raise TypeError('to_bytes must receive a unicode, str or bytes ' + raise TypeError('to_bytes must receive a str or bytes ' 'object, got %s' % type(text).__name__) if encoding is None: encoding = 'utf-8' return text.encode(encoding, errors) +@deprecated('to_unicode') def to_native_str(text, encoding=None, errors='strict'): - """ Return str representation of ``text`` - (bytes in Python 2.x and unicode in Python 3.x). """ + """ Return str representation of ``text``. """ return to_unicode(text, encoding, errors) From 92ffd2f249ec276436277385e525397a2fb8b0a0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 20 Aug 2019 21:27:52 +0500 Subject: [PATCH 56/70] Simplify some more imports. --- scrapy/downloadermiddlewares/httpproxy.py | 5 +---- scrapy/loader/processors.py | 5 +---- tests/__init__.py | 5 ----- tests/test_downloader_handlers.py | 5 +---- tests/test_downloadermiddleware.py | 3 ++- tests/test_downloadermiddleware_robotstxt.py | 4 +++- tests/test_extension_telnet.py | 5 ----- tests/test_feedexport.py | 2 +- tests/test_http_request.py | 3 +-- tests/test_item.py | 2 +- tests/test_pipeline_files.py | 6 +----- tests/test_settings/__init__.py | 3 +-- tests/test_spider.py | 3 +-- tests/test_spidermiddleware.py | 3 ++- tests/test_stats.py | 6 +----- tests/test_utils_deprecate.py | 3 +-- tests/test_utils_misc/__init__.py | 2 +- tests/test_utils_trackref.py | 2 +- 18 files changed, 20 insertions(+), 47 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 2c35d1b90..2212d9688 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,10 +1,7 @@ import base64 from six.moves.urllib.parse import unquote, urlunparse from six.moves.urllib.request import getproxies, proxy_bypass -try: - from urllib2 import _parse_proxy -except ImportError: - from urllib.request import _parse_proxy +from urllib.request import _parse_proxy from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py index 2acdc8093..02c625acc 100644 --- a/scrapy/loader/processors.py +++ b/scrapy/loader/processors.py @@ -3,10 +3,7 @@ This module provides some commonly used processors for Item Loaders. See documentation in docs/topics/loaders.rst """ -try: - from collections import ChainMap -except ImportError: - from scrapy.utils.datatypes import MergeDict as ChainMap +from collections import ChainMap from scrapy.utils.misc import arg_to_iter from scrapy.loader.common import wrap_loader_context diff --git a/tests/__init__.py b/tests/__init__.py index a54367f8c..12ce79fa9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -21,11 +21,6 @@ if 'COV_CORE_CONFIG' in os.environ: os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot, os.environ['COV_CORE_CONFIG']) -try: - import unittest.mock as mock -except ImportError: - import mock - tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data') diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 109469503..6090998d4 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -2,11 +2,8 @@ import os import six import shutil import tempfile +from unittest import mock import contextlib -try: - from unittest import mock -except ImportError: - import mock from testfixtures import LogCapture from twisted.trial import unittest diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 03564e748..6b9a5bee8 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,3 +1,5 @@ +from unittest import mock + from twisted.trial.unittest import TestCase from twisted.python.failure import Failure @@ -7,7 +9,6 @@ from scrapy.exceptions import _InvalidOutput from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.utils.test import get_crawler from scrapy.utils.python import to_bytes -from tests import mock class ManagerTestCase(TestCase): diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index fbc46cba4..8266bf35f 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import + +from unittest import mock + from twisted.internet import reactor, error from twisted.internet.defer import Deferred, DeferredList, maybeDeferred from twisted.python import failure @@ -9,7 +12,6 @@ from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware, from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.settings import Settings -from tests import mock from tests.test_robotstxt_interface import rerp_available, reppy_available diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 4f389e5cb..875ceb83c 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -1,8 +1,3 @@ -try: - import unittest.mock as mock -except ImportError: - import mock - from twisted.trial import unittest from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 11a5a8279..0c70bf80e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -7,6 +7,7 @@ from io import BytesIO import tempfile import shutil import string +from unittest import mock from six.moves.urllib.parse import urljoin, urlparse, quote from six.moves.urllib.request import pathname2url @@ -15,7 +16,6 @@ from twisted.trial import unittest from twisted.internet import defer from scrapy.crawler import CrawlerRunner from scrapy.settings import Settings -from tests import mock from tests.mockserver import MockServer from w3lib.url import path_to_file_uri diff --git a/tests/test_http_request.py b/tests/test_http_request.py index bb451b5f4..9fe201579 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -3,6 +3,7 @@ import cgi import unittest import re import json +from unittest import mock from urllib.parse import unquote_to_bytes import warnings @@ -13,8 +14,6 @@ from six.moves.urllib.parse import urlparse, parse_qs, unquote from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_native_str -from tests import mock - class RequestTest(unittest.TestCase): diff --git a/tests/test_item.py b/tests/test_item.py index 0ad278701..d98c63ddd 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -1,12 +1,12 @@ import sys import unittest +from unittest import mock from warnings import catch_warnings import six from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta -from tests import mock PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index cb8f8da18..bd40e4103 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,10 +1,9 @@ import os import random import time -import hashlib -import warnings from tempfile import mkdtemp from shutil import rmtree +from unittest import mock from six.moves.urllib.parse import urlparse from six import BytesIO @@ -15,13 +14,10 @@ from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GC from scrapy.item import Item, Field from scrapy.http import Request, Response from scrapy.settings import Settings -from scrapy.utils.python import to_bytes from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete from scrapy.utils.boto import is_botocore -from tests import mock - def _mocked_download_func(request, info): response = request.meta.get('response') diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 08286ff02..32e65bed5 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,10 +1,9 @@ import six import unittest -import warnings +from unittest import mock from scrapy.settings import (BaseSettings, Settings, SettingsAttribute, SETTINGS_PRIORITIES, get_settings_priority) -from tests import mock from . import default_settings diff --git a/tests/test_spider.py b/tests/test_spider.py index 6f6cdb8ff..c0fccfdd6 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -1,5 +1,6 @@ import gzip import inspect +from unittest import mock import warnings from io import BytesIO @@ -17,8 +18,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref from scrapy.utils.test import get_crawler -from tests import mock - class SpiderTest(unittest.TestCase): diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 832fd3330..55d665e79 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,3 +1,5 @@ +from unittest import mock + from twisted.trial.unittest import TestCase from twisted.python.failure import Failure @@ -6,7 +8,6 @@ from scrapy.http import Request, Response from scrapy.exceptions import _InvalidOutput from scrapy.utils.test import get_crawler from scrapy.core.spidermw import SpiderMiddlewareManager -from tests import mock class SpiderMiddlewareTestCase(TestCase): diff --git a/tests/test_stats.py b/tests/test_stats.py index 2033dbe07..2bbbb9e2c 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -1,10 +1,6 @@ from datetime import datetime import unittest - -try: - from unittest import mock -except ImportError: - import mock +from unittest import mock from scrapy.extensions.corestats import CoreStats from scrapy.spiders import Spider diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 3e7236fb1..ce04e7f29 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -2,12 +2,11 @@ from __future__ import absolute_import import inspect import unittest +from unittest import mock import warnings from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import create_deprecated_class, update_classpath -from tests import mock - class MyWarning(UserWarning): pass diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index e109d5343..de9da9104 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -1,11 +1,11 @@ import sys import os import unittest +from unittest import mock from scrapy.item import Item, Field from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules -from tests import mock __doctests__ = ['scrapy.utils.misc'] diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index c6072fc0d..480a717e7 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,7 +1,7 @@ import six import unittest +from unittest import mock from scrapy.utils import trackref -from tests import mock class Foo(trackref.object_ref): From a138fb05d4f0d90e2002e85a348a5be34904d3d8 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 20 Aug 2019 21:35:13 +0500 Subject: [PATCH 57/70] Replace to_native_str calls with to_unicode. --- scrapy/core/downloader/handlers/http11.py | 2 +- scrapy/downloadermiddlewares/cookies.py | 6 +++--- scrapy/downloadermiddlewares/robotstxt.py | 3 --- scrapy/exporters.py | 4 ++-- scrapy/http/cookies.py | 10 +++++----- scrapy/http/response/text.py | 8 ++++---- scrapy/linkextractors/lxmlhtml.py | 4 ++-- scrapy/responsetypes.py | 6 +++--- scrapy/robotstxt.py | 8 ++++---- scrapy/spidermiddlewares/referer.py | 5 ++--- scrapy/utils/reqser.py | 4 ++-- scrapy/utils/request.py | 4 ++-- scrapy/utils/response.py | 4 ++-- scrapy/utils/ssl.py | 4 ++-- tests/test_command_parse.py | 5 ++--- tests/test_commands.py | 7 ++----- tests/test_feedexport.py | 7 +++---- tests/test_http_request.py | 7 +++---- tests/test_http_response.py | 6 +++--- tests/test_robotstxt_interface.py | 1 - 20 files changed, 47 insertions(+), 58 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 91b45a8fc..7d917cb74 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -174,7 +174,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None): r""" Return binary content of a CONNECT request. - >>> from scrapy.utils.python import to_native_str as s + >>> from scrapy.utils.python import to_unicode as s >>> s(tunnel_request_data("example.com", 8080)) 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n' >>> s(tunnel_request_data("example.com", 8080, b"123")) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 321c0171b..0d2b9900c 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -6,7 +6,7 @@ from collections import defaultdict from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode logger = logging.getLogger(__name__) @@ -53,7 +53,7 @@ class CookiesMiddleware(object): def _debug_cookie(self, request, spider): if self.debug: - cl = [to_native_str(c, errors='replace') + cl = [to_unicode(c, errors='replace') for c in request.headers.getlist('Cookie')] if cl: cookies = "\n".join("Cookie: {}\n".format(c) for c in cl) @@ -62,7 +62,7 @@ class CookiesMiddleware(object): def _debug_set_cookie(self, response, spider): if self.debug: - cl = [to_native_str(c, errors='replace') + cl = [to_unicode(c, errors='replace') for c in response.headers.getlist('Set-Cookie')] if cl: cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 6a5dfb79c..251706c50 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -5,15 +5,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. """ import logging -import sys -import re from twisted.internet.defer import Deferred, maybeDeferred from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.python import to_native_str from scrapy.utils.misc import load_object logger = logging.getLogger(__name__) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 1aa195b0b..8eb52995e 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -12,7 +12,7 @@ from six.moves import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder -from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike +from scrapy.utils.python import to_bytes, to_unicode, is_listlike from scrapy.item import BaseItem from scrapy.exceptions import ScrapyDeprecationWarning import warnings @@ -232,7 +232,7 @@ class CsvItemExporter(BaseItemExporter): def _build_row(self, values): for s in values: try: - yield to_native_str(s, self.encoding) + yield to_unicode(s, self.encoding) except TypeError: yield s diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 4e8056750..4532c3ab7 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -3,7 +3,7 @@ from six.moves.http_cookiejar import ( CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE ) from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode class CookieJar(object): @@ -165,13 +165,13 @@ class WrappedRequest(object): return name in self.request.headers def get_header(self, name, default=None): - return to_native_str(self.request.headers.get(name, default), + return to_unicode(self.request.headers.get(name, default), errors='replace') def header_items(self): return [ - (to_native_str(k, errors='replace'), - [to_native_str(x, errors='replace') for x in v]) + (to_unicode(k, errors='replace'), + [to_unicode(x, errors='replace') for x in v]) for k, v in self.request.headers.items() ] @@ -189,7 +189,7 @@ class WrappedResponse(object): # python3 cookiejars calls get_all def get_all(self, name, default=None): - return [to_native_str(v, errors='replace') + return [to_unicode(v, errors='replace') for v in self.response.headers.getlist(name)] # python2 cookiejars calls getheaders getheaders = get_all diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index a8010877c..37f450e54 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -16,7 +16,7 @@ from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request from scrapy.http.response import Response from scrapy.utils.response import get_base_url -from scrapy.utils.python import memoizemethod_noargs, to_native_str +from scrapy.utils.python import memoizemethod_noargs, to_unicode class TextResponse(Response): @@ -32,7 +32,7 @@ class TextResponse(Response): def _set_url(self, url): if isinstance(url, six.text_type): - self._url = to_native_str(url, self.encoding) + self._url = to_unicode(url, self.encoding) else: super(TextResponse, self)._set_url(url) @@ -81,11 +81,11 @@ class TextResponse(Response): @memoizemethod_noargs def _headers_encoding(self): content_type = self.headers.get(b'Content-Type', b'') - return http_content_type_encoding(to_native_str(content_type)) + return http_content_type_encoding(to_unicode(content_type)) def _body_inferred_encoding(self): if self._cached_benc is None: - content_type = to_native_str(self.headers.get(b'Content-Type', b'')) + content_type = to_unicode(self.headers.get(b'Content-Type', b'')) benc, ubody = html_to_unicode(content_type, self.body, auto_detect_fun=self._auto_detect_fun, default_encoding=self._DEFAULT_ENCODING) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 8f6f93a44..890c019c8 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -10,7 +10,7 @@ from w3lib.url import canonicalize_url from scrapy.link import Link from scrapy.utils.misc import arg_to_iter, rel_has_nofollow -from scrapy.utils.python import unique as unique_list, to_native_str +from scrapy.utils.python import unique as unique_list, to_unicode from scrapy.utils.response import get_base_url from scrapy.linkextractors import FilteringLinkExtractor @@ -67,7 +67,7 @@ class LxmlParserLinkExtractor(object): url = self.process_attr(attr_val) if url is None: continue - url = to_native_str(url, encoding=response_encoding) + url = to_unicode(url, encoding=response_encoding) # to fix relative links after process_value url = urljoin(response_url, url) link = Link(url, _collect_string_content(el) or u'', diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 4a2d5bf52..de62276c8 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -10,7 +10,7 @@ import six from scrapy.http import Response from scrapy.utils.misc import load_object -from scrapy.utils.python import binary_is_text, to_bytes, to_native_str +from scrapy.utils.python import binary_is_text, to_bytes, to_unicode class ResponseTypes(object): @@ -55,12 +55,12 @@ class ResponseTypes(object): header """ if content_encoding: return Response - mimetype = to_native_str(content_type).split(';')[0].strip().lower() + mimetype = to_unicode(content_type).split(';')[0].strip().lower() return self.from_mimetype(mimetype) def from_content_disposition(self, content_disposition): try: - filename = to_native_str(content_disposition, + filename = to_unicode(content_disposition, encoding='latin-1', errors='replace').split(';')[1].split('=')[1] filename = filename.strip('"\'') return self.from_filename(filename) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 189f165d1..95a8c09b8 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -3,14 +3,14 @@ import logging from abc import ABCMeta, abstractmethod from six import with_metaclass -from scrapy.utils.python import to_native_str, to_unicode +from scrapy.utils.python import to_unicode logger = logging.getLogger(__name__) def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): try: if to_native_str_type: - robotstxt_body = to_native_str(robotstxt_body) + robotstxt_body = to_unicode(robotstxt_body) else: robotstxt_body = robotstxt_body.decode('utf-8') except UnicodeDecodeError: @@ -66,8 +66,8 @@ class PythonRobotParser(RobotParser): return o def allowed(self, url, user_agent): - user_agent = to_native_str(user_agent) - url = to_native_str(url) + user_agent = to_unicode(user_agent) + url = to_unicode(url) return self.rp.can_fetch(user_agent, url) diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 1ddfb37f4..c76e4d5a2 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -10,8 +10,7 @@ from w3lib.url import safe_url_string from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals -from scrapy.utils.python import to_native_str -from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object from scrapy.utils.url import strip_url @@ -322,7 +321,7 @@ class RefererMiddleware(object): if isinstance(resp_or_url, Response): policy_header = resp_or_url.headers.get('Referrer-Policy') if policy_header is not None: - policy_name = to_native_str(policy_header.decode('latin1')) + policy_name = to_unicode(policy_header.decode('latin1')) if policy_name is None: return self.default_policy() diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index c7ea7b425..495564ac0 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -4,7 +4,7 @@ Helper functions for serializing (and deserializing) requests. import six from scrapy.http import Request -from scrapy.utils.python import to_unicode, to_native_str +from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object @@ -54,7 +54,7 @@ def request_from_dict(d, spider=None): eb = _get_method(spider, eb) request_cls = load_object(d['_class']) if '_class' in d else Request return request_cls( - url=to_native_str(d['url']), + url=to_unicode(d['url']), callback=cb, errback=eb, method=d['method'], diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index fb5af66a2..63d0ae772 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -9,7 +9,7 @@ import weakref from six.moves.urllib.parse import urlunparse from w3lib.http import basic_auth_header -from scrapy.utils.python import to_bytes, to_native_str +from scrapy.utils.python import to_bytes, to_unicode from w3lib.url import canonicalize_url from scrapy.utils.httpobj import urlparse_cached @@ -97,4 +97,4 @@ def referer_str(request): referrer = request.headers.get('Referer') if referrer is None: return referrer - return to_native_str(referrer, errors='replace') + return to_unicode(referrer, errors='replace') diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c3236afd4..feab07431 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -8,7 +8,7 @@ import webbrowser import tempfile from twisted.web import http -from scrapy.utils.python import to_bytes, to_native_str +from scrapy.utils.python import to_bytes, to_unicode from w3lib import html @@ -36,7 +36,7 @@ def response_status_message(status): """Return status code plus status text descriptive message """ message = http.RESPONSES.get(int(status), "Unknown Status") - return '%s %s' % (status, to_native_str(message)) + return '%s %s' % (status, to_unicode(message)) def response_httprepr(response): diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 02aed60ee..6e81b33ff 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -3,7 +3,7 @@ import OpenSSL import OpenSSL._util as pyOpenSSLutil -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode # The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL. @@ -12,7 +12,7 @@ SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0) def ffi_buf_to_string(buf): - return to_native_str(pyOpenSSLutil.ffi.string(buf)) + return to_unicode(pyOpenSSLutil.ffi.string(buf)) def x509name_to_string(x509name): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 62d5d76b4..b134beb88 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -1,17 +1,16 @@ import os from os.path import join, abspath -from twisted.trial import unittest from twisted.internet import defer from scrapy.utils.testsite import SiteTest from scrapy.utils.testproc import ProcessTest -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode from tests.test_commands import CommandTest def _textmode(bstr): """Normalize input the same as writing to a file and reading from it in text mode""" - return to_native_str(bstr).replace(os.linesep, '\n') + return to_unicode(bstr).replace(os.linesep, '\n') class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' diff --git a/tests/test_commands.py b/tests/test_commands.py index b8445ae6c..536379170 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -10,13 +10,10 @@ from contextlib import contextmanager from threading import Timer from twisted.trial import unittest -from twisted.internet import defer import scrapy -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv -from scrapy.utils.testsite import SiteTest -from scrapy.utils.testproc import ProcessTest from tests.test_crawler import ExceptionSpider, NoRequestsSpider @@ -56,7 +53,7 @@ class ProjectTest(unittest.TestCase): finally: timer.cancel() - return p, to_native_str(stdout), to_native_str(stderr) + return p, to_unicode(stdout), to_unicode(stderr) class StartprojectTest(ProjectTest): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 0c70bf80e..87139e81f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -26,8 +26,7 @@ from scrapy.extensions.feedexport import ( S3FeedStorage, StdoutFeedStorage, BlockingFeedStorage) from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler -from scrapy.utils.python import to_native_str -from scrapy.utils.project import get_project_settings +from scrapy.utils.python import to_unicode from pathlib import Path @@ -459,7 +458,7 @@ class FeedExportTest(unittest.TestCase): settings.update({'FEED_FORMAT': 'csv'}) data = yield self.exported_data(items, settings) - reader = csv.DictReader(to_native_str(data).splitlines()) + reader = csv.DictReader(to_unicode(data).splitlines()) got_rows = list(reader) if ordered: self.assertEqual(reader.fieldnames, header) @@ -473,7 +472,7 @@ class FeedExportTest(unittest.TestCase): settings = settings or {} settings.update({'FEED_FORMAT': 'jl'}) data = yield self.exported_data(items, settings) - parsed = [json.loads(to_native_str(line)) for line in data.splitlines()] + parsed = [json.loads(to_unicode(line)) for line in data.splitlines()] rows = [{k: v for k, v in row.items() if v} for row in rows] self.assertEqual(rows, parsed) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 9fe201579..3518da21c 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -7,12 +7,11 @@ from unittest import mock from urllib.parse import unquote_to_bytes import warnings -import six from six.moves import xmlrpc_client as xmlrpclib from six.moves.urllib.parse import urlparse, parse_qs, unquote from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse -from scrapy.utils.python import to_bytes, to_native_str +from scrapy.utils.python import to_bytes, to_unicode class RequestTest(unittest.TestCase): @@ -349,8 +348,8 @@ class FormRequestTest(RequestTest): request_class = FormRequest def assertQueryEqual(self, first, second, msg=None): - first = to_native_str(first).split("&") - second = to_native_str(second).split("&") + first = to_unicode(first).split("&") + second = to_unicode(second).split("&") return self.assertEqual(sorted(first), sorted(second), msg) def test_empty_formdata(self): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index ec3b51086..883c943da 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -7,7 +7,7 @@ from w3lib.encoding import resolve_encoding from scrapy.http import (Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers) from scrapy.selector import Selector -from scrapy.utils.python import to_native_str +from scrapy.utils.python import to_unicode from scrapy.exceptions import NotSupported from scrapy.link import Link from tests import get_testdata @@ -204,11 +204,11 @@ class TextResponseTest(BaseResponseTest): assert isinstance(resp.url, str) resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8') - self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3')) + self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1') self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]}) - self.assertEqual(resp.url, to_native_str(b'http://www.example.com/price/\xc2\xa3')) + self.assertEqual(resp.url, to_unicode(b'http://www.example.com/price/\xc2\xa3')) resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]}) self.assertEqual(resp.url, 'http://www.example.com/price/\xa3') diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 9aaab560a..cd7480e33 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,6 +1,5 @@ # coding=utf-8 from twisted.trial import unittest -from scrapy.utils.python import to_native_str def reppy_available(): From 87c23ba22d2ef714d778b62c794333acaf232f60 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 Aug 2019 16:29:53 +0500 Subject: [PATCH 58/70] Remove Py2-only code that checks sys.version_info. --- tests/test_utils_reqser.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 11ac56897..92cd16de7 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -80,8 +80,6 @@ class RequestSerializationTest(unittest.TestCase): self._assert_serializes_ok(r, spider=self.spider) def test_mixin_private_callback_serialization(self): - if sys.version_info[0] < 3: - return r = Request("http://www.example.com", callback=self.spider._TestSpiderMixin__mixin_callback, errback=self.spider.handle_error) @@ -119,9 +117,8 @@ class RequestSerializationTest(unittest.TestCase): def test_private_name_mangling(self): self._assert_mangles_to( self.spider, '_TestSpider__parse_item_private') - if sys.version_info[0] >= 3: - self._assert_mangles_to( - self.spider, '_TestSpiderMixin__mixin_callback') + self._assert_mangles_to( + self.spider, '_TestSpiderMixin__mixin_callback') def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) From a9c891399d1bf8a888392afe80c82a8ec8f2e8e7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 31 Oct 2019 22:55:58 +0500 Subject: [PATCH 59/70] Fix a duplicate ref name in docs. --- docs/topics/downloader-middleware.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 366b95510..e93645077 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -474,7 +474,7 @@ DBM storage backend A DBM_ storage backend is also available for the HTTP cache middleware. - By default, it uses the dbm_ module, but you can change it with the + By default, it uses the `dbm module`_, but you can change it with the :setting:`HTTPCACHE_DBM_MODULE` setting. .. _httpcache-storage-custom: @@ -1202,4 +1202,4 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. .. _DBM: https://en.wikipedia.org/wiki/Dbm -.. _dbm: https://docs.python.org/3/library/dbm.html +.. _dbm module: https://docs.python.org/3/library/dbm.html From dd367438fa7a7fec923b28648c4e909cbed1b47d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 1 Nov 2019 20:05:37 +0500 Subject: [PATCH 60/70] Improve the dbm module ref. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- docs/topics/downloader-middleware.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index e93645077..ae6d41809 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -474,7 +474,7 @@ DBM storage backend A DBM_ storage backend is also available for the HTTP cache middleware. - By default, it uses the `dbm module`_, but you can change it with the + By default, it uses the :mod:`dbm`, but you can change it with the :setting:`HTTPCACHE_DBM_MODULE` setting. .. _httpcache-storage-custom: @@ -1202,4 +1202,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. .. _DBM: https://en.wikipedia.org/wiki/Dbm -.. _dbm module: https://docs.python.org/3/library/dbm.html From be6da52019990c1db45b1101dd99787752a14313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 14 Nov 2019 10:31:55 +0100 Subject: [PATCH 61/70] Include extensions from #2067 --- scrapy/linkextractors/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 3c75e683d..a2ac963fe 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -19,9 +19,12 @@ from scrapy.utils.url import ( # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ + # archives + '7z', '7zip', 'bz2', 'rar', 'tar', 'tar.gz', 'xz', 'zip', + # images 'mng', 'pct', 'bmp', 'gif', 'jpg', 'jpeg', 'png', 'pst', 'psp', 'tif', - 'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg', + 'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg', 'cdr', 'ico', # audio 'mp3', 'wma', 'ogg', 'wav', 'ra', 'aac', 'mid', 'au', 'aiff', @@ -35,7 +38,7 @@ IGNORED_EXTENSIONS = [ 'odp', # other - 'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', 'dmg', 'iso', 'apk', 'xz' + 'css', 'pdf', 'exe', 'bin', 'rss', 'dmg', 'iso', 'apk' ] From 3631453bfb1fb2426916919dbfd489ba9ffd9505 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Thu, 14 Nov 2019 15:07:53 +0500 Subject: [PATCH 62/70] Remove spaces on a blank line. --- scrapy/linkextractors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index a2ac963fe..e4c62f87b 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -21,7 +21,7 @@ from scrapy.utils.url import ( IGNORED_EXTENSIONS = [ # archives '7z', '7zip', 'bz2', 'rar', 'tar', 'tar.gz', 'xz', 'zip', - + # images 'mng', 'pct', 'bmp', 'gif', 'jpg', 'jpeg', 'png', 'pst', 'psp', 'tif', 'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg', 'cdr', 'ico', From e291460db67ef8c9e52de02a0a4a86f4bce39b9d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 14 Nov 2019 15:24:37 +0500 Subject: [PATCH 63/70] Fix flake8-detected errors. --- scrapy/crawler.py | 1 - scrapy/exporters.py | 1 - scrapy/http/cookies.py | 2 +- scrapy/link.py | 2 ++ tests/test_pipeline_media.py | 2 -- 5 files changed, 3 insertions(+), 5 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 19b998e0d..8868a985b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -3,7 +3,6 @@ import signal import logging import warnings -import sys from twisted.internet import reactor, defer from zope.interface.verify import verifyClass, DoesNotImplement diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 8eb52995e..3defafd60 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -4,7 +4,6 @@ Item Exporters are used to export/serialize items into different formats. import csv import io -import sys import pprint import marshal import six diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 4532c3ab7..60a14c6f8 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -166,7 +166,7 @@ class WrappedRequest(object): def get_header(self, name, default=None): return to_unicode(self.request.headers.get(name, default), - errors='replace') + errors='replace') def header_items(self): return [ diff --git a/scrapy/link.py b/scrapy/link.py index be1888ef0..a809c5ca4 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -4,6 +4,8 @@ This module defines the Link object used in Link extractors. For actual link extractors implementation see scrapy.linkextractors, or its documentation in: docs/topics/link-extractors.rst """ + + class Link(object): """Link objects represent an extracted link by the LinkExtractor.""" diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index ad2618ec9..ad958e25f 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,7 +1,5 @@ from __future__ import print_function -import sys - from testfixtures import LogCapture from twisted.trial import unittest from twisted.python.failure import Failure From b8ef12cd4707f9e095abdd26cc137558009c335f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 14 Nov 2019 12:10:25 +0100 Subject: [PATCH 64/70] Add bandit to CI --- .bandit.yml | 16 ++++++++++++++++ .travis.yml | 2 ++ tox.ini | 7 +++++++ 3 files changed, 25 insertions(+) create mode 100644 .bandit.yml diff --git a/.bandit.yml b/.bandit.yml new file mode 100644 index 000000000..00554587a --- /dev/null +++ b/.bandit.yml @@ -0,0 +1,16 @@ +skips: +- B101 +- B105 +- B303 +- B306 +- B307 +- B311 +- B320 +- B321 +- B402 +- B404 +- B406 +- B410 +- B503 +- B603 +- B605 diff --git a/.travis.yml b/.travis.yml index 0e77af9fd..9f477e860 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ branches: - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: include: + - env: TOXENV=security + python: 3.8 - env: TOXENV=flake8 python: 3.8 - env: TOXENV=pypy3 diff --git a/tox.ini b/tox.ini index 3668058c3..cd575f3c5 100644 --- a/tox.ini +++ b/tox.ini @@ -62,6 +62,13 @@ basepython = pypy3 commands = py.test {posargs:docs scrapy tests} +[testenv:security] +basepython = python3.8 +deps = + bandit +commands = + bandit -r -c .bandit.yml {posargs:scrapy} + [testenv:flake8] basepython = python3.8 deps = From 5ee5508cc33116b3700c9b745138b56cc67951f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 14 Nov 2019 15:42:34 +0100 Subject: [PATCH 65/70] Have CI record the 10 slowest tests --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 3668058c3..ec04035f5 100644 --- a/tox.ini +++ b/tox.ini @@ -21,7 +21,7 @@ passenv = GCS_TEST_FILE_URI GCS_PROJECT_ID commands = - py.test --cov=scrapy --cov-report= {posargs:docs scrapy tests} + py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} [testenv:py35] basepython = python3.5 @@ -60,7 +60,7 @@ basepython = python3.8 [testenv:pypy3] basepython = pypy3 commands = - py.test {posargs:docs scrapy tests} + py.test {posargs:--durations=10 docs scrapy tests} [testenv:flake8] basepython = python3.8 From fe3a121f1358fc904915f5b32e276520523c553a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 14 Nov 2019 22:50:53 +0500 Subject: [PATCH 66/70] Use kwargs when calling get_func_args. --- tests/test_utils_python.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 096aa50b7..a94398796 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -232,10 +232,10 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual(get_func_args(operator.itemgetter(2)), []) else: self.assertEqual( - get_func_args(six.text_type.split, True), ['sep', 'maxsplit']) - self.assertEqual(get_func_args(" ".join, True), ['list']) + get_func_args(six.text_type.split, stripself=True), ['sep', 'maxsplit']) + self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) self.assertEqual( - get_func_args(operator.itemgetter(2), True), ['obj']) + get_func_args(operator.itemgetter(2), stripself=True), ['obj']) def test_without_none_values(self): From 3b2289ad012043b94b495d411316b2778bf3db35 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 14 Nov 2019 22:53:28 +0500 Subject: [PATCH 67/70] Rename test_non_str_url_py2 to test_bytes_url. --- tests/test_link.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_link.py b/tests/test_link.py index 5e2ce5eeb..e0f1efffa 100644 --- a/tests/test_link.py +++ b/tests/test_link.py @@ -43,6 +43,6 @@ class LinkTest(unittest.TestCase): l2 = eval(repr(l1)) self._assert_same_links(l1, l2) - def test_non_str_url_py2(self): + def test_bytes_url(self): with self.assertRaises(TypeError): Link(b"http://www.example.com/\xc2\xa3") From 77a84f620ffa5d001072c7122c0f38048fe15606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Alberto=20/=20Speedy?= Date: Fri, 15 Nov 2019 11:09:24 +0100 Subject: [PATCH 68/70] Fix string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 87eaac2af..4c038030f 100644 --- a/README.rst +++ b/README.rst @@ -62,7 +62,7 @@ directory. Releases ======== -You can check https://docs.scrapy.org/en/latest/news.html for release notes. +You can check https://docs.scrapy.org/en/latest/news.html for the release notes. Community (blog, twitter, mail list, IRC) ========================================= From 0e252f5a13be3195fdc3ec1d66a111ae01a0ab80 Mon Sep 17 00:00:00 2001 From: Marc Hernandez Cabot Date: Fri, 15 Nov 2019 19:12:43 +0100 Subject: [PATCH 69/70] fix E711 and E713 --- pytest.ini | 4 ++-- tests/test_downloader_handlers.py | 4 ++-- tests/test_downloadermiddleware_httpcache.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pytest.ini b/pytest.ini index fa6e7287e..529ad5d27 100644 --- a/pytest.ini +++ b/pytest.ini @@ -192,14 +192,14 @@ flake8-ignore = tests/test_crawl.py E501 E741 E265 tests/test_crawler.py F841 E306 E501 tests/test_dependencies.py E302 F841 E501 E305 - tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E711 E126 E226 E123 + tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E126 E226 E123 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E302 E501 tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126 tests/test_downloadermiddleware_decompression.py E127 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 - tests/test_downloadermiddleware_httpcache.py E713 E501 E302 E305 F401 + tests/test_downloadermiddleware_httpcache.py E501 E302 E305 F401 tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123 tests/test_downloadermiddleware_httpproxy.py F401 E501 E128 tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305 diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 6090998d4..59d4a3eec 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -615,7 +615,7 @@ class Http11MockServerTestCase(unittest.TestCase): crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=Request(url=self.mockserver.url(''))) failure = crawler.spider.meta.get('failure') - self.assertTrue(failure == None) + self.assertTrue(failure is None) reason = crawler.spider.meta['close_reason'] self.assertTrue(reason, 'finished') @@ -636,7 +636,7 @@ class Http11MockServerTestCase(unittest.TestCase): yield crawler.crawl(seed=request) # download_maxsize = 50 is enough for the gzipped response failure = crawler.spider.meta.get('failure') - self.assertTrue(failure == None) + self.assertTrue(failure is None) reason = crawler.spider.meta['close_reason'] self.assertTrue(reason, 'finished') else: diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 950664ffe..9d863b6e3 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -85,8 +85,8 @@ class _BaseTest(unittest.TestCase): def assertEqualRequestButWithCacheValidators(self, request1, request2): self.assertEqual(request1.url, request2.url) - assert not b'If-None-Match' in request1.headers - assert not b'If-Modified-Since' in request1.headers + assert b'If-None-Match' not in request1.headers + assert b'If-Modified-Since' not in request1.headers assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since')) self.assertEqual(request1.body, request2.body) From 78ad01632f16a97fb180d8c2972075b19f471380 Mon Sep 17 00:00:00 2001 From: Andrey Rahmatullin Date: Tue, 19 Nov 2019 14:43:30 +0500 Subject: [PATCH 70/70] Fix flake8 problems in PR #3989 (#4176) --- tests/test_logformatter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 5b5d68f4f..afbd25d0c 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -62,7 +62,7 @@ class LogFormatterTestCase(unittest.TestCase): lines = logline.splitlines() assert all(isinstance(x, six.text_type) for x in lines) self.assertEqual(lines, [u"Dropped: \u2018", '{}']) - + def test_error(self): # In practice, the complete traceback is shown by passing the # 'exc_info' argument to the logging function @@ -121,7 +121,7 @@ class LogformatterSubclassTest(LogFormatterTestCase): "Crawled (200) (referer: http://example.com) ['cached']") def test_flags_in_request(self): - req = Request("http://www.example.com", flags=['test','flag']) + req = Request("http://www.example.com", flags=['test', 'flag']) res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args']