From 3b6f7ac9f2f5b48b9f2f3ce106d1205599d2164f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 22 Oct 2019 19:43:02 +0200 Subject: [PATCH 01/50] Use pylint --- .travis.yml | 2 + docs/utils/linkfix.py | 85 ++++++++++++++++++++++------------------- pylintrc | 88 +++++++++++++++++++++++++++++++++++++++++++ tox.ini | 14 +++++++ 4 files changed, 150 insertions(+), 39 deletions(-) create mode 100644 pylintrc diff --git a/.travis.yml b/.travis.yml index 0190a7f4d..28a19f4f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ branches: - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: include: + - env: TOXENV=pylint + python: 3.7 - env: TOXENV=py27 python: 2.7 - env: TOXENV=py27-pinned diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 6290adbe2..9acfc3b23 100755 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -14,50 +14,57 @@ Author: dufferzafar import re -# Used for remembering the file (and its contents) -# so we don't have to open the same file again. -_filename = None -_contents = None -# A regex that matches standard linkcheck output lines -line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') +def main(): -# Read lines from the linkcheck output file -try: - with open("build/linkcheck/output.txt") as out: - output_lines = out.readlines() -except IOError: - print("linkcheck output not found; please run linkcheck first.") - exit(1) + # Used for remembering the file (and its contents) + # so we don't have to open the same file again. + _filename = None + _contents = None -# For every line, fix the respective file -for line in output_lines: - match = re.match(line_re, line) + # A regex that matches standard linkcheck output lines + line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') - if match: - newfilename = match.group(1) - errortype = match.group(2) + # Read lines from the linkcheck output file + try: + with open("build/linkcheck/output.txt") as out: + output_lines = out.readlines() + except IOError: + print("linkcheck output not found; please run linkcheck first.") + exit(1) - # Broken links can't be fixed and - # I am not sure what do with the local ones. - if errortype.lower() in ["broken", "local"]: - print("Not Fixed: " + line) + # For every line, fix the respective file + for line in output_lines: + match = re.match(line_re, line) + + if match: + newfilename = match.group(1) + errortype = match.group(2) + + # Broken links can't be fixed and + # I am not sure what do with the local ones. + if errortype.lower() in ["broken", "local"]: + print("Not Fixed: " + line) + else: + # If this is a new file + if newfilename != _filename: + + # Update the previous file + if _filename: + with open(_filename, "w") as _file: + _file.write(_contents) + + _filename = newfilename + + # Read the new file to memory + with open(_filename) as _file: + _contents = _file.read() + + _contents = _contents.replace(match.group(3), match.group(4)) else: - # If this is a new file - if newfilename != _filename: + # We don't understand what the current line means! + print("Not Understood: " + line) - # Update the previous file - if _filename: - with open(_filename, "w") as _file: - _file.write(_contents) - _filename = newfilename - - # Read the new file to memory - with open(_filename) as _file: - _contents = _file.read() - - _contents = _contents.replace(match.group(3), match.group(4)) - else: - # We don't understand what the current line means! - print("Not Understood: " + line) +if __name__ == '__main__': + main() diff --git a/pylintrc b/pylintrc new file mode 100644 index 000000000..b83bc9f82 --- /dev/null +++ b/pylintrc @@ -0,0 +1,88 @@ +[MASTER] +persistent=no +jobs=1 # >1 hides results + +[MESSAGES CONTROL] +disable=abstract-method, + anomalous-backslash-in-string, + arguments-differ, + attribute-defined-outside-init, + bad-classmethod-argument, + bad-continuation, + bad-indentation, + bad-mcs-classmethod-argument, + bad-whitespace, + broad-except, + c-extension-no-member, + catching-non-exception, + cell-var-from-loop, + comparison-with-callable, + consider-using-in, + cyclic-import, + dangerous-default-value, + deprecated-method, + deprecated-module, + duplicate-code, # https://github.com/PyCQA/pylint/issues/214 + eval-used, + expression-not-assigned, + fixme, + function-redefined, + global-statement, + import-error, + import-outside-toplevel, + inconsistent-return-statements, + inherit-non-class, + invalid-name, + keyword-arg-before-vararg, + line-too-long, + logging-format-interpolation, + logging-not-lazy, + lost-exception, + method-hidden, + missing-docstring, + missing-final-newline, + multiple-imports, + multiple-statements, + no-else-continue, + no-else-raise, + no-else-return, + no-init, + no-member, + no-method-argument, + no-name-in-module, + no-self-argument, + no-self-use, + pointless-string-statement, + protected-access, + redefined-argument-from-local, + redefined-builtin, + redefined-outer-name, + reimported, + signature-differs, + super-init-not-called, + superfluous-parens, + too-few-public-methods, + too-many-ancestors, + too-many-arguments, + too-many-branches, + too-many-function-args, + too-many-instance-attributes, + too-many-locals, + too-many-return-statements, + trailing-newlines, + trailing-whitespace, + unexpected-special-method-signature, + ungrouped-imports, + unidiomatic-typecheck, + unnecessary-comprehension, + unnecessary-pass, + unsubscriptable-object, + unused-argument, + unused-import, + unused-variable, + unused-wildcard-import, + used-before-assignment, + useless-object-inheritance, # Required for Python 2 support + wildcard-import, + wrong-import-order, + wrong-import-position diff --git a/tox.ini b/tox.ini index ffe7360d3..e7d366fe9 100644 --- a/tox.ini +++ b/tox.ini @@ -98,6 +98,20 @@ deps = {[testenv:py35]deps} commands = py.test {posargs:scrapy tests} +[testenv:pylint] +basepython = python3.7 +deps = + {[testenv:py35]deps} + # Optional dependencies + boto + reppy + robotexclusionrulesparser + # Test dependencies + pylint + +commands = + pylint scrapy + [docs] changedir = docs deps = From 02577f55a0586bc3e6c13a4a3ea572c7eefc82b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 24 Oct 2019 13:25:11 +0200 Subject: [PATCH 02/50] Have PyLint cover all Python files in the repository --- pylintrc | 19 +++++++++++++++++++ tox.ini | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/pylintrc b/pylintrc index b83bc9f82..ca3ea1c57 100644 --- a/pylintrc +++ b/pylintrc @@ -11,13 +11,18 @@ disable=abstract-method, bad-continuation, bad-indentation, bad-mcs-classmethod-argument, + bad-super-call, bad-whitespace, + blacklisted-name, broad-except, c-extension-no-member, catching-non-exception, cell-var-from-loop, comparison-with-callable, + consider-iterating-dictionary, consider-using-in, + consider-using-set-comprehension, + consider-using-sys-exit, cyclic-import, dangerous-default-value, deprecated-method, @@ -30,6 +35,7 @@ disable=abstract-method, global-statement, import-error, import-outside-toplevel, + import-self, inconsistent-return-statements, inherit-non-class, invalid-name, @@ -39,6 +45,7 @@ disable=abstract-method, logging-not-lazy, lost-exception, method-hidden, + misplaced-comparison-constant, missing-docstring, missing-final-newline, multiple-imports, @@ -52,6 +59,9 @@ disable=abstract-method, no-name-in-module, no-self-argument, no-self-use, + no-value-for-parameter, + not-callable, + pointless-statement, pointless-string-statement, protected-access, redefined-argument-from-local, @@ -59,6 +69,7 @@ disable=abstract-method, redefined-outer-name, reimported, signature-differs, + singleton-comparison, super-init-not-called, superfluous-parens, too-few-public-methods, @@ -67,15 +78,21 @@ disable=abstract-method, too-many-branches, too-many-function-args, too-many-instance-attributes, + too-many-lines, too-many-locals, + too-many-public-methods, too-many-return-statements, trailing-newlines, trailing-whitespace, + unbalanced-tuple-unpacking, + undefined-variable, unexpected-special-method-signature, ungrouped-imports, unidiomatic-typecheck, unnecessary-comprehension, + unnecessary-lambda, unnecessary-pass, + unreachable, unsubscriptable-object, unused-argument, unused-import, @@ -83,6 +100,8 @@ disable=abstract-method, unused-wildcard-import, used-before-assignment, useless-object-inheritance, # Required for Python 2 support + useless-return, + useless-super-delegation, wildcard-import, wrong-import-order, wrong-import-position diff --git a/tox.ini b/tox.ini index e7d366fe9..428571ef2 100644 --- a/tox.ini +++ b/tox.ini @@ -110,7 +110,7 @@ deps = pylint commands = - pylint scrapy + pylint conftest.py docs extras scrapy setup.py tests [docs] changedir = docs From c7f9b955bdf2405fce58907b0395abce2400a66d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 19 Dec 2019 12:44:52 +0100 Subject: [PATCH 03/50] Pylint: ignore not-an-iterable --- pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/pylintrc b/pylintrc index ca3ea1c57..c52a4c2d0 100644 --- a/pylintrc +++ b/pylintrc @@ -60,6 +60,7 @@ disable=abstract-method, no-self-argument, no-self-use, no-value-for-parameter, + not-an-iterable, not-callable, pointless-statement, pointless-string-statement, From 72b8613ee9827af031862bd84f1bea9acefcbebe Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 27 Nov 2019 14:46:20 -0300 Subject: [PATCH 04/50] bytes_received signal (no tests) --- docs/topics/signals.rst | 45 ++++++++++++++++------- scrapy/core/downloader/handlers/http11.py | 25 +++++++++++-- scrapy/signals.py | 1 + 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3f29aa323..6efb73abb 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -73,7 +73,7 @@ engine_started Sent when the Scrapy engine has started crawling. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. .. note:: This signal may be fired *after* the :signal:`spider_opened` signal, depending on how the spider was started. So **don't** rely on this signal @@ -88,7 +88,7 @@ engine_stopped Sent when the Scrapy engine is stopped (for example, when a crawling process has finished). - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. item_scraped ------------ @@ -99,7 +99,7 @@ item_scraped Sent when an item has been scraped, after it has passed all the :ref:`topics-item-pipeline` stages (without being dropped). - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param item: the item scraped :type item: dict or :class:`~scrapy.item.Item` object @@ -119,7 +119,7 @@ item_dropped Sent after an item has been dropped from the :ref:`topics-item-pipeline` when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param item: the item dropped from the :ref:`topics-item-pipeline` :type item: dict or :class:`~scrapy.item.Item` object @@ -144,7 +144,7 @@ item_error Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises an exception), except :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param item: the item dropped from the :ref:`topics-item-pipeline` :type item: dict or :class:`~scrapy.item.Item` object @@ -158,6 +158,23 @@ item_error :param failure: the exception raised :type failure: twisted.python.failure.Failure +bytes_received +-------------- + +.. signal:: bytes_received +.. function:: bytes_received(data, request) + + Sent by the HTTP 1.1 download handler when a group of bytes is + received for a specific request. + + This signal does not support returning deferreds from its handlers. + + :param data: the data received by the download handler + :type spider: :class:`bytes` object + + :param request: the request that generated the response + :type request: :class:`~scrapy.http.Request` object + spider_closed ------------- @@ -167,7 +184,7 @@ spider_closed Sent after a spider has been closed. This can be used to release per-spider resources reserved on :signal:`spider_opened`. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param spider: the spider which has been closed :type spider: :class:`~scrapy.spiders.Spider` object @@ -191,7 +208,7 @@ spider_opened reserve per-spider resources, but can be used for any task that needs to be performed when a spider is opened. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param spider: the spider which has been opened :type spider: :class:`~scrapy.spiders.Spider` object @@ -215,7 +232,7 @@ spider_idle You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to prevent the spider from being closed. - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param spider: the spider which has gone idle :type spider: :class:`~scrapy.spiders.Spider` object @@ -234,7 +251,7 @@ spider_error Sent when a spider callback generates an error (ie. raises an exception). - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param failure: the exception raised :type failure: twisted.python.failure.Failure @@ -254,7 +271,7 @@ request_scheduled Sent when the engine schedules a :class:`~scrapy.http.Request`, to be downloaded later. - The signal does not support returning deferreds from their handlers. + The signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler :type request: :class:`~scrapy.http.Request` object @@ -271,7 +288,7 @@ request_dropped Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be downloaded later, is rejected by the scheduler. - The signal does not support returning deferreds from their handlers. + The signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler :type request: :class:`~scrapy.http.Request` object @@ -287,7 +304,7 @@ request_reached_downloader Sent when a :class:`~scrapy.http.Request` reached downloader. - The signal does not support returning deferreds from their handlers. + The signal does not support returning deferreds from its handlers. :param request: the request that reached downloader :type request: :class:`~scrapy.http.Request` object @@ -304,7 +321,7 @@ response_received Sent when the engine receives a new :class:`~scrapy.http.Response` from the downloader. - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param response: the response received :type response: :class:`~scrapy.http.Response` object @@ -323,7 +340,7 @@ response_downloaded Sent by the downloader right after a ``HTTPResponse`` is downloaded. - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param response: the response downloaded :type response: :class:`~scrapy.http.Response` object diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 5a5f6cf0a..92c3d5f5c 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -16,6 +16,7 @@ from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from zope.interface import implementer +from scrapy import signals from scrapy.core.downloader.tls import openssl_methods from scrapy.core.downloader.webclient import _parse from scrapy.exceptions import ScrapyDeprecationWarning @@ -32,6 +33,7 @@ class HTTP11DownloadHandler: lazy = False def __init__(self, settings, crawler=None): + self.crawler = crawler self._pool = HTTPConnectionPool(reactor, persistent=True) self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self._pool._factory.noisy = False @@ -76,6 +78,7 @@ class HTTP11DownloadHandler: maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), fail_on_dataloss=self._fail_on_dataloss, + crawler=self.crawler, ) return agent.download_request(request) @@ -272,7 +275,7 @@ class ScrapyAgent(object): _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, - maxsize=0, warnsize=0, fail_on_dataloss=True): + maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress @@ -281,6 +284,7 @@ class ScrapyAgent(object): self._warnsize = warnsize self._fail_on_dataloss = fail_on_dataloss self._txresponse = None + self._crawler = crawler def _get_agent(self, request, timeout): bindaddress = request.meta.get('bindaddress') or self._bindAddress @@ -409,7 +413,15 @@ class ScrapyAgent(object): d = defer.Deferred(_cancel) txresponse.deliverBody( - _ResponseReader(d, txresponse, request, maxsize, warnsize, fail_on_dataloss) + _ResponseReader( + d, + txresponse, + request, + maxsize, + warnsize, + fail_on_dataloss, + self._crawler, + ) ) # save response for timeouts @@ -445,7 +457,7 @@ class _RequestBodyProducer(object): class _ResponseReader(protocol.Protocol): - def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss): + def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler): self._finished = finished self._txresponse = txresponse self._request = request @@ -456,6 +468,7 @@ class _ResponseReader(protocol.Protocol): self._fail_on_dataloss_warned = False self._reached_warnsize = False self._bytes_received = 0 + self._crawler = crawler def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. @@ -465,6 +478,12 @@ class _ResponseReader(protocol.Protocol): self._bodybuf.write(bodyBytes) self._bytes_received += len(bodyBytes) + self._crawler.signals.send_catch_log( + signal=signals.bytes_received, + data=bodyBytes, + request=self._request, + ) + if self._maxsize and self._bytes_received > self._maxsize: logger.error("Received (%(bytes)s) bytes larger than download " "max size (%(maxsize)s) in request %(request)s.", diff --git a/scrapy/signals.py b/scrapy/signals.py index 6b9125302..590421893 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -16,6 +16,7 @@ request_dropped = object() request_reached_downloader = object() response_received = object() response_downloaded = object() +bytes_received = object() item_scraped = object() item_dropped = object() item_error = object() From cab449b1952020b86fbe2915a537150fc885c567 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 29 Nov 2019 11:37:40 -0300 Subject: [PATCH 05/50] Typo fix --- tests/test_engine.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index 25dee7c1f..9d68836cc 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -107,7 +107,7 @@ class CrawlerRun(object): self.reqreached = [] self.itemerror = [] self.itemresp = [] - self.signals_catched = {} + self.signals_caught = {} self.spider_class = spider_class def run(self): @@ -172,7 +172,7 @@ class CrawlerRun(object): signalargs = kwargs.copy() sig = signalargs.pop('signal') signalargs.pop('sender', None) - self.signals_catched[sig] = signalargs + self.signals_caught[sig] = signalargs class EngineTest(unittest.TestCase): @@ -186,7 +186,7 @@ class EngineTest(unittest.TestCase): self._assert_scheduled_requests(urls_to_visit=8) self._assert_downloaded_responses() self._assert_scraped_items() - self._assert_signals_catched() + self._assert_signals_caught() @defer.inlineCallbacks def test_crawler_dupefilter(self): @@ -263,19 +263,19 @@ class EngineTest(unittest.TestCase): self.assertEqual('Item 2 name', item['name']) self.assertEqual('200', item['price']) - def _assert_signals_catched(self): - assert signals.engine_started in self.run.signals_catched - assert signals.engine_stopped in self.run.signals_catched - assert signals.spider_opened in self.run.signals_catched - assert signals.spider_idle in self.run.signals_catched - assert signals.spider_closed in self.run.signals_catched + def _assert_signals_caught(self): + assert signals.engine_started in self.run.signals_caught + assert signals.engine_stopped in self.run.signals_caught + assert signals.spider_opened in self.run.signals_caught + assert signals.spider_idle in self.run.signals_caught + assert signals.spider_closed in self.run.signals_caught self.assertEqual({'spider': self.run.spider}, - self.run.signals_catched[signals.spider_opened]) + self.run.signals_caught[signals.spider_opened]) self.assertEqual({'spider': self.run.spider}, - self.run.signals_catched[signals.spider_idle]) + self.run.signals_caught[signals.spider_idle]) self.assertEqual({'spider': self.run.spider, 'reason': 'finished'}, - self.run.signals_catched[signals.spider_closed]) + self.run.signals_caught[signals.spider_closed]) @defer.inlineCallbacks def test_close_downloader(self): From bda37e38bd53d5aae691b56d4136fbff99f78158 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 29 Nov 2019 12:02:27 -0300 Subject: [PATCH 06/50] [Tests] bytes_received signal --- tests/test_engine.py | 53 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index 9d68836cc..b63c7e232 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -13,22 +13,24 @@ module with the ``runserver`` argument:: import os import re import sys +from collections import defaultdict from urllib.parse import urlparse from twisted.internet import reactor, defer -from twisted.web import server, static, util from twisted.trial import unittest +from twisted.web import server, static, util +from pydispatch import dispatcher from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.utils.test import get_crawler -from pydispatch import dispatcher -from tests import tests_datadir -from scrapy.spiders import Spider +from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor -from scrapy.http import Request +from scrapy.spiders import Spider from scrapy.utils.signal import disconnect_all +from scrapy.utils.test import get_crawler + +from tests import tests_datadir, get_testdata class TestItem(Item): @@ -107,6 +109,7 @@ class CrawlerRun(object): self.reqreached = [] self.itemerror = [] self.itemresp = [] + self.bytes = defaultdict(lambda: b"") self.signals_caught = {} self.spider_class = spider_class @@ -124,6 +127,7 @@ class CrawlerRun(object): self.crawler = get_crawler(self.spider_class) self.crawler.signals.connect(self.item_scraped, signals.item_scraped) self.crawler.signals.connect(self.item_error, signals.item_error) + self.crawler.signals.connect(self.bytes_received, signals.bytes_received) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.request_dropped, signals.request_dropped) self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader) @@ -155,6 +159,9 @@ class CrawlerRun(object): def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) + def bytes_received(self, data, request): + self.bytes[request] += data + def request_scheduled(self, request, spider): self.reqplug.append((request, spider)) @@ -187,6 +194,7 @@ class EngineTest(unittest.TestCase): self._assert_downloaded_responses() self._assert_scraped_items() self._assert_signals_caught() + self._assert_bytes_received() @defer.inlineCallbacks def test_crawler_dupefilter(self): @@ -263,6 +271,39 @@ class EngineTest(unittest.TestCase): self.assertEqual('Item 2 name', item['name']) self.assertEqual('200', item['price']) + def _assert_bytes_received(self): + self.assertEqual(8, len(self.run.bytes)) + for request, data in self.run.bytes.items(): + if self.run.getpath(request.url) == "/": + self.assertEqual(data, get_testdata("test_site", "index.html")) + elif self.run.getpath(request.url) == "/item1.html": + self.assertEqual(data, get_testdata("test_site", "item1.html")) + elif self.run.getpath(request.url) == "/item2.html": + self.assertEqual(data, get_testdata("test_site", "item2.html")) + elif self.run.getpath(request.url) == "/redirected": + self.assertEqual(data, b"Redirected here") + elif self.run.getpath(request.url) == '/redirect': + self.assertEqual(data, + b"\n\n" + b" \n" + b" \n" + b" \n" + b" \n" + b" click here\n" + b" \n" + b"\n" + ) + elif self.run.getpath(request.url) == "/tem999.html": + self.assertEqual(data, + b"\n\n" + b" 404 - No Such Resource\n" + b" \n" + b"

No Such Resource

\n" + b"

File not found.

\n" + b" \n" + b"\n" + ) + def _assert_signals_caught(self): assert signals.engine_started in self.run.signals_caught assert signals.engine_stopped in self.run.signals_caught From 89483ce9f709e230ee5ff9050d206430d2d17c9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 3 Dec 2019 12:06:08 +0100 Subject: [PATCH 07/50] Fix Flake8 issues --- tests/test_engine.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index b63c7e232..c0769c992 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -283,7 +283,8 @@ class EngineTest(unittest.TestCase): elif self.run.getpath(request.url) == "/redirected": self.assertEqual(data, b"Redirected here") elif self.run.getpath(request.url) == '/redirect': - self.assertEqual(data, + self.assertEqual( + data, b"\n\n" b" \n" b" \n" @@ -294,7 +295,8 @@ class EngineTest(unittest.TestCase): b"\n" ) elif self.run.getpath(request.url) == "/tem999.html": - self.assertEqual(data, + self.assertEqual( + data, b"\n\n" b" 404 - No Such Resource\n" b" \n" From dbe20a863ff63dce937b2d3b159782d8268e6838 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 27 Jan 2020 12:21:18 -0300 Subject: [PATCH 08/50] bytes_received signal: send spider argument --- docs/topics/signals.rst | 5 ++++- scrapy/core/downloader/handlers/http11.py | 1 + tests/test_engine.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 6efb73abb..3e70ca067 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -162,7 +162,7 @@ bytes_received -------------- .. signal:: bytes_received -.. function:: bytes_received(data, request) +.. function:: bytes_received(data, request, spider) Sent by the HTTP 1.1 download handler when a group of bytes is received for a specific request. @@ -175,6 +175,9 @@ bytes_received :param request: the request that generated the response :type request: :class:`~scrapy.http.Request` object + :param spider: the spider associated with the response + :type spider: :class:`~scrapy.spiders.Spider` object + spider_closed ------------- diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 92c3d5f5c..c53c9bb2d 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -482,6 +482,7 @@ class _ResponseReader(protocol.Protocol): signal=signals.bytes_received, data=bodyBytes, request=self._request, + spider=self._crawler.spider, ) if self._maxsize and self._bytes_received > self._maxsize: diff --git a/tests/test_engine.py b/tests/test_engine.py index c0769c992..57cc89ba3 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -159,7 +159,7 @@ class CrawlerRun(object): def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) - def bytes_received(self, data, request): + def bytes_received(self, data, request, spider): self.bytes[request] += data def request_scheduled(self, request, spider): From 613fd41f44d1455f9c9369087958674f3fdfcc8d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 27 Jan 2020 12:30:26 -0300 Subject: [PATCH 09/50] bytes_received signal: improve test performance --- tests/test_engine.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index 57cc89ba3..bb475958e 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -109,7 +109,7 @@ class CrawlerRun(object): self.reqreached = [] self.itemerror = [] self.itemresp = [] - self.bytes = defaultdict(lambda: b"") + self.bytes = defaultdict(lambda: list()) self.signals_caught = {} self.spider_class = spider_class @@ -160,7 +160,7 @@ class CrawlerRun(object): self.itemresp.append((item, response)) def bytes_received(self, data, request, spider): - self.bytes[request] += data + self.bytes[request].append(data) def request_scheduled(self, request, spider): self.reqplug.append((request, spider)) @@ -274,17 +274,18 @@ class EngineTest(unittest.TestCase): def _assert_bytes_received(self): self.assertEqual(8, len(self.run.bytes)) for request, data in self.run.bytes.items(): + joined_data = b"".join(data) if self.run.getpath(request.url) == "/": - self.assertEqual(data, get_testdata("test_site", "index.html")) + self.assertEqual(joined_data, get_testdata("test_site", "index.html")) elif self.run.getpath(request.url) == "/item1.html": - self.assertEqual(data, get_testdata("test_site", "item1.html")) + self.assertEqual(joined_data, get_testdata("test_site", "item1.html")) elif self.run.getpath(request.url) == "/item2.html": - self.assertEqual(data, get_testdata("test_site", "item2.html")) + self.assertEqual(joined_data, get_testdata("test_site", "item2.html")) elif self.run.getpath(request.url) == "/redirected": - self.assertEqual(data, b"Redirected here") + self.assertEqual(joined_data, b"Redirected here") elif self.run.getpath(request.url) == '/redirect': self.assertEqual( - data, + joined_data, b"\n\n" b" \n" b" \n" @@ -296,7 +297,7 @@ class EngineTest(unittest.TestCase): ) elif self.run.getpath(request.url) == "/tem999.html": self.assertEqual( - data, + joined_data, b"\n\n" b" 404 - No Such Resource\n" b" \n" From 4ffd18fb11ff89863569b8b4de44241e3ca2f86e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 27 Jan 2020 13:29:33 -0300 Subject: [PATCH 10/50] [docs] Mention that signals.bytes_received could be fired multiple times --- docs/topics/signals.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3e70ca067..f490911f3 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -165,7 +165,8 @@ bytes_received .. function:: bytes_received(data, request, spider) Sent by the HTTP 1.1 download handler when a group of bytes is - received for a specific request. + received for a specific request. This signal might be fired + multiple times for the same request. This signal does not support returning deferreds from its handlers. From 2c9643d38cc076c4d2032efd994fda4cfcc9f88a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 29 Jan 2020 14:11:56 -0300 Subject: [PATCH 11/50] Test: bytes_received signal fired multiple times --- tests/test_engine.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index bb475958e..3c5cc403b 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -12,6 +12,7 @@ module with the ``runserver`` argument:: import os import re +import string import sys from collections import defaultdict from urllib.parse import urlparse @@ -90,6 +91,7 @@ def start_test_site(debug=False): r = static.File(root_dir) r.putChild(b"redirect", util.Redirect(b"/redirected")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) + r.putChild(b"random", static.Data(string.ascii_letters.encode("utf8") * 2**14, "text/plain")) port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") if debug: @@ -117,8 +119,12 @@ class CrawlerRun(object): self.port = start_test_site() self.portno = self.port.getHost().port - start_urls = [self.geturl("/"), self.geturl("/redirect"), - self.geturl("/redirect")] # a duplicate + start_urls = [ + self.geturl("/"), + self.geturl("/redirect"), + self.geturl("/redirect"), # duplicate + self.geturl("/random"), + ] for name, signal in vars(signals).items(): if not name.startswith('_'): @@ -190,7 +196,7 @@ class EngineTest(unittest.TestCase): self.run = CrawlerRun(spider) yield self.run.run() self._assert_visited_urls() - self._assert_scheduled_requests(urls_to_visit=8) + self._assert_scheduled_requests(urls_to_visit=9) self._assert_downloaded_responses() self._assert_scraped_items() self._assert_signals_caught() @@ -200,7 +206,7 @@ class EngineTest(unittest.TestCase): def test_crawler_dupefilter(self): self.run = CrawlerRun(TestDupeFilterSpider) yield self.run.run() - self._assert_scheduled_requests(urls_to_visit=7) + self._assert_scheduled_requests(urls_to_visit=8) self._assert_dropped_requests() @defer.inlineCallbacks @@ -237,8 +243,8 @@ class EngineTest(unittest.TestCase): def _assert_downloaded_responses(self): # response tests - self.assertEqual(8, len(self.run.respplug)) - self.assertEqual(8, len(self.run.reqreached)) + self.assertEqual(9, len(self.run.respplug)) + self.assertEqual(9, len(self.run.reqreached)) for response, _ in self.run.respplug: if self.run.getpath(response.url) == '/item999.html': @@ -272,7 +278,7 @@ class EngineTest(unittest.TestCase): self.assertEqual('200', item['price']) def _assert_bytes_received(self): - self.assertEqual(8, len(self.run.bytes)) + self.assertEqual(9, len(self.run.bytes)) for request, data in self.run.bytes.items(): joined_data = b"".join(data) if self.run.getpath(request.url) == "/": @@ -306,6 +312,8 @@ class EngineTest(unittest.TestCase): b" \n" b"\n" ) + elif self.run.getpath(request.url) == "/random": + self.assertTrue(len(data) > 1) # signal was fired multiple times def _assert_signals_caught(self): assert signals.engine_started in self.run.signals_caught From a499f38b14d16338d20084c0dcb24528a1f1f22f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 29 Jan 2020 14:35:17 -0300 Subject: [PATCH 12/50] Remove object parent class --- scrapy/core/downloader/handlers/http11.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c53c9bb2d..6f1bd9ad6 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -268,7 +268,7 @@ class ScrapyProxyAgent(Agent): ) -class ScrapyAgent(object): +class ScrapyAgent: _Agent = Agent _ProxyAgent = ScrapyProxyAgent @@ -438,7 +438,7 @@ class ScrapyAgent(object): @implementer(IBodyProducer) -class _RequestBodyProducer(object): +class _RequestBodyProducer: def __init__(self, body): self.body = body From 6f02a8dccb95373f22bac18c08d9fda8169dcb02 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 29 Jan 2020 14:53:23 -0300 Subject: [PATCH 13/50] Add source parameter to bytes_received signal --- docs/topics/signals.rst | 12 ++++++++---- scrapy/core/downloader/handlers/http11.py | 18 +++++++++++++----- scrapy/core/downloader/handlers/s3.py | 1 + tests/test_downloader_handlers.py | 3 +++ tests/test_engine.py | 5 ++++- 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index f490911f3..3a15bf95c 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -162,11 +162,11 @@ bytes_received -------------- .. signal:: bytes_received -.. function:: bytes_received(data, request, spider) +.. function:: bytes_received(data, request, spider, source) - Sent by the HTTP 1.1 download handler when a group of bytes is - received for a specific request. This signal might be fired - multiple times for the same request. + Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is + received for a specific request. This signal might be fired multiple + times for the same request, with partial data each time. This signal does not support returning deferreds from its handlers. @@ -179,6 +179,10 @@ bytes_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.spiders.Spider` object + :param source: a string to identify which handler sent the signal + (current values could be "http11" or "s3") + :type source: :class:`str` object + spider_closed ------------- diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 6f1bd9ad6..49c9eacac 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -32,8 +32,9 @@ logger = logging.getLogger(__name__) class HTTP11DownloadHandler: lazy = False - def __init__(self, settings, crawler=None): + def __init__(self, settings, crawler=None, source="http11"): self.crawler = crawler + self.source = source self._pool = HTTPConnectionPool(reactor, persistent=True) self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self._pool._factory.noisy = False @@ -67,8 +68,8 @@ class HTTP11DownloadHandler: self._disconnect_timeout = 1 @classmethod - def from_crawler(cls, crawler): - return cls(crawler.settings, crawler) + def from_crawler(cls, crawler, **kwargs): + return cls(crawler.settings, crawler, **kwargs) def download_request(self, request, spider): """Return a deferred for the HTTP download""" @@ -79,6 +80,7 @@ class HTTP11DownloadHandler: warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), fail_on_dataloss=self._fail_on_dataloss, crawler=self.crawler, + source=self.source, ) return agent.download_request(request) @@ -275,7 +277,7 @@ class ScrapyAgent: _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, - maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None): + maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None, source=None): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress @@ -285,6 +287,7 @@ class ScrapyAgent: self._fail_on_dataloss = fail_on_dataloss self._txresponse = None self._crawler = crawler + self._source = source def _get_agent(self, request, timeout): bindaddress = request.meta.get('bindaddress') or self._bindAddress @@ -421,6 +424,7 @@ class ScrapyAgent: warnsize, fail_on_dataloss, self._crawler, + self._source, ) ) @@ -457,7 +461,9 @@ class _RequestBodyProducer: class _ResponseReader(protocol.Protocol): - def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler): + def __init__( + self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler, source + ): self._finished = finished self._txresponse = txresponse self._request = request @@ -469,6 +475,7 @@ class _ResponseReader(protocol.Protocol): self._reached_warnsize = False self._bytes_received = 0 self._crawler = crawler + self._source = source def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. @@ -483,6 +490,7 @@ class _ResponseReader(protocol.Protocol): data=bodyBytes, request=self._request, spider=self._crawler.spider, + source=self._source, ) if self._maxsize and self._bytes_received > self._maxsize: diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 40a1fa48e..2366b6394 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -73,6 +73,7 @@ class S3DownloadHandler: objcls=httpdownloadhandler, settings=settings, crawler=crawler, + source="s3", ) self._download_http = _http_handler.download_request diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 8d95d7cac..22a813647 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -730,6 +730,9 @@ class Http11ProxyTestCase(HttpProxyTestCase): class HttpDownloadHandlerMock: + def __init__(self, *args, **kwargs): + pass + def download_request(self, request, spider): return request diff --git a/tests/test_engine.py b/tests/test_engine.py index 3c5cc403b..c83a23b55 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -112,6 +112,7 @@ class CrawlerRun(object): self.itemerror = [] self.itemresp = [] self.bytes = defaultdict(lambda: list()) + self.bytes_source = set() self.signals_caught = {} self.spider_class = spider_class @@ -165,8 +166,9 @@ class CrawlerRun(object): def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) - def bytes_received(self, data, request, spider): + def bytes_received(self, data, request, spider, source): self.bytes[request].append(data) + self.bytes_source.add(source) def request_scheduled(self, request, spider): self.reqplug.append((request, spider)) @@ -279,6 +281,7 @@ class EngineTest(unittest.TestCase): def _assert_bytes_received(self): self.assertEqual(9, len(self.run.bytes)) + self.assertEqual(self.run.bytes_source, set(["http11"])) for request, data in self.run.bytes.items(): joined_data = b"".join(data) if self.run.getpath(request.url) == "/": From a64fa2f0866c10594f1e5cf00a0161f9fea1eb62 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 10 Feb 2020 10:16:05 -0300 Subject: [PATCH 14/50] Keyword arguments when creating a _ResponseReader --- scrapy/core/downloader/handlers/http11.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 49c9eacac..7a1a77b23 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -417,14 +417,14 @@ class ScrapyAgent: d = defer.Deferred(_cancel) txresponse.deliverBody( _ResponseReader( - d, - txresponse, - request, - maxsize, - warnsize, - fail_on_dataloss, - self._crawler, - self._source, + finished=d, + txresponse=txresponse, + request=request, + maxsize=maxsize, + warnsize=warnsize, + fail_on_dataloss=fail_on_dataloss, + crawler=self._crawler, + source=self._source, ) ) From 122ce6d6fb3861d99ba2f2810b2370056bae1190 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 10 Feb 2020 10:20:26 -0300 Subject: [PATCH 15/50] Check bytes are received in order (bytes_received signal) --- tests/test_engine.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index c83a23b55..0d970928b 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -12,7 +12,6 @@ module with the ``runserver`` argument:: import os import re -import string import sys from collections import defaultdict from urllib.parse import urlparse @@ -91,7 +90,8 @@ def start_test_site(debug=False): r = static.File(root_dir) r.putChild(b"redirect", util.Redirect(b"/redirected")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) - r.putChild(b"random", static.Data(string.ascii_letters.encode("utf8") * 2**14, "text/plain")) + numbers = [str(x).encode("utf8") for x in range(2**14)] + r.putChild(b"numbers", static.Data(b"".join(numbers), "text/plain")) port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") if debug: @@ -124,7 +124,7 @@ class CrawlerRun(object): self.geturl("/"), self.geturl("/redirect"), self.geturl("/redirect"), # duplicate - self.geturl("/random"), + self.geturl("/numbers"), ] for name, signal in vars(signals).items(): @@ -315,8 +315,12 @@ class EngineTest(unittest.TestCase): b" \n" b"\n" ) - elif self.run.getpath(request.url) == "/random": - self.assertTrue(len(data) > 1) # signal was fired multiple times + elif self.run.getpath(request.url) == "/numbers": + # signal was fired multiple times + self.assertTrue(len(data) > 1) + # bytes were received in order + numbers = [str(x).encode("utf8") for x in range(2**14)] + self.assertEqual(joined_data, b"".join(numbers)) def _assert_signals_caught(self): assert signals.engine_started in self.run.signals_caught From 42b4e9b3372ce3f9da57c7512b31a3c455b8a161 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 10 Feb 2020 11:23:38 -0300 Subject: [PATCH 16/50] Reword signal docs --- docs/topics/signals.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3a15bf95c..dfb87cef3 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -279,7 +279,7 @@ request_scheduled Sent when the engine schedules a :class:`~scrapy.http.Request`, to be downloaded later. - The signal does not support returning deferreds from its handlers. + This signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler :type request: :class:`~scrapy.http.Request` object @@ -296,7 +296,7 @@ request_dropped Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be downloaded later, is rejected by the scheduler. - The signal does not support returning deferreds from its handlers. + This signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler :type request: :class:`~scrapy.http.Request` object @@ -312,7 +312,7 @@ request_reached_downloader Sent when a :class:`~scrapy.http.Request` reached downloader. - The signal does not support returning deferreds from its handlers. + This signal does not support returning deferreds from its handlers. :param request: the request that reached downloader :type request: :class:`~scrapy.http.Request` object From c4a5e3f0da3e674ecb7393c0894098984d6aa571 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 6 Apr 2020 09:26:13 -0300 Subject: [PATCH 17/50] Simplify bytes_received signal Remove "source" parameter --- docs/topics/signals.rst | 6 +----- scrapy/core/downloader/handlers/http11.py | 18 +++++------------- scrapy/core/downloader/handlers/s3.py | 1 - tests/test_engine.py | 5 +---- 4 files changed, 7 insertions(+), 23 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 97be46f2a..02fa6e287 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -163,7 +163,7 @@ bytes_received -------------- .. signal:: bytes_received -.. function:: bytes_received(data, request, spider, source) +.. function:: bytes_received(data, request, spider) Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is received for a specific request. This signal might be fired multiple @@ -180,10 +180,6 @@ bytes_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.spiders.Spider` object - :param source: a string to identify which handler sent the signal - (current values could be "http11" or "s3") - :type source: :class:`str` object - spider_closed ------------- diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c350cd3c2..bda21b6b9 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -33,9 +33,8 @@ logger = logging.getLogger(__name__) class HTTP11DownloadHandler: lazy = False - def __init__(self, settings, crawler=None, source="http11"): + def __init__(self, settings, crawler=None): self._crawler = crawler - self._source = source from twisted.internet import reactor self._pool = HTTPConnectionPool(reactor, persistent=True) @@ -71,8 +70,8 @@ class HTTP11DownloadHandler: self._disconnect_timeout = 1 @classmethod - def from_crawler(cls, crawler, **kwargs): - return cls(crawler.settings, crawler, **kwargs) + def from_crawler(cls, crawler): + return cls(crawler.settings, crawler) def download_request(self, request, spider): """Return a deferred for the HTTP download""" @@ -83,7 +82,6 @@ class HTTP11DownloadHandler: warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), fail_on_dataloss=self._fail_on_dataloss, crawler=self._crawler, - source=self._source, ) return agent.download_request(request) @@ -281,7 +279,7 @@ class ScrapyAgent: _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, - maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None, source=None): + maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress @@ -291,7 +289,6 @@ class ScrapyAgent: self._fail_on_dataloss = fail_on_dataloss self._txresponse = None self._crawler = crawler - self._source = source def _get_agent(self, request, timeout): from twisted.internet import reactor @@ -430,7 +427,6 @@ class ScrapyAgent: warnsize=warnsize, fail_on_dataloss=fail_on_dataloss, crawler=self._crawler, - source=self._source, ) ) @@ -468,9 +464,7 @@ class _RequestBodyProducer: class _ResponseReader(protocol.Protocol): - def __init__( - self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler, source - ): + def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler): self._finished = finished self._txresponse = txresponse self._request = request @@ -483,7 +477,6 @@ class _ResponseReader(protocol.Protocol): self._bytes_received = 0 self._certificate = None self._crawler = crawler - self._source = source def connectionMade(self): if self._certificate is None: @@ -503,7 +496,6 @@ class _ResponseReader(protocol.Protocol): data=bodyBytes, request=self._request, spider=self._crawler.spider, - source=self._source, ) if self._maxsize and self._bytes_received > self._maxsize: diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 2366b6394..40a1fa48e 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -73,7 +73,6 @@ class S3DownloadHandler: objcls=httpdownloadhandler, settings=settings, crawler=crawler, - source="s3", ) self._download_http = _http_handler.download_request diff --git a/tests/test_engine.py b/tests/test_engine.py index 26f3163cf..acfe94f63 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -112,7 +112,6 @@ class CrawlerRun: self.itemerror = [] self.itemresp = [] self.bytes = defaultdict(lambda: list()) - self.bytes_source = set() self.signals_caught = {} self.spider_class = spider_class @@ -166,9 +165,8 @@ class CrawlerRun: def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) - def bytes_received(self, data, request, spider, source): + def bytes_received(self, data, request, spider): self.bytes[request].append(data) - self.bytes_source.add(source) def request_scheduled(self, request, spider): self.reqplug.append((request, spider)) @@ -281,7 +279,6 @@ class EngineTest(unittest.TestCase): def _assert_bytes_received(self): self.assertEqual(9, len(self.run.bytes)) - self.assertEqual(self.run.bytes_source, set(["http11"])) for request, data in self.run.bytes.items(): joined_data = b"".join(data) if self.run.getpath(request.url) == "/": From e1948b492317eb5b11550d119d91c61b74b3a37f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 4 May 2020 09:07:27 -0300 Subject: [PATCH 18/50] Add example about bytes_received signal --- docs/topics/signals.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index bc04faad5..7fe63a7b0 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -378,7 +378,9 @@ bytes_received Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is received for a specific request. This signal might be fired multiple - times for the same request, with partial data each time. + times for the same request, with partial data each time. For instance, + a possible scenario for a 25 kb response would be two signals fired + with 10 kb of data, and a final one with 5 kb of data. This signal does not support returning deferreds from its handlers. From d0bb04f08936435202488404d08c0b82f25aa1e5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 13:37:23 -0300 Subject: [PATCH 19/50] Switch to pickle protocol 4 --- scrapy/exporters.py | 2 +- scrapy/extensions/httpcache.py | 4 ++-- scrapy/extensions/spiderstate.py | 2 +- scrapy/squeues.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 0cb6cef98..349a9586b 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -250,7 +250,7 @@ class CsvItemExporter(BaseItemExporter): class PickleItemExporter(BaseItemExporter): - def __init__(self, file, protocol=2, **kwargs): + def __init__(self, file, protocol=4, **kwargs): super().__init__(**kwargs) self.file = file self.protocol = protocol diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 8546628a8..7972b58b1 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -250,7 +250,7 @@ class DbmCacheStorage: 'headers': dict(response.headers), 'body': response.body, } - self.db['%s_data' % key] = pickle.dumps(data, protocol=2) + self.db['%s_data' % key] = pickle.dumps(data, protocol=4) self.db['%s_time' % key] = str(time()) def _read_data(self, spider, request): @@ -317,7 +317,7 @@ class FilesystemCacheStorage: with self._open(os.path.join(rpath, 'meta'), 'wb') as f: f.write(to_bytes(repr(metadata))) with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f: - pickle.dump(metadata, f, protocol=2) + pickle.dump(metadata, f, protocol=4) with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f: f.write(headers_dict_to_raw(response.headers)) with self._open(os.path.join(rpath, 'response_body'), 'wb') as f: diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 2e5ff569f..bea00596e 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -26,7 +26,7 @@ class SpiderState: def spider_closed(self, spider): if self.jobdir: with open(self.statefn, 'wb') as f: - pickle.dump(spider.state, f, protocol=2) + pickle.dump(spider.state, f, protocol=4) def spider_opened(self, spider): if self.jobdir and os.path.exists(self.statefn): diff --git a/scrapy/squeues.py b/scrapy/squeues.py index d0686dac3..8d05bd0d0 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -81,7 +81,7 @@ def _scrapy_non_serialization_queue(queue_class): def _pickle_serialize(obj): try: - return pickle.dumps(obj, protocol=2) + return pickle.dumps(obj, protocol=4) # Python <= 3.4 raises pickle.PicklingError here while # 3.5 <= Python < 3.6 raises AttributeError and # Python >= 3.6 raises TypeError From b1ddd7bd7b84d8d8417228aa7392d418463c9728 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 13:44:02 -0300 Subject: [PATCH 20/50] Refactor test_squeues.py --- tests/test_squeues.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 5ad8035f7..7e997a25e 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -47,12 +47,7 @@ def nonserializable_object_test(self): self.assertRaises(ValueError, q.push, sel) -class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): - - chunksize = 100000 - - def queue(self): - return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize) +class FifoDiskQueueTestMixin: def test_serialize(self): q = self.queue() @@ -66,6 +61,13 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): test_nonserializable_object = nonserializable_object_test +class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): + chunksize = 100000 + + def queue(self): + return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize) + + class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 @@ -82,7 +84,7 @@ class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 4 -class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest): +class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): chunksize = 100000 @@ -133,10 +135,7 @@ class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 4 -class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): - - def queue(self): - return MarshalLifoDiskQueue(self.qpath) +class LifoDiskQueueTestMixin: def test_serialize(self): q = self.queue() @@ -150,7 +149,13 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): test_nonserializable_object = nonserializable_object_test -class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest): +class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): + + def queue(self): + return MarshalLifoDiskQueue(self.qpath) + + +class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def queue(self): return PickleLifoDiskQueue(self.qpath) From 93436f9d3a67cd8abe3b321321c2d36d94f75b8b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 14:05:27 -0300 Subject: [PATCH 21/50] Chain pickling exception, test_squeues.py updates --- scrapy/squeues.py | 7 +++---- tests/test_squeues.py | 28 ++++++++++++++-------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 8d05bd0d0..c7ad4d53d 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -82,11 +82,10 @@ def _scrapy_non_serialization_queue(queue_class): def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=4) - # Python <= 3.4 raises pickle.PicklingError here while - # 3.5 <= Python < 3.6 raises AttributeError and - # Python >= 3.6 raises TypeError + # Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s) + # TypeError is raised from parsel.Selector except (pickle.PicklingError, AttributeError, TypeError) as e: - raise ValueError(str(e)) + raise ValueError(str(e)) from e PickleFifoDiskQueueNonRequest = _serializable_queue( diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 7e997a25e..a20d242f4 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -28,20 +28,7 @@ class TestLoader(ItemLoader): def nonserializable_object_test(self): q = self.queue() - try: - pickle.dumps(lambda x: x) - except Exception: - # Trigger Twisted bug #7989 - import twisted.persisted.styles # NOQA - self.assertRaises(ValueError, q.push, lambda x: x) - else: - # Use a different unpickleable object - class A: - pass - - a = A() - a.__reduce__ = a.__reduce_ex__ = None - self.assertRaises(ValueError, q.push, a) + self.assertRaises(ValueError, q.push, lambda x: x) # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) sel = Selector(text='

some text

') self.assertRaises(ValueError, q.push, sel) @@ -118,6 +105,19 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): self.assertEqual(r.url, r2.url) assert r2.meta['request'] is r2 + def test_non_pickable_object(self): + q = self.queue() + try: + q.push(lambda x: x) + except ValueError as exc: + self.assertIsInstance(exc.__context__, AttributeError) + + sel = Selector(text='

some text

') + try: + q.push(sel) + except ValueError as exc: + self.assertIsInstance(exc.__context__, TypeError) + class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 1 From 0e382c816024baffca05b0da29def95f723d27fd Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 14:09:10 -0300 Subject: [PATCH 22/50] Remove unused import --- tests/test_squeues.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index a20d242f4..51c0c028a 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,5 +1,3 @@ -import pickle - from queuelib.tests import test_queue as t from scrapy.squeues import ( MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue, From 4c12a234ae65d49678a9840708ff5e7b9d6dcecc Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 16:10:21 -0300 Subject: [PATCH 23/50] Flake8: Remove E126 --- pytest.ini | 34 +++++++++++----------- scrapy/core/downloader/contextfactory.py | 10 +++---- scrapy/core/downloader/handlers/s3.py | 13 +++++---- scrapy/core/downloader/webclient.py | 4 +-- scrapy/downloadermiddlewares/retry.py | 12 ++++++-- scrapy/spiderloader.py | 25 +++++++++------- tests/test_downloadermiddleware_cookies.py | 10 ++++--- tests/test_downloadermiddleware_retry.py | 12 ++++++-- tests/test_http_request.py | 32 +++++++++++--------- tests/test_pipeline_crawl.py | 4 +-- tests/test_webclient.py | 13 +++++---- 11 files changed, 96 insertions(+), 73 deletions(-) diff --git a/pytest.ini b/pytest.ini index 998633d54..1570a3a75 100644 --- a/pytest.ini +++ b/pytest.ini @@ -50,26 +50,26 @@ flake8-ignore = scrapy/core/engine.py E501 E128 scrapy/core/scheduler.py E501 scrapy/core/scraper.py E501 E128 - scrapy/core/spidermw.py E501 E126 + scrapy/core/spidermw.py E501 scrapy/core/downloader/__init__.py E501 - scrapy/core/downloader/contextfactory.py E501 E128 E126 + scrapy/core/downloader/contextfactory.py E501 E128 scrapy/core/downloader/middleware.py E501 scrapy/core/downloader/tls.py E501 - scrapy/core/downloader/webclient.py E501 E128 E126 + scrapy/core/downloader/webclient.py E501 E128 scrapy/core/downloader/handlers/__init__.py E501 scrapy/core/downloader/handlers/ftp.py E501 E128 scrapy/core/downloader/handlers/http10.py E501 scrapy/core/downloader/handlers/http11.py E501 - scrapy/core/downloader/handlers/s3.py E501 E128 E126 + scrapy/core/downloader/handlers/s3.py E501 E128 # scrapy/downloadermiddlewares scrapy/downloadermiddlewares/ajaxcrawl.py E501 scrapy/downloadermiddlewares/decompression.py E501 scrapy/downloadermiddlewares/defaultheaders.py E501 - scrapy/downloadermiddlewares/httpcache.py E501 E126 + scrapy/downloadermiddlewares/httpcache.py E501 scrapy/downloadermiddlewares/httpcompression.py E501 E128 scrapy/downloadermiddlewares/httpproxy.py E501 scrapy/downloadermiddlewares/redirect.py E501 - scrapy/downloadermiddlewares/retry.py E501 E126 + scrapy/downloadermiddlewares/retry.py E501 scrapy/downloadermiddlewares/robotstxt.py E501 scrapy/downloadermiddlewares/stats.py E501 # scrapy/extensions @@ -164,12 +164,12 @@ flake8-ignore = scrapy/robotstxt.py E501 scrapy/shell.py E501 scrapy/signalmanager.py E501 - scrapy/spiderloader.py F841 E501 E126 + scrapy/spiderloader.py F841 E501 scrapy/squeues.py E128 scrapy/statscollectors.py E501 # tests tests/__init__.py E402 E501 - tests/mockserver.py E501 E126 + tests/mockserver.py E501 tests/pipelines.py F841 tests/spiders.py E501 tests/test_closespider.py E501 @@ -181,18 +181,18 @@ flake8-ignore = tests/test_crawl.py E501 E741 tests/test_crawler.py F841 E501 tests/test_dependencies.py F841 E501 - tests/test_downloader_handlers.py E128 E501 E126 + tests/test_downloader_handlers.py E128 E501 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 - tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126 + tests/test_downloadermiddleware_cookies.py E741 E501 E128 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 tests/test_downloadermiddleware_httpcache.py E501 - tests/test_downloadermiddleware_httpcompression.py E501 E126 + tests/test_downloadermiddleware_httpcompression.py E501 tests/test_downloadermiddleware_decompression.py E501 tests/test_downloadermiddleware_httpproxy.py E501 E128 tests/test_downloadermiddleware_redirect.py E501 E128 - tests/test_downloadermiddleware_retry.py E501 E128 E126 + tests/test_downloadermiddleware_retry.py E501 E128 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 tests/test_dupefilters.py E501 E741 E128 @@ -202,7 +202,7 @@ flake8-ignore = tests/test_feedexport.py E501 F841 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 - tests/test_http_request.py E402 E501 E128 E128 E126 + tests/test_http_request.py E402 E501 E128 E128 tests/test_http_response.py E501 E128 tests/test_item.py E128 F841 tests/test_link.py E501 @@ -211,7 +211,7 @@ flake8-ignore = tests/test_logformatter.py E128 E501 tests/test_mail.py E128 E501 tests/test_middleware.py E501 E128 - tests/test_pipeline_crawl.py E501 E128 E126 + tests/test_pipeline_crawl.py E501 E128 tests/test_pipeline_files.py E501 tests/test_pipeline_images.py F841 E501 tests/test_pipeline_media.py E501 E741 E128 @@ -219,7 +219,7 @@ flake8-ignore = tests/test_request_cb_kwargs.py E501 tests/test_responsetypes.py E501 tests/test_robotstxt_interface.py E501 E501 - tests/test_scheduler.py E501 E126 + tests/test_scheduler.py E501 tests/test_selector.py E501 tests/test_spider.py E501 tests/test_spidermiddleware.py E501 @@ -243,8 +243,8 @@ flake8-ignore = tests/test_utils_response.py E501 tests/test_utils_signal.py E741 F841 tests/test_utils_sitemap.py E128 E501 - tests/test_utils_url.py E501 E501 E126 - tests/test_webclient.py E501 E128 E402 E126 + tests/test_utils_url.py E501 E501 + tests/test_webclient.py E501 E128 E402 tests/test_cmdline/__init__.py E501 tests/test_settings/__init__.py E501 E128 tests/test_spiderloader/__init__.py E128 E501 diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 6e023ebcc..ab73e12c8 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -86,8 +86,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): # # This means that a website like https://www.cacert.org will be rejected # by default, since CAcert.org CA certificate is seldom shipped. - return optionsForClientTLS(hostname.decode("ascii"), - trustRoot=platformTrust(), - extraCertificateOptions={ - 'method': self._ssl_method, - }) + return optionsForClientTLS( + hostname=hostname.decode("ascii"), + trustRoot=platformTrust(), + extraCertificateOptions={'method': self._ssl_method}, + ) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 40a1fa48e..8f63ad974 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -100,11 +100,12 @@ class S3DownloadHandler: url=url, headers=awsrequest.headers.items()) else: signed_headers = self.conn.make_request( - method=request.method, - bucket=bucket, - key=unquote(p.path), - query_args=unquote(p.query), - headers=request.headers, - data=request.body) + method=request.method, + bucket=bucket, + key=unquote(p.path), + query_args=unquote(p.query), + headers=request.headers, + data=request.body, + ) request = request.replace(url=url, headers=signed_headers) return self._download_http(request, spider) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index a90a77b2b..355045d74 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient): self.transport.stopProducing() self.factory.noPage( - defer.TimeoutError("Getting %s took longer than %s seconds." % - (self.factory.url, self.factory.timeout))) + defer.TimeoutError("Getting %s took longer than %s seconds." + % (self.factory.url, self.factory.timeout))) class ScrapyHTTPClientFactory(HTTPClientFactory): diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index bbf5fca05..6d11af5b2 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -12,9 +12,15 @@ once the spider has finished crawling all regular (non failed) pages. import logging from twisted.internet import defer -from twisted.internet.error import TimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost, TCPTimedOutError +from twisted.internet.error import ( + ConnectError, + ConnectionDone, + ConnectionLost, + ConnectionRefusedError, + DNSLookupError, + TCPTimedOutError, + TimeoutError, +) from twisted.web.client import ResponseFailed from scrapy.exceptions import NotConfigured diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 3be5aaec5..8dc89c2e9 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -24,15 +24,17 @@ class SpiderLoader: self._load_all_spiders() def _check_name_duplicates(self): - dupes = ["\n".join(" {cls} named {name!r} (in {module})".format( - module=mod, cls=cls, name=name) - for (mod, cls) in locations) - for name, locations in self._found.items() - if len(locations) > 1] + dupes = [] + for name, locations in self._found.items(): + dupes.extend([ + " {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name) + for mod, cls in locations + ]) + if dupes: + dupes_string = "\n\n".join(dupes) msg = ("There are several spiders with the same name:\n\n" - "{}\n\n This can cause unexpected behavior.".format( - "\n\n".join(dupes))) + "{}\n\n This can cause unexpected behavior.".format(dupes_string)) warnings.warn(msg, UserWarning) def _load_spiders(self, module): @@ -45,11 +47,12 @@ class SpiderLoader: try: for module in walk_modules(name): self._load_spiders(module) - except ImportError as e: + except ImportError: if self.warn_only: - msg = ("\n{tb}Could not load spiders from module '{modname}'. " - "See above traceback for details.".format( - modname=name, tb=traceback.format_exc())) + msg = ( + "\n{tb}Could not load spiders from module '{modname}'. " + "See above traceback for details.".format(modname=name, tb=traceback.format_exc()) + ) warnings.warn(msg, RuntimeWarning) else: raise diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index b686a14d6..f86c50f50 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -139,10 +139,12 @@ class CookiesMiddlewareTest(TestCase): def test_complex_cookies(self): # merge some cookies into jar - cookies = [{'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'}, - {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'}, - {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'}, - {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}] + cookies = [ + {'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'}, + {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'}, + {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'}, + {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}, + ] req = Request('http://scrapytest.org/', cookies=cookies) self.mw.process_request(req, self.spider) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 9c989977e..e118750e3 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,8 +1,14 @@ import unittest from twisted.internet import defer -from twisted.internet.error import TimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost, TCPTimedOutError +from twisted.internet.error import ( + ConnectError, + ConnectionDone, + ConnectionLost, + ConnectionRefusedError, + DNSLookupError, + TCPTimedOutError, + TimeoutError, +) from twisted.web.client import ResponseFailed from scrapy.downloadermiddlewares.retry import RetryMiddleware diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 3b6d119a9..77da15ce6 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -502,11 +502,13 @@ class FormRequestTest(RequestTest): def test_from_response_duplicate_form_key(self): response = _buildresponse( - '
', - url='http://www.example.com') - req = self.request_class.from_response(response, - method='GET', - formdata=(('foo', 'bar'), ('foo', 'baz'))) + '
', + url='http://www.example.com') + req = self.request_class.from_response( + response=response, + method='GET', + formdata=(('foo', 'bar'), ('foo', 'baz')), + ) self.assertEqual(urlparse(req.url).hostname, 'www.example.com') self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz') @@ -530,9 +532,11 @@ class FormRequestTest(RequestTest): """) - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}, - headers={"Accept-Encoding": "gzip,deflate"}) + req = self.request_class.from_response( + response=response, + formdata={'one': ['two', 'three'], 'six': 'seven'}, + headers={"Accept-Encoding": "gzip,deflate"}, + ) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.headers['Accept-Encoding'], b'gzip,deflate') @@ -580,9 +584,9 @@ class FormRequestTest(RequestTest): def test_from_response_override_method(self): response = _buildresponse( - ''' -
- ''') + ''' +
+ ''') request = FormRequest.from_response(response) self.assertEqual(request.method, 'GET') request = FormRequest.from_response(response, method='POST') @@ -590,9 +594,9 @@ class FormRequestTest(RequestTest): def test_from_response_override_url(self): response = _buildresponse( - ''' -
- ''') + ''' +
+ ''') request = FormRequest.from_response(response) self.assertEqual(request.url, 'http://example.com/app') request = FormRequest.from_response(response, url='http://foo.bar/absolute') diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 962c33144..24c516473 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -44,9 +44,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): name = 'redirectedmedia' def _process_url(self, url): - return add_or_replace_parameter( - self.mockserver.url('/redirect-to'), - 'goto', url) + return add_or_replace_parameter(self.mockserver.url('/redirect-to'), 'goto', url) class FileDownloadCrawlTestCase(TestCase): diff --git a/tests/test_webclient.py b/tests/test_webclient.py index b657c7ab6..307fadb5c 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -18,6 +18,14 @@ except ImportError: from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory from twisted.internet.defer import inlineCallbacks +from twisted.web.test.test_webclient import ( + ForeverTakingResource, + ErrorResource, + NoLengthResource, + HostHeaderResource, + PayloadResource, + BrokenDownloadResource, +) from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory @@ -202,11 +210,6 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): Headers({'Hello': ['World'], 'Foo': ['Bar']})) -from twisted.web.test.test_webclient import ForeverTakingResource, \ - ErrorResource, NoLengthResource, HostHeaderResource, \ - PayloadResource, BrokenDownloadResource - - class EncodingResource(resource.Resource): out_encoding = 'cp1251' From 2af34873cc385e47cc962903ff3d6a13c9847615 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 17:10:10 -0300 Subject: [PATCH 24/50] Flake8: Remove E128 (commands module) --- pytest.ini | 16 ++++++++-------- scrapy/commands/__init__.py | 12 ++++++------ scrapy/commands/fetch.py | 4 ++-- scrapy/commands/genspider.py | 10 +++++----- scrapy/commands/parse.py | 24 ++++++++++++------------ scrapy/commands/settings.py | 10 +++++----- scrapy/commands/shell.py | 8 ++++---- scrapy/commands/startproject.py | 6 ++---- scrapy/commands/version.py | 2 +- 9 files changed, 45 insertions(+), 47 deletions(-) diff --git a/pytest.ini b/pytest.ini index 1570a3a75..d96ac9684 100644 --- a/pytest.ini +++ b/pytest.ini @@ -31,18 +31,18 @@ flake8-ignore = extras/qps-bench-server.py E501 extras/qpsclient.py E501 E501 # scrapy/commands - scrapy/commands/__init__.py E128 E501 + scrapy/commands/__init__.py E501 scrapy/commands/check.py E501 scrapy/commands/crawl.py E501 scrapy/commands/edit.py E501 - scrapy/commands/fetch.py E501 E128 - scrapy/commands/genspider.py E128 E501 - scrapy/commands/parse.py E128 E501 + scrapy/commands/fetch.py E501 + scrapy/commands/genspider.py E501 + scrapy/commands/parse.py E501 scrapy/commands/runspider.py E501 - scrapy/commands/settings.py E128 - scrapy/commands/shell.py E128 E501 - scrapy/commands/startproject.py E501 E128 - scrapy/commands/version.py E501 E128 + scrapy/commands/settings.py E501 + scrapy/commands/shell.py E501 + scrapy/commands/startproject.py E501 + scrapy/commands/version.py E501 # scrapy/contracts scrapy/contracts/__init__.py E501 scrapy/contracts/default.py E128 diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 9f8e6986a..97f818cb2 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -59,17 +59,17 @@ class ScrapyCommand: """ group = OptionGroup(parser, "Global Options") group.add_option("--logfile", metavar="FILE", - help="log file. if omitted stderr will be used") + help="log file. if omitted stderr will be used") group.add_option("-L", "--loglevel", metavar="LEVEL", default=None, - help="log level (default: %s)" % self.settings['LOG_LEVEL']) + help="log level (default: %s)" % self.settings['LOG_LEVEL']) group.add_option("--nolog", action="store_true", - help="disable logging completely") + help="disable logging completely") group.add_option("--profile", metavar="FILE", default=None, - help="write python cProfile stats to FILE") + help="write python cProfile stats to FILE") group.add_option("--pidfile", metavar="FILE", - help="write process ID to FILE") + help="write process ID to FILE") group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", - help="set/override setting (may be repeated)") + help="set/override setting (may be repeated)") group.add_option("--pdb", action="store_true", help="enable pdb on failure") parser.add_option_group(group) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 506d1f1b7..063195f50 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -27,8 +27,8 @@ class Command(ScrapyCommand): parser.add_option("--spider", dest="spider", help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", help="print response HTTP headers instead of body") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", - default=False, help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2e837abed..abf3b7a5c 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -36,15 +36,15 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("-l", "--list", dest="list", action="store_true", - help="List available templates") + help="List available templates") parser.add_option("-e", "--edit", dest="edit", action="store_true", - help="Edit spider after creating it") + help="Edit spider after creating it") parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE", - help="Dump template to standard output") + help="Dump template to standard output") parser.add_option("-t", "--template", dest="template", default="basic", - help="Uses a custom template.") + help="Uses a custom template.") parser.add_option("--force", dest="force", action="store_true", - help="If the spider already exists, overwrite it with the template") + help="If the spider already exists, overwrite it with the template") def run(self, args, opts): if opts.list: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 1cefed106..d9ab2126a 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -33,29 +33,29 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("--spider", dest="spider", default=None, - help="use this spider without looking for one") + help="use this spider without looking for one") parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") + help="set spider argument (may be repeated)") parser.add_option("--pipelines", action="store_true", - help="process items through pipelines") + help="process items through pipelines") parser.add_option("--nolinks", dest="nolinks", action="store_true", - help="don't show links to follow (extracted requests)") + help="don't show links to follow (extracted requests)") parser.add_option("--noitems", dest="noitems", action="store_true", - help="don't show scraped items") + help="don't show scraped items") parser.add_option("--nocolour", dest="nocolour", action="store_true", - help="avoid using pygments to colorize the output") + help="avoid using pygments to colorize the output") parser.add_option("-r", "--rules", dest="rules", action="store_true", - help="use CrawlSpider rules to discover the callback") + help="use CrawlSpider rules to discover the callback") parser.add_option("-c", "--callback", dest="callback", - help="use this callback for parsing, instead looking for a callback") + help="use this callback for parsing, instead looking for a callback") parser.add_option("-m", "--meta", dest="meta", - help="inject extra meta into the Request, it must be a valid raw json string") + help="inject extra meta into the Request, it must be a valid raw json string") parser.add_option("--cbkwargs", dest="cbkwargs", - help="inject extra callback kwargs into the Request, it must be a valid raw json string") + help="inject extra callback kwargs into the Request, it must be a valid raw json string") parser.add_option("-d", "--depth", dest="depth", type="int", default=1, - help="maximum depth for parsing requests [default: %default]") + help="maximum depth for parsing requests [default: %default]") parser.add_option("-v", "--verbose", dest="verbose", action="store_true", - help="print each depth level one by one") + help="print each depth level one by one") @property def max_level(self): diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 603bafb9f..8d49e440f 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -19,15 +19,15 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("--get", dest="get", metavar="SETTING", - help="print raw setting value") + help="print raw setting value") parser.add_option("--getbool", dest="getbool", metavar="SETTING", - help="print setting value, interpreted as a boolean") + help="print setting value, interpreted as a boolean") parser.add_option("--getint", dest="getint", metavar="SETTING", - help="print setting value, interpreted as an integer") + help="print setting value, interpreted as an integer") parser.add_option("--getfloat", dest="getfloat", metavar="SETTING", - help="print setting value, interpreted as a float") + help="print setting value, interpreted as a float") parser.add_option("--getlist", dest="getlist", metavar="SETTING", - help="print setting value, interpreted as a list") + help="print setting value, interpreted as a list") def run(self, args, opts): settings = self.crawler_process.settings diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 5946f21e8..d1944df3d 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -34,11 +34,11 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("-c", dest="code", - help="evaluate the code in the shell, print the result and exit") + help="evaluate the code in the shell, print the result and exit") parser.add_option("--spider", dest="spider", - help="use this spider") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", - default=False, help="do not handle HTTP 3xx status codes and print response as-is") + help="use this spider") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index b123e5c84..ebe3a9c2c 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -102,10 +102,8 @@ class Command(ScrapyCommand): move(join(project_dir, 'module'), join(project_dir, project_name)) for paths in TEMPLATES_TO_RENDER: path = join(*paths) - tplfile = join(project_dir, - string.Template(path).substitute(project_name=project_name)) - render_templatefile(tplfile, project_name=project_name, - ProjectName=string_camelcase(project_name)) + tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name)) + render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) print("New Scrapy project '%s', using template directory '%s', " "created in:" % (project_name, self.templates_dir)) print(" %s\n" % abspath(project_dir)) diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 1516c5997..d0ea72a67 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -17,7 +17,7 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("--verbose", "-v", dest="verbose", action="store_true", - help="also display twisted/python/platform info (useful for bug reports)") + help="also display twisted/python/platform info (useful for bug reports)") def run(self, args, opts): if opts.verbose: From 7383b2b428b08e84d6e3e40e6229c892e5802f7a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 17:27:44 -0300 Subject: [PATCH 25/50] Flake8: Remove E128 (core module) --- pytest.ini | 12 ++++++------ scrapy/core/downloader/contextfactory.py | 11 ++++++----- scrapy/core/downloader/handlers/ftp.py | 7 +++---- scrapy/core/engine.py | 13 +++++-------- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/pytest.ini b/pytest.ini index d96ac9684..7c6ee07d6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -47,20 +47,20 @@ flake8-ignore = scrapy/contracts/__init__.py E501 scrapy/contracts/default.py E128 # scrapy/core - scrapy/core/engine.py E501 E128 + scrapy/core/engine.py E501 scrapy/core/scheduler.py E501 - scrapy/core/scraper.py E501 E128 + scrapy/core/scraper.py E501 scrapy/core/spidermw.py E501 scrapy/core/downloader/__init__.py E501 - scrapy/core/downloader/contextfactory.py E501 E128 + scrapy/core/downloader/contextfactory.py E501 scrapy/core/downloader/middleware.py E501 scrapy/core/downloader/tls.py E501 - scrapy/core/downloader/webclient.py E501 E128 + scrapy/core/downloader/webclient.py E501 scrapy/core/downloader/handlers/__init__.py E501 - scrapy/core/downloader/handlers/ftp.py E501 E128 + scrapy/core/downloader/handlers/ftp.py E501 scrapy/core/downloader/handlers/http10.py E501 scrapy/core/downloader/handlers/http11.py E501 - scrapy/core/downloader/handlers/s3.py E501 E128 + scrapy/core/downloader/handlers/s3.py E501 # scrapy/downloadermiddlewares scrapy/downloadermiddlewares/ajaxcrawl.py E501 scrapy/downloadermiddlewares/decompression.py E501 diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index ab73e12c8..452242d47 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -46,11 +46,12 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # # * getattr() for `_ssl_method` attribute for context factories # not calling super(..., self).__init__ - return CertificateOptions(verify=False, - method=getattr(self, 'method', - getattr(self, '_ssl_method', None)), - fixBrokenPeers=True, - acceptableCiphers=self.tls_ciphers) + return CertificateOptions( + verify=False, + method=getattr(self, 'method', getattr(self, '_ssl_method', None)), + fixBrokenPeers=True, + acceptableCiphers=self.tls_ciphers, + ) # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 94b55c347..3ef129587 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -86,10 +86,9 @@ class FTPDownloadHandler: password = request.meta.get("ftp_password", self.default_password) passive_mode = 1 if bool(request.meta.get("ftp_passive", self.passive_mode)) else 0 - creator = ClientCreator(reactor, FTPClient, user, password, - passive=passive_mode) - return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient, - request, unquote(parsed_url.path)) + creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode) + dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) + return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path)) def gotClient(self, client, request, filepath): self.client = client diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 324d21716..de0da4b70 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -217,11 +217,9 @@ class ExecutionEngine: self.slot.nextcall.schedule() def schedule(self, request, spider): - self.signals.send_catch_log(signal=signals.request_scheduled, - request=request, spider=spider) + self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider) if not self.slot.scheduler.enqueue_request(request): - self.signals.send_catch_log(signal=signals.request_dropped, - request=request, spider=spider) + self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider) def download(self, request, spider): d = self._download(request, spider) @@ -247,8 +245,8 @@ class ExecutionEngine: logkws = self.logformatter.crawled(request, response, spider) if logkws is not None: logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) - self.signals.send_catch_log(signal=signals.response_received, - response=response, request=request, spider=spider) + self.signals.send_catch_log(signals.response_received, + response=response, request=request, spider=spider) return response def _on_complete(_): @@ -286,8 +284,7 @@ class ExecutionEngine: next loop and this function is guaranteed to be called (at least) once again for this spider. """ - res = self.signals.send_catch_log(signal=signals.spider_idle, - spider=spider, dont_log=DontCloseSpider) + res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider) if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res): return From a72f5aadcf0b7ee8ef77f43602aecf34ffea7fd6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 17:38:39 -0300 Subject: [PATCH 26/50] Flake8: Remove E128 (utils module) --- pytest.ini | 10 +++++----- scrapy/utils/defer.py | 7 +++++-- scrapy/utils/log.py | 10 ++++++---- scrapy/utils/response.py | 7 +++---- scrapy/utils/signal.py | 6 ++---- scrapy/utils/url.py | 3 +-- 6 files changed, 22 insertions(+), 21 deletions(-) diff --git a/pytest.ini b/pytest.ini index 7c6ee07d6..63a3051cf 100644 --- a/pytest.ini +++ b/pytest.ini @@ -123,13 +123,13 @@ flake8-ignore = scrapy/utils/conf.py E402 E501 scrapy/utils/datatypes.py E501 scrapy/utils/decorators.py E501 - scrapy/utils/defer.py E501 E128 + scrapy/utils/defer.py E501 scrapy/utils/deprecate.py E501 scrapy/utils/gz.py E501 scrapy/utils/http.py F403 scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 - scrapy/utils/log.py E128 E501 + scrapy/utils/log.py E501 scrapy/utils/markup.py F403 scrapy/utils/misc.py E501 scrapy/utils/multipart.py F403 @@ -138,13 +138,13 @@ flake8-ignore = scrapy/utils/reactor.py E501 scrapy/utils/reqser.py E501 scrapy/utils/request.py E501 - scrapy/utils/response.py E501 E128 - scrapy/utils/signal.py E501 E128 + scrapy/utils/response.py E501 + scrapy/utils/signal.py E501 scrapy/utils/sitemap.py E501 scrapy/utils/spider.py E501 scrapy/utils/ssl.py E501 scrapy/utils/test.py E501 - scrapy/utils/url.py E501 F403 E128 F405 + scrapy/utils/url.py E501 F403 F405 # scrapy scrapy/__init__.py E402 E501 scrapy/cmdline.py E501 diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 34b8d9774..a3950db75 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -88,8 +88,11 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw): """Return a Deferred built by chaining the given callbacks and errbacks""" d = defer.Deferred() for cb, eb in zip(callbacks, errbacks): - d.addCallbacks(cb, eb, callbackArgs=a, callbackKeywords=kw, - errbackArgs=a, errbackKeywords=kw) + d.addCallbacks( + callback=cb, errback=eb, + callbackArgs=a, callbackKeywords=kw, + errbackArgs=a, errbackKeywords=kw, + ) if isinstance(input, failure.Failure): d.errback(input) else: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 5998dc33b..83c359bd4 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -144,10 +144,12 @@ def _get_handler(settings): def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) - logger.info("Versions: %(versions)s", - {'versions': ", ".join("%s %s" % (name, version) - for name, version in scrapy_components_versions() - if name != "Scrapy")}) + versions = [ + "%s %s" % (name, version) + for name, version in scrapy_components_versions() + if name != "Scrapy" + ] + logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 29fdaaf2c..edbc0db25 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -19,8 +19,7 @@ def get_base_url(response): """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: text = response.text[0:4096] - _baseurl_cache[response] = html.get_base_url(text, response.url, - response.encoding) + _baseurl_cache[response] = html.get_base_url(text, response.url, response.encoding) return _baseurl_cache[response] @@ -31,8 +30,8 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')): """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] - _metaref_cache[response] = html.get_meta_refresh(text, response.url, - response.encoding, ignore_tags=ignore_tags) + _metaref_cache[response] = html.get_meta_refresh( + text, response.url, response.encoding, ignore_tags=ignore_tags) return _metaref_cache[response] diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 60c561da6..a311e9257 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -28,8 +28,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): responses = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): try: - response = robustApply(receiver, signal=signal, sender=sender, - *arguments, **named) + response = robustApply(receiver, signal=signal, sender=sender, *arguments, **named) if isinstance(response, Deferred): logger.error("Cannot return deferreds from signal handler: %(receiver)s", {'receiver': receiver}, extra={'spider': spider}) @@ -63,8 +62,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): spider = named.get('spider', None) dfds = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, - *arguments, **named) + d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, *arguments, **named) d.addErrback(logerror, receiver) d.addBoth(lambda result: (receiver, result)) dfds.append(d) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index c9abb12d5..955b63d4b 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -27,8 +27,7 @@ def url_is_from_any_domain(url, domains): def url_is_from_spider(url, spider): """Return True if the url belongs to the given spider""" - return url_is_from_any_domain(url, - [spider.name] + list(getattr(spider, 'allowed_domains', []))) + return url_is_from_any_domain(url, [spider.name] + list(getattr(spider, 'allowed_domains', []))) def url_has_any_extension(url, extensions): From 4171b3f663aea5706ba6acf15f64e92be0e34f2a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 18:28:12 -0300 Subject: [PATCH 27/50] Flake8: Remove E128 (extensions module) --- pytest.ini | 6 +++--- scrapy/extensions/httpcache.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pytest.ini b/pytest.ini index 63a3051cf..1af3ea0d1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -73,10 +73,10 @@ flake8-ignore = scrapy/downloadermiddlewares/robotstxt.py E501 scrapy/downloadermiddlewares/stats.py E501 # scrapy/extensions - scrapy/extensions/closespider.py E501 E128 + scrapy/extensions/closespider.py E501 scrapy/extensions/corestats.py E501 - scrapy/extensions/feedexport.py E128 E501 - scrapy/extensions/httpcache.py E128 E501 + scrapy/extensions/feedexport.py E501 + scrapy/extensions/httpcache.py E501 scrapy/extensions/memdebug.py E501 scrapy/extensions/spiderstate.py E501 scrapy/extensions/telnet.py E501 diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 8546628a8..6289efec0 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -46,9 +46,10 @@ class RFC2616Policy: def __init__(self, settings): self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE') self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') - self.ignore_response_cache_controls = [to_bytes(cc) for cc in - settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')] self._cc_parsed = WeakKeyDictionary() + self.ignore_response_cache_controls = [ + to_bytes(cc) for cc in settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS') + ] def _parse_cachecontrol(self, r): if r not in self._cc_parsed: From 2fbbca56fb1e3bba98bf6c4c84591121c4c1da57 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 18:34:40 -0300 Subject: [PATCH 28/50] pytest.ini: remove unnecessary E128 lines --- pytest.ini | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pytest.ini b/pytest.ini index 1af3ea0d1..c0d74f7d9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -45,7 +45,6 @@ flake8-ignore = scrapy/commands/version.py E501 # scrapy/contracts scrapy/contracts/__init__.py E501 - scrapy/contracts/default.py E128 # scrapy/core scrapy/core/engine.py E501 scrapy/core/scheduler.py E501 @@ -66,7 +65,7 @@ flake8-ignore = scrapy/downloadermiddlewares/decompression.py E501 scrapy/downloadermiddlewares/defaultheaders.py E501 scrapy/downloadermiddlewares/httpcache.py E501 - scrapy/downloadermiddlewares/httpcompression.py E501 E128 + scrapy/downloadermiddlewares/httpcompression.py E501 scrapy/downloadermiddlewares/httpproxy.py E501 scrapy/downloadermiddlewares/redirect.py E501 scrapy/downloadermiddlewares/retry.py E501 @@ -87,13 +86,13 @@ flake8-ignore = scrapy/http/request/__init__.py E501 scrapy/http/request/form.py E501 scrapy/http/request/json_request.py E501 - scrapy/http/response/__init__.py E501 E128 - scrapy/http/response/text.py E501 E128 + scrapy/http/response/__init__.py E501 + scrapy/http/response/text.py E501 # scrapy/linkextractors scrapy/linkextractors/__init__.py E501 E402 scrapy/linkextractors/lxmlhtml.py E501 # scrapy/loader - scrapy/loader/__init__.py E501 E128 + scrapy/loader/__init__.py E501 scrapy/loader/processors.py E501 # scrapy/pipelines scrapy/pipelines/__init__.py E501 @@ -157,7 +156,7 @@ flake8-ignore = scrapy/link.py E501 scrapy/logformatter.py E501 scrapy/mail.py E402 E128 E501 - scrapy/middleware.py E128 E501 + scrapy/middleware.py E501 scrapy/pqueues.py E501 scrapy/resolver.py E501 scrapy/responsetypes.py E128 E501 @@ -165,7 +164,6 @@ flake8-ignore = scrapy/shell.py E501 scrapy/signalmanager.py E501 scrapy/spiderloader.py F841 E501 - scrapy/squeues.py E128 scrapy/statscollectors.py E501 # tests tests/__init__.py E402 E501 From 2851f641f2c105350948a34be55aa00dfed9933a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 18:35:50 -0300 Subject: [PATCH 29/50] Flake8: Remove E128 (item, mail, responsetypes) --- pytest.ini | 6 +++--- scrapy/item.py | 6 ++---- scrapy/mail.py | 32 ++++++++++++++++++++++---------- scrapy/responsetypes.py | 6 +++--- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/pytest.ini b/pytest.ini index c0d74f7d9..0a5a172b3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -152,14 +152,14 @@ flake8-ignore = scrapy/exceptions.py E501 scrapy/exporters.py E501 scrapy/interfaces.py E501 - scrapy/item.py E501 E128 + scrapy/item.py E501 scrapy/link.py E501 scrapy/logformatter.py E501 - scrapy/mail.py E402 E128 E501 + scrapy/mail.py E402 E501 scrapy/middleware.py E501 scrapy/pqueues.py E501 scrapy/resolver.py E501 - scrapy/responsetypes.py E128 E501 + scrapy/responsetypes.py E501 scrapy/robotstxt.py E501 scrapy/shell.py E501 scrapy/signalmanager.py E501 diff --git a/scrapy/item.py b/scrapy/item.py index 748368932..b75d04404 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -86,8 +86,7 @@ class DictItem(MutableMapping, BaseItem): if key in self.fields: self._values[key] = value else: - raise KeyError("%s does not support field: %s" % - (self.__class__.__name__, key)) + raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key)) def __delitem__(self, key): del self._values[key] @@ -99,8 +98,7 @@ class DictItem(MutableMapping, BaseItem): def __setattr__(self, name, value): if not name.startswith('_'): - raise AttributeError("Use item[%r] = %r to set field value" % - (name, value)) + raise AttributeError("Use item[%r] = %r to set field value" % (name, value)) super(DictItem, self).__setattr__(name, value) def __len__(self): diff --git a/scrapy/mail.py b/scrapy/mail.py index 9d7896ef6..7d7a2c435 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -28,8 +28,10 @@ def _to_bytes_or_none(text): class MailSender: - def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost', - smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): + def __init__( + self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None, + smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False + ): self.smtphost = smtphost self.smtpport = smtpport self.smtpuser = _to_bytes_or_none(smtpuser) @@ -41,9 +43,15 @@ class MailSender: @classmethod def from_settings(cls, settings): - return cls(settings['MAIL_HOST'], settings['MAIL_FROM'], settings['MAIL_USER'], - settings['MAIL_PASS'], settings.getint('MAIL_PORT'), - settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL')) + return cls( + smtphost=settings['MAIL_HOST'], + mailfrom=settings['MAIL_FROM'], + smtpuser=settings['MAIL_USER'], + smtppass=settings['MAIL_PASS'], + smtpport=settings.getint('MAIL_PORT'), + smtptls=settings.getbool('MAIL_TLS'), + smtpssl=settings.getbool('MAIL_SSL'), + ) def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None): from twisted.internet import reactor @@ -89,9 +97,12 @@ class MailSender: return dfd = self._sendmail(rcpts, msg.as_string().encode(charset or 'utf-8')) - dfd.addCallbacks(self._sent_ok, self._sent_failed, + dfd.addCallbacks( + callback=self._sent_ok, + errback=self._sent_failed, callbackArgs=[to, cc, subject, len(attachs)], - errbackArgs=[to, cc, subject, len(attachs)]) + errbackArgs=[to, cc, subject, len(attachs)], + ) reactor.addSystemEventTrigger('before', 'shutdown', lambda: dfd) return dfd @@ -115,9 +126,10 @@ class MailSender: from twisted.mail.smtp import ESMTPSenderFactory msg = BytesIO(msg) d = defer.Deferred() - factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, - to_addrs, msg, d, heloFallback=True, requireAuthentication=False, - requireTransportSecurity=self.smtptls) + factory = ESMTPSenderFactory( + self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d, + heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls, + ) factory.noisy = False if self.smtpssl: diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 7c5eeac21..d207088e6 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -58,9 +58,9 @@ class ResponseTypes: def from_content_disposition(self, content_disposition): try: - filename = to_unicode(content_disposition, - encoding='latin-1', errors='replace').split(';')[1].split('=')[1] - filename = filename.strip('"\'') + filename = to_unicode( + content_disposition, encoding='latin-1', errors='replace' + ).split(';')[1].split('=')[1].strip('"\'') return self.from_filename(filename) except IndexError: return Response From 9c6d13075a2ab06c0d670e19457f9d9f219ceeee Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 18:56:14 -0300 Subject: [PATCH 30/50] Flake8: Remove E128 (tests, part 1) --- pytest.ini | 28 +++++++++--------- tests/test_command_parse.py | 4 +-- tests/test_command_shell.py | 6 ++-- tests/test_contracts.py | 3 +- tests/test_downloader_handlers.py | 8 ++--- tests/test_downloadermiddleware_httpproxy.py | 7 +++-- tests/test_downloadermiddleware_redirect.py | 2 +- tests/test_downloadermiddleware_retry.py | 14 +++++++-- tests/test_dupefilters.py | 31 +++++++++++++------- tests/test_http_request.py | 18 ++++-------- tests/test_http_response.py | 20 ++++++++----- 11 files changed, 80 insertions(+), 61 deletions(-) diff --git a/pytest.ini b/pytest.ini index 0a5a172b3..d8eefea7e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -172,37 +172,37 @@ flake8-ignore = tests/spiders.py E501 tests/test_closespider.py E501 tests/test_command_fetch.py E501 - tests/test_command_parse.py E501 E128 - tests/test_command_shell.py E501 E128 - tests/test_commands.py E128 E501 - tests/test_contracts.py E501 E128 + tests/test_command_parse.py E501 + tests/test_command_shell.py E501 + tests/test_commands.py E501 + tests/test_contracts.py E501 tests/test_crawl.py E501 E741 tests/test_crawler.py F841 E501 tests/test_dependencies.py F841 E501 - tests/test_downloader_handlers.py E128 E501 + tests/test_downloader_handlers.py E501 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 - tests/test_downloadermiddleware_cookies.py E741 E501 E128 + tests/test_downloadermiddleware_cookies.py E741 E501 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 tests/test_downloadermiddleware_httpcache.py E501 tests/test_downloadermiddleware_httpcompression.py E501 tests/test_downloadermiddleware_decompression.py E501 - tests/test_downloadermiddleware_httpproxy.py E501 E128 - tests/test_downloadermiddleware_redirect.py E501 E128 - tests/test_downloadermiddleware_retry.py E501 E128 + tests/test_downloadermiddleware_httpproxy.py E501 + tests/test_downloadermiddleware_redirect.py E501 + tests/test_downloadermiddleware_retry.py E501 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E501 E741 E128 - tests/test_engine.py E501 E128 - tests/test_exporters.py E501 E128 + tests/test_dupefilters.py E501 E741 + tests/test_engine.py E501 + tests/test_exporters.py E501 tests/test_extension_telnet.py F841 tests/test_feedexport.py E501 F841 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 - tests/test_http_request.py E402 E501 E128 E128 - tests/test_http_response.py E501 E128 tests/test_item.py E128 F841 + tests/test_http_request.py E402 E501 + tests/test_http_response.py E501 tests/test_link.py E501 tests/test_linkextractors.py E501 E128 tests/test_loader.py E501 E741 E128 E117 diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 85a24d0bc..a09dcf072 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -142,8 +142,8 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} @defer.inlineCallbacks def test_request_without_meta(self): _, _, stderr = yield self.execute(['--spider', self.spider_name, - '-c', 'parse_request_without_meta', - '--nolinks', + '-c', 'parse_request_without_meta', + '--nolinks', self.url('/html')]) self.assertIn("DEBUG: It Works!", _textmode(stderr)) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index d664b6ade..93fda2648 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -101,15 +101,13 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_local_nofile(self): filepath = 'file:///tests/sample_data/test_site/nothinghere.html' - errcode, out, err = yield self.execute([filepath, '-c', 'item'], - check_code=False) + errcode, out, err = yield self.execute([filepath, '-c', 'item'], check_code=False) self.assertEqual(errcode, 1, out or err) self.assertIn(b'No such file or directory', err) @defer.inlineCallbacks def test_dns_failures(self): url = 'www.somedomainthatdoesntexi.st' - errcode, out, err = yield self.execute([url, '-c', 'item'], - check_code=False) + errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False) self.assertEqual(errcode, 1, out or err) self.assertIn(b'DNS lookup failed', err) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index d1ce80f9d..99120b128 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -232,7 +232,8 @@ class ContractsManagerTest(unittest.TestCase): # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request) self.assertEqual(len(contracts), 2) - self.assertEqual(frozenset(type(x) for x in contracts), + self.assertEqual( + frozenset(type(x) for x in contracts), frozenset([UrlContract, ReturnsContract])) # returns request for valid method diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index f93bce8ef..1a05b679a 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -853,8 +853,7 @@ class S3TestCase(unittest.TestCase): def test_request_signing4(self): # fetches the access control policy sub-resource for the 'johnsmith' bucket. date = 'Tue, 27 Mar 2007 19:44:46 +0000' - req = Request('s3://johnsmith/?acl', - method='GET', headers={'Date': date}) + req = Request('s3://johnsmith/?acl', method='GET', headers={'Date': date}) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], @@ -879,8 +878,9 @@ class S3TestCase(unittest.TestCase): with self._mocked_date(date): httpreq = self.download_request(req, self.spider) # botocore does not override Date with x-amz-date - self.assertEqual(httpreq.headers['Authorization'], - b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') + self.assertEqual( + httpreq.headers['Authorization'], + b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') def test_request_signing6(self): # uploads an object to a CNAME style virtual hosted bucket with metadata. diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 36743b1de..9841d7a76 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -43,8 +43,11 @@ class TestHttpProxyMiddleware(TestCase): os.environ.pop('file_proxy', None) mw = HttpProxyMiddleware() - for url, proxy in [('http://e.com', http_proxy), - ('https://e.com', https_proxy), ('file://tmp/a', None)]: + for url, proxy in [ + ('http://e.com', http_proxy), + ('https://e.com', https_proxy), + ('file://tmp/a', None), + ]: req = Request(url) assert mw.process_request(req, spider) is None self.assertEqual(req.url, url) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 551e124ab..52c4d71a6 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -72,7 +72,7 @@ class RedirectMiddlewareTest(unittest.TestCase): url = 'http://www.example.com/302' url2 = 'http://www.example.com/redirected2' req = Request(url, method='POST', body='test', - headers={'Content-Type': 'text/plain', 'Content-length': '4'}) + headers={'Content-Type': 'text/plain', 'Content-length': '4'}) rsp = Response(url, headers={'Location': url2}, status=302) req2 = self.mw.process_response(req, rsp, self.spider) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index e118750e3..29357ba94 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -81,9 +81,17 @@ class RetryTest(unittest.TestCase): assert self.crawler.stats.get_value('retry/count') == 2 def test_twistederrors(self): - exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError, - DNSLookupError, ConnectionRefusedError, ConnectionDone, - ConnectError, ConnectionLost, ResponseFailed] + exceptions = [ + ConnectError, + ConnectionDone, + ConnectionLost, + ConnectionRefusedError, + defer.TimeoutError, + DNSLookupError, + ResponseFailed, + TCPTimedOutError, + TimeoutError, + ] for exc in exceptions: req = Request('http://www.scrapytest.org/%s' % exc.__name__) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 7426107c1..38b95b710 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -177,10 +177,13 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.log(r2, spider) assert crawler.stats.get_value('dupefilter/filtered') == 2 - l.check_present(('scrapy.dupefilters', 'DEBUG', - ('Filtered duplicate request: ' - ' - no more duplicates will be shown' - ' (see DUPEFILTER_DEBUG to show all duplicates)'))) + l.check_present( + ( + 'scrapy.dupefilters', + 'DEBUG', + 'Filtered duplicate request: - no more duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)' + ) + ) dupefilter.close('finished') @@ -203,11 +206,19 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.log(r2, spider) assert crawler.stats.get_value('dupefilter/filtered') == 2 - l.check_present(('scrapy.dupefilters', 'DEBUG', - ('Filtered duplicate request: ' - ' (referer: None)'))) - l.check_present(('scrapy.dupefilters', 'DEBUG', - ('Filtered duplicate request: ' - ' (referer: http://scrapytest.org/INDEX.html)'))) + l.check_present( + ( + 'scrapy.dupefilters', + 'DEBUG', + 'Filtered duplicate request: (referer: None)' + ) + ) + l.check_present( + ( + 'scrapy.dupefilters', + 'DEBUG', + 'Filtered duplicate request: (referer: http://scrapytest.org/INDEX.html)' + ) + ) dupefilter.close('finished') diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 77da15ce6..a672963f3 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -415,8 +415,7 @@ class FormRequestTest(RequestTest): # using multiples values for a single key data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']} r3 = self.request_class("http://www.example.com", formdata=data) - self.assertQueryEqual(r3.body, - b'colours=red&colours=blue&colours=green&price=%C2%A3+100') + self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100') def test_from_response_post(self): response = _buildresponse( @@ -426,8 +425,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') @@ -446,8 +444,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') @@ -468,8 +465,7 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", encoding='latin1', ) - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') @@ -488,8 +484,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') @@ -549,8 +544,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html") - r1 = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(r1.method, 'GET') self.assertEqual(urlparse(r1.url).hostname, "www.example.com") self.assertEqual(urlparse(r1.url).path, "/this/get.php") diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 522ec4875..04f58d305 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -485,8 +485,10 @@ class TextResponseTest(BaseResponseTest): response.xpath("normalize-space(//p[@class=\"content\"])").getall(), ) self.assertEqual( - response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()", - pclass="content", pcount=1).getall(), + response.xpath( + "//title[count(following::p[@class=$pclass])=$pcount]/text()", + pclass="content", pcount=1, + ).getall(), response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(), ) @@ -566,12 +568,14 @@ class TextResponseTest(BaseResponseTest): 'http://example.com', body=b'''click me''' ) - self._assert_followed_url(resp.css('a')[0], - 'http://example.com/foo', - response=resp) - self._assert_followed_url(resp.css('a::attr(href)')[0], - 'http://example.com/foo', - response=resp) + self._assert_followed_url( + resp.css('a')[0], + 'http://example.com/foo', + response=resp) + self._assert_followed_url( + resp.css('a::attr(href)')[0], + 'http://example.com/foo', + response=resp) def test_follow_encoding(self): resp1 = self.response_class( From bf56517abfc3d2287c08e1c88f07de7d3b1c499c Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 19:18:29 -0300 Subject: [PATCH 31/50] Flake8: Remove E128 (tests, part 2) --- pytest.ini | 20 ++++----- tests/test_item.py | 22 +++++----- tests/test_linkextractors.py | 6 +-- tests/test_logformatter.py | 21 +++++---- tests/test_mail.py | 2 +- tests/test_middleware.py | 9 ++-- tests/test_pipeline_crawl.py | 12 ++++-- tests/test_pipeline_media.py | 12 +++--- tests/test_spidermiddleware_httperror.py | 55 ++++++++---------------- tests/test_spidermiddleware_offsite.py | 32 ++++++++------ 10 files changed, 94 insertions(+), 97 deletions(-) diff --git a/pytest.ini b/pytest.ini index d8eefea7e..0b2766031 100644 --- a/pytest.ini +++ b/pytest.ini @@ -200,19 +200,19 @@ flake8-ignore = tests/test_feedexport.py E501 F841 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 - tests/test_item.py E128 F841 tests/test_http_request.py E402 E501 tests/test_http_response.py E501 + tests/test_item.py E501 F841 tests/test_link.py E501 - tests/test_linkextractors.py E501 E128 - tests/test_loader.py E501 E741 E128 E117 - tests/test_logformatter.py E128 E501 - tests/test_mail.py E128 E501 - tests/test_middleware.py E501 E128 - tests/test_pipeline_crawl.py E501 E128 + tests/test_linkextractors.py E501 + tests/test_loader.py E501 E741 E117 + tests/test_logformatter.py E501 + tests/test_mail.py E501 + tests/test_middleware.py E501 + tests/test_pipeline_crawl.py E501 tests/test_pipeline_files.py E501 tests/test_pipeline_images.py F841 E501 - tests/test_pipeline_media.py E501 E741 E128 + tests/test_pipeline_media.py E501 E741 tests/test_proxy_connect.py E501 E741 tests/test_request_cb_kwargs.py E501 tests/test_responsetypes.py E501 @@ -221,8 +221,8 @@ flake8-ignore = tests/test_selector.py E501 tests/test_spider.py E501 tests/test_spidermiddleware.py E501 - tests/test_spidermiddleware_httperror.py E128 E501 E121 - tests/test_spidermiddleware_offsite.py E501 E128 E111 + tests/test_spidermiddleware_httperror.py E501 E121 + tests/test_spidermiddleware_offsite.py E501 E111 tests/test_spidermiddleware_output_chain.py E501 tests/test_spidermiddleware_referer.py E501 F841 E501 E121 tests/test_squeues.py E501 E741 diff --git a/tests/test_item.py b/tests/test_item.py index 4017f6e84..4fe247cda 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -162,8 +162,7 @@ class ItemTest(unittest.TestCase): item = D(save='X', load='Y') self.assertEqual(item['save'], 'X') self.assertEqual(item['load'], 'Y') - self.assertEqual(D.fields, {'load': {'default': 'A'}, - 'save': {'default': 'A'}}) + self.assertEqual(D.fields, {'load': {'default': 'A'}, 'save': {'default': 'A'}}) # D class inverted class E(C, B): @@ -171,8 +170,7 @@ class ItemTest(unittest.TestCase): self.assertEqual(E(save='X')['save'], 'X') self.assertEqual(E(load='X')['load'], 'X') - self.assertEqual(E.fields, {'load': {'default': 'C'}, - 'save': {'default': 'C'}}) + self.assertEqual(E.fields, {'load': {'default': 'C'}, 'save': {'default': 'C'}}) def test_metaclass_multiple_inheritance_diamond(self): class A(Item): @@ -193,8 +191,9 @@ class ItemTest(unittest.TestCase): self.assertEqual(D(save='X')['save'], 'X') self.assertEqual(D(load='X')['load'], 'X') - self.assertEqual(D.fields, {'save': {'default': 'C'}, - 'load': {'default': 'D'}, 'update': {'default': 'D'}}) + self.assertEqual( + D.fields, + {'save': {'default': 'C'}, 'load': {'default': 'D'}, 'update': {'default': 'D'}}) # D class inverted class E(C, B): @@ -202,8 +201,9 @@ class ItemTest(unittest.TestCase): self.assertEqual(E(save='X')['save'], 'X') self.assertEqual(E(load='X')['load'], 'X') - self.assertEqual(E.fields, {'save': {'default': 'C'}, - 'load': {'default': 'E'}, 'update': {'default': 'C'}}) + self.assertEqual( + E.fields, + {'save': {'default': 'C'}, 'load': {'default': 'E'}, 'update': {'default': 'C'}}) def test_metaclass_multiple_inheritance_without_metaclass(self): class A(Item): @@ -223,8 +223,7 @@ class ItemTest(unittest.TestCase): self.assertRaises(KeyError, D, not_allowed='value') self.assertEqual(D(save='X')['save'], 'X') - self.assertEqual(D.fields, {'save': {'default': 'A'}, - 'load': {'default': 'A'}}) + self.assertEqual(D.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}}) # D class inverted class E(C, B): @@ -232,8 +231,7 @@ class ItemTest(unittest.TestCase): self.assertRaises(KeyError, E, not_allowed='value') self.assertEqual(E(save='X')['save'], 'X') - self.assertEqual(E.fields, {'save': {'default': 'A'}, - 'load': {'default': 'A'}}) + self.assertEqual(E.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}}) def test_to_dict(self): class TestItem(Item): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 68e8514ba..7e1b62b7f 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -171,9 +171,9 @@ class Base: self.assertEqual(lx.matches(url1), False) self.assertEqual(lx.matches(url2), True) - lx = self.extractor_cls(allow=('blah1',), deny=('blah2',), - allow_domains=('blah1.com',), - deny_domains=('blah2.com',)) + lx = self.extractor_cls(allow=['blah1'], deny=['blah2'], + allow_domains=['blah1.com'], + deny_domains=['blah2.com']) self.assertEqual(lx.matches('http://blah1.com/blah1'), True) self.assertEqual(lx.matches('http://blah1.com/blah2'), False) self.assertEqual(lx.matches('http://blah2.com/blah1'), False) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index cd6cb8016..7064337ad 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -34,15 +34,15 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, - "Crawled (200) (referer: None)") + self.assertEqual(logline, "Crawled (200) (referer: None)") def test_crawled_without_referer(self): req = Request("http://www.example.com", headers={'referer': 'http://example.com'}) res = Response("http://www.example.com", flags=['cached']) logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, + self.assertEqual( + logline, "Crawled (200) (referer: http://example.com) ['cached']") def test_flags_in_request(self): @@ -50,8 +50,9 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, - "Crawled (200) ['test', 'flag'] (referer: None)") + self.assertEqual( + logline, + "Crawled (200) ['test', 'flag'] (referer: None)") def test_dropped(self): item = {} @@ -140,7 +141,8 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, + self.assertEqual( + logline, "Crawled (200) (referer: None) []") def test_crawled_without_referer(self): @@ -148,7 +150,8 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, + self.assertEqual( + logline, "Crawled (200) (referer: http://example.com) ['cached']") def test_flags_in_request(self): @@ -156,7 +159,9 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, "Crawled (200) (referer: None) ['test', 'flag']") + self.assertEqual( + logline, + "Crawled (200) (referer: None) ['test', 'flag']") class SkipMessagesLogFormatter(LogFormatter): diff --git a/tests/test_mail.py b/tests/test_mail.py index f5cb81a8b..53dbc0686 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -49,7 +49,7 @@ class MailSenderTest(unittest.TestCase): mailsender = MailSender(debug=True) mailsender.send(to=['test@scrapy.org'], subject='subject', body='body', - attachs=attachs, _callback=self._catch_mail_sent) + attachs=attachs, _callback=self._catch_mail_sent) assert self.catched_msg self.assertEqual(self.catched_msg['to'], ['test@scrapy.org']) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 3af514bb0..3364d2258 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -69,11 +69,14 @@ class MiddlewareManagerTest(unittest.TestCase): def test_methods(self): mwman = TestMiddlewareManager(M1(), M2(), M3()) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']], + self.assertEqual( + [x.__self__.__class__ for x in mwman.methods['open_spider']], [M1, M2]) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']], + self.assertEqual( + [x.__self__.__class__ for x in mwman.methods['close_spider']], [M2, M1]) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']], + self.assertEqual( + [x.__self__.__class__ for x in mwman.methods['process']], [M1, M3]) def test_enabled(self): diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 24c516473..188c5c3cf 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -132,7 +132,8 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media(self): crawler = self._create_crawler(MediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_downloaded(self.items, str(log)) @@ -141,7 +142,8 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_wrong_urls(self): crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_download_failure(crawler, self.items, 404, str(log)) @@ -150,7 +152,8 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_redirected_default_failure(self): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver) @@ -164,7 +167,8 @@ class FileDownloadCrawlTestCase(TestCase): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 949f0dea1..b51f82422 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -214,9 +214,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = dict(requests=req) new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item['results'], [(True, rsp)]) - self.assertEqual(self.pipe._mockcalled, - ['get_media_requests', 'media_to_download', - 'media_downloaded', 'request_callback', 'item_completed']) + self.assertEqual( + self.pipe._mockcalled, + ['get_media_requests', 'media_to_download', 'media_downloaded', 'request_callback', 'item_completed']) @inlineCallbacks def test_result_failure(self): @@ -227,9 +227,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = dict(requests=req) new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item['results'], [(False, fail)]) - self.assertEqual(self.pipe._mockcalled, - ['get_media_requests', 'media_to_download', - 'media_failed', 'request_errback', 'item_completed']) + self.assertEqual( + self.pipe._mockcalled, + ['get_media_requests', 'media_to_download', 'media_failed', 'request_errback', 'item_completed']) @inlineCallbacks def test_mix_of_success_and_failure(self): diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 6b61df56f..714279ae0 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -68,29 +68,23 @@ class TestHttpErrorMiddleware(TestCase): self.res200, self.res404 = _responses(self.req, [200, 404]) def test_process_spider_input(self): - self.assertEqual(None, - self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, self.res404, self.spider) + self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) def test_process_spider_exception(self): - self.assertEqual([], - self.mw.process_spider_exception(self.res404, - HttpError(self.res404), self.spider)) - self.assertEqual(None, - self.mw.process_spider_exception(self.res404, - Exception(), self.spider)) + self.assertEqual( + [], + self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider)) + self.assertIsNone(self.mw.process_spider_exception(self.res404, Exception(), self.spider)) def test_handle_httpstatus_list(self): res = self.res404.copy() res.request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) - self.assertEqual(None, - self.mw.process_spider_input(res, self.spider)) + self.assertIsNone(self.mw.process_spider_input(res, self.spider)) self.spider.handle_httpstatus_list = [404] - self.assertEqual(None, - self.mw.process_spider_input(self.res404, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) class TestHttpErrorMiddlewareSettings(TestCase): @@ -103,12 +97,9 @@ class TestHttpErrorMiddlewareSettings(TestCase): self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertEqual(None, - self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, self.res404, self.spider) - self.assertEqual(None, - self.mw.process_spider_input(self.res402, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) + self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) @@ -117,17 +108,13 @@ class TestHttpErrorMiddlewareSettings(TestCase): res402 = self.res402.copy() res402.request = request - self.assertEqual(None, - self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, res402, self.spider) + self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] - self.assertEqual(None, - self.mw.process_spider_input(self.res404, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, self.res402, self.spider) + self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider) class TestHttpErrorMiddlewareHandleAll(TestCase): @@ -139,10 +126,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertEqual(None, - self.mw.process_spider_input(self.res200, self.spider)) - self.assertEqual(None, - self.mw.process_spider_input(self.res404, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) def test_meta_overrides_settings(self): request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) @@ -151,10 +136,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402 = self.res402.copy() res402.request = request - self.assertEqual(None, - self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, res402, self.spider) + self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) class TestHttpErrorMiddlewareIntegrational(TrialTestCase): diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index b96807bc2..0f4b98a07 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -22,20 +22,24 @@ class TestOffsiteMiddleware(TestCase): def test_process_spider_output(self): res = Response('http://scrapytest.org') - onsite_reqs = [Request('http://scrapytest.org/1'), - Request('http://scrapy.org/1'), - Request('http://sub.scrapy.org/1'), - Request('http://offsite.tld/letmepass', dont_filter=True), - Request('http://scrapy.test.org/'), - Request('http://scrapy.test.org:8000/')] - offsite_reqs = [Request('http://scrapy2.org'), - Request('http://offsite.tld/'), - Request('http://offsite.tld/scrapytest.org'), - Request('http://offsite.tld/rogue.scrapytest.org'), - Request('http://rogue.scrapytest.org.haha.com'), - Request('http://roguescrapytest.org'), - Request('http://test.org/'), - Request('http://notscrapy.test.org/')] + onsite_reqs = [ + Request('http://scrapytest.org/1'), + Request('http://scrapy.org/1'), + Request('http://sub.scrapy.org/1'), + Request('http://offsite.tld/letmepass', dont_filter=True), + Request('http://scrapy.test.org/'), + Request('http://scrapy.test.org:8000/'), + ] + offsite_reqs = [ + Request('http://scrapy2.org'), + Request('http://offsite.tld/'), + Request('http://offsite.tld/scrapytest.org'), + Request('http://offsite.tld/rogue.scrapytest.org'), + Request('http://rogue.scrapytest.org.haha.com'), + Request('http://roguescrapytest.org'), + Request('http://test.org/'), + Request('http://notscrapy.test.org/'), + ] reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider)) From 881b4f417f645fa9719e6c54eb3788cc67d56053 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 19:33:22 -0300 Subject: [PATCH 32/50] Flake8: Remove E128 (tests, part 3) --- pytest.ini | 18 ++++++------- tests/test_settings/__init__.py | 18 ++++++++----- tests/test_spiderloader/__init__.py | 21 ++++++++++----- tests/test_utils_conf.py | 3 ++- tests/test_utils_iterators.py | 23 +++++++++++------ tests/test_utils_reqser.py | 3 ++- tests/test_utils_request.py | 5 ++-- tests/test_utils_sitemap.py | 9 +++++-- tests/test_webclient.py | 40 +++++++++++++++++------------ 9 files changed, 87 insertions(+), 53 deletions(-) diff --git a/pytest.ini b/pytest.ini index 0b2766031..7c3c9cc5b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -227,23 +227,23 @@ flake8-ignore = tests/test_spidermiddleware_referer.py E501 F841 E501 E121 tests/test_squeues.py E501 E741 tests/test_utils_asyncio.py E501 - tests/test_utils_conf.py E501 E128 + tests/test_utils_conf.py E501 tests/test_utils_curl.py E501 tests/test_utils_datatypes.py E402 E501 tests/test_utils_defer.py E501 F841 tests/test_utils_deprecate.py F841 E501 - tests/test_utils_http.py E501 E128 - tests/test_utils_iterators.py E501 E128 E129 + tests/test_utils_http.py E501 + tests/test_utils_iterators.py E501 E129 tests/test_utils_log.py E741 tests/test_utils_python.py E501 - tests/test_utils_reqser.py E501 E128 - tests/test_utils_request.py E501 E128 + tests/test_utils_reqser.py E501 + tests/test_utils_request.py E501 tests/test_utils_response.py E501 tests/test_utils_signal.py E741 F841 - tests/test_utils_sitemap.py E128 E501 + tests/test_utils_sitemap.py E501 tests/test_utils_url.py E501 E501 - tests/test_webclient.py E501 E128 E402 + tests/test_webclient.py E501 E402 tests/test_cmdline/__init__.py E501 - tests/test_settings/__init__.py E501 E128 - tests/test_spiderloader/__init__.py E128 E501 + tests/test_settings/__init__.py E501 + tests/test_spiderloader/__init__.py E501 tests/test_utils_misc/__init__.py E501 diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index fda44653a..2da6aa4b5 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -314,13 +314,17 @@ class BaseSettingsTest(unittest.TestCase): 'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'), 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) - self.assertDictEqual(s.copy_to_dict(), - {'HASNOBASE': {3: 3000}, - 'TEST': {1: 10, 3: 30}, - 'TEST_BASE': {1: 1, 2: 2}, - 'TEST_BOOLEAN': False, - 'TEST_LIST': [1, 2], - 'TEST_STRING': 'a string'}) + self.assertDictEqual( + s.copy_to_dict(), + { + 'HASNOBASE': {3: 3000}, + 'TEST': {1: 10, 3: 30}, + 'TEST_BASE': {1: 1, 2: 2}, + 'TEST_LIST': [1, 2], + 'TEST_BOOLEAN': False, + 'TEST_STRING': 'a string', + } + ) def test_freeze(self): self.settings.freeze() diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index d8be6e277..b20e1b3f2 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -40,7 +40,8 @@ class SpiderLoaderTest(unittest.TestCase): verifyObject(ISpiderLoader, self.spider_loader) def test_list(self): - self.assertEqual(set(self.spider_loader.list()), + self.assertEqual( + set(self.spider_loader.list()), set(['spider1', 'spider2', 'spider3', 'spider4'])) def test_load(self): @@ -48,17 +49,23 @@ class SpiderLoaderTest(unittest.TestCase): self.assertEqual(spider1.__name__, 'Spider1') def test_find_by_request(self): - self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy1.org/test')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://scrapy1.org/test')), ['spider1']) - self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy2.org/test')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://scrapy2.org/test')), ['spider2']) - self.assertEqual(set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))), + self.assertEqual( + set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))), set(['spider1', 'spider2'])) - self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy999.org/test')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://scrapy999.org/test')), []) - self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://spider3.com')), []) - self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')), ['spider3']) def test_load_spider_module(self): diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 332120021..e5d3ef582 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -93,7 +93,8 @@ class BuildComponentListTest(unittest.TestCase): class UtilsConfTestCase(unittest.TestCase): def test_arglist_to_dict(self): - self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']), + self.assertEqual( + arglist_to_dict(['arg1=val1', 'arg2=val2']), {'arg1': 'val1', 'arg2': 'val2'}) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index a85087619..c677dabe5 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -47,8 +47,7 @@ class XmliterTestCase(unittest.TestCase): """ response = XmlResponse(url="http://example.com", body=body) - nodenames = [e.xpath('name()').getall() - for e in self.xmliter(response, 'matchme...')] + nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')] self.assertEqual(nodenames, [['matchme...']]) def test_xmliter_unicode(self): @@ -359,15 +358,23 @@ class UtilsCsvTestCase(unittest.TestCase): response = TextResponse(url="http://example.com/", body=body1, encoding='latin1') csv = csviter(response) - self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'latin1', u'value': u'test'}, - {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}]) + self.assertEqual( + list(csv), + [ + {u'id': u'1', u'name': u'latin1', u'value': u'test'}, + {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}, + ] + ) response = TextResponse(url="http://example.com/", body=body2, encoding='cp852') csv = csviter(response) - self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'cp852', u'value': u'test'}, - {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}]) + self.assertEqual( + list(csv), + [ + {u'id': u'1', u'name': u'cp852', u'value': u'test'}, + {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}, + ] + ) class TestHelper(unittest.TestCase): diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 50b026d1c..450e4bdca 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -15,7 +15,8 @@ class RequestSerializationTest(unittest.TestCase): self._assert_serializes_ok(r) def test_all_attributes(self): - r = Request("http://www.example.com", + r = Request( + url="http://www.example.com", callback=self.spider.parse_item, errback=self.spider.handle_error, method="POST", diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 45f0f59e4..50efb63ca 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -36,8 +36,9 @@ class UtilsRequestTest(unittest.TestCase): self.assertEqual(request_fingerprint(r1), request_fingerprint(r1, include_headers=['Accept-Language'])) - self.assertNotEqual(request_fingerprint(r1), - request_fingerprint(r2, include_headers=['Accept-Language'])) + self.assertNotEqual( + request_fingerprint(r1), + request_fingerprint(r2, include_headers=['Accept-Language'])) self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']), request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language'])) diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index 08b215434..d77978ff1 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -22,8 +22,13 @@ class SitemapTest(unittest.TestCase): """) assert s.type == 'urlset' - self.assertEqual(list(s), - [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}]) + self.assertEqual( + list(s), + [ + {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, + {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}, + ] + ) def test_sitemap_index(self): s = Sitemap(b""" diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 307fadb5c..188e54602 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -47,8 +47,9 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): return f from twisted.web.client import _makeGetterFactory - return _makeGetterFactory(to_bytes(url), _clientfactory, - contextFactory=contextFactory, *args, **kwargs).deferred + return _makeGetterFactory( + to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs + ).deferred class ParseUrlTestCase(unittest.TestCase): @@ -105,7 +106,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): 'Content-Length': '12981', 'Useful': 'value'})) - self._test(factory, + self._test( + factory, b"GET /bar HTTP/1.0\r\n" b"Content-Length: 9\r\n" b"Useful: value\r\n" @@ -118,7 +120,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): # test minimal sent headers factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar')) - self._test(factory, + self._test( + factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"\r\n") @@ -130,7 +133,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): body='name=value', headers={'Content-Type': 'application/x-www-form-urlencoded'})) - self._test(factory, + self._test( + factory, b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Connection: close\r\n" @@ -145,7 +149,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): url='http://foo/bar' )) - self._test(factory, + self._test( + factory, b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Content-Length: 0\r\n" @@ -160,7 +165,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): }, )) - self._test(factory, + self._test( + factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"X-Meta-Multivalued: value1\r\n" @@ -177,7 +183,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): }), )) - self._test(factory, + self._test( + factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"X-Meta-Multivalued: value1\r\n" @@ -206,8 +213,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): protocol.dataReceived(b"Hello: World\n") protocol.dataReceived(b"Foo: Bar\n") protocol.dataReceived(b"\n") - self.assertEqual(protocol.headers, - Headers({'Hello': ['World'], 'Foo': ['Bar']})) + self.assertEqual(protocol.headers, Headers({'Hello': ['World'], 'Foo': ['Bar']})) class EncodingResource(resource.Resource): @@ -340,10 +346,11 @@ class WebClientTestCase(unittest.TestCase): return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) def _cbRedirect(self, pageData): - self.assertEqual(pageData, - b'\n\n \n \n' - b' \n \n ' - b'click here\n \n\n') + self.assertEqual( + pageData, + b'\n\n \n \n' + b' \n \n ' + b'click here\n \n\n') def test_encoding(self): """ Test that non-standart body encoding matches @@ -403,8 +410,9 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): s = "0123456789" * 10 settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': self.custom_ciphers}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) - return getPage(self.getURL("payload"), body=s, - contextFactory=client_context_factory).addCallback(self.assertEqual, to_bytes(s)) + return getPage( + self.getURL("payload"), body=s, contextFactory=client_context_factory + ).addCallback(self.assertEqual, to_bytes(s)) def testPayloadDefaultCiphers(self): s = "0123456789" * 10 From d472402a0232781753515d9552b7a1997b43543a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 14:39:17 -0300 Subject: [PATCH 33/50] Fix pickle test for pypy --- pytest.ini | 1 + tests/test_squeues.py | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 4f3494e0e..d107c1fbe 100644 --- a/pytest.ini +++ b/pytest.ini @@ -166,6 +166,7 @@ flake8-ignore = scrapy/signalmanager.py E501 scrapy/spiderloader.py F841 E501 E126 scrapy/squeues.py E128 + scrapy/squeues.py E501 scrapy/statscollectors.py E501 # tests tests/__init__.py E402 E501 diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 51c0c028a..d2cf9135f 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,3 +1,6 @@ +import pickle +import sys + from queuelib.tests import test_queue as t from scrapy.squeues import ( MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue, @@ -108,8 +111,10 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): try: q.push(lambda x: x) except ValueError as exc: - self.assertIsInstance(exc.__context__, AttributeError) - + if hasattr(sys, "pypy_version_info"): + self.assertIsInstance(exc.__context__, pickle.PicklingError) + else: + self.assertIsInstance(exc.__context__, AttributeError) sel = Selector(text='

some text

') try: q.push(sel) From b59dfb75fa72346f8268b83dedd2c1f9af460c71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 7 May 2020 14:14:59 +0200 Subject: [PATCH 34/50] Update disabled Pylint checks --- pylintrc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pylintrc b/pylintrc index c52a4c2d0..129c7bf7d 100644 --- a/pylintrc +++ b/pylintrc @@ -13,6 +13,7 @@ disable=abstract-method, bad-mcs-classmethod-argument, bad-super-call, bad-whitespace, + bare-except, blacklisted-name, broad-except, c-extension-no-member, @@ -39,6 +40,8 @@ disable=abstract-method, inconsistent-return-statements, inherit-non-class, invalid-name, + invalid-overridden-method, + isinstance-second-argument-not-valid-type, keyword-arg-before-vararg, line-too-long, logging-format-interpolation, @@ -77,6 +80,7 @@ disable=abstract-method, too-many-ancestors, too-many-arguments, too-many-branches, + too-many-format-args, too-many-function-args, too-many-instance-attributes, too-many-lines, @@ -87,6 +91,7 @@ disable=abstract-method, trailing-whitespace, unbalanced-tuple-unpacking, undefined-variable, + undefined-loop-variable, unexpected-special-method-signature, ungrouped-imports, unidiomatic-typecheck, From 422e6429b56e42b8344a0e46c45f4106d374d024 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 7 May 2020 09:22:14 -0300 Subject: [PATCH 35/50] Add mising len check in spiderloader --- scrapy/spiderloader.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 8dc89c2e9..92aed9b8e 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -from collections import defaultdict import traceback import warnings +from collections import defaultdict from zope.interface import implementer @@ -16,6 +16,7 @@ class SpiderLoader: SpiderLoader is a class which locates and loads spiders in a Scrapy project. """ + def __init__(self, settings): self.spider_modules = settings.getlist('SPIDER_MODULES') self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY') @@ -29,6 +30,7 @@ class SpiderLoader: dupes.extend([ " {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name) for mod, cls in locations + if len(locations) > 1 ]) if dupes: @@ -49,10 +51,9 @@ class SpiderLoader: self._load_spiders(module) except ImportError: if self.warn_only: - msg = ( - "\n{tb}Could not load spiders from module '{modname}'. " - "See above traceback for details.".format(modname=name, tb=traceback.format_exc()) - ) + msg = ("\n{tb}Could not load spiders from module '{modname}'. " + "See above traceback for details.".format( + modname=name, tb=traceback.format_exc())) warnings.warn(msg, RuntimeWarning) else: raise @@ -76,8 +77,10 @@ class SpiderLoader: """ Return the list of spider names that can handle the given request. """ - return [name for name, cls in self._spiders.items() - if cls.handles_request(request)] + return [ + name for name, cls in self._spiders.items() + if cls.handles_request(request) + ] def list(self): """ From e0127a31230d4be13b1bd29e62d75c2954b47d9e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 7 May 2020 12:48:43 -0300 Subject: [PATCH 36/50] Refactor warnings in spiderloader --- scrapy/spiderloader.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 92aed9b8e..63da55718 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -35,9 +35,11 @@ class SpiderLoader: if dupes: dupes_string = "\n\n".join(dupes) - msg = ("There are several spiders with the same name:\n\n" - "{}\n\n This can cause unexpected behavior.".format(dupes_string)) - warnings.warn(msg, UserWarning) + warnings.warn( + "There are several spiders with the same name:\n\n" + "{}\n\n This can cause unexpected behavior.".format(dupes_string), + category=UserWarning, + ) def _load_spiders(self, module): for spcls in iter_spider_classes(module): @@ -51,10 +53,13 @@ class SpiderLoader: self._load_spiders(module) except ImportError: if self.warn_only: - msg = ("\n{tb}Could not load spiders from module '{modname}'. " - "See above traceback for details.".format( - modname=name, tb=traceback.format_exc())) - warnings.warn(msg, RuntimeWarning) + warnings.warn( + "\n{tb}Could not load spiders from module '{modname}'. " + "See above traceback for details.".format( + modname=name, tb=traceback.format_exc() + ), + category=RuntimeWarning, + ) else: raise self._check_name_duplicates() From cf09af787eafa6770bc5ab00bb0ee9759c75df23 Mon Sep 17 00:00:00 2001 From: Antonio Gordillo Toledo Date: Fri, 8 May 2020 06:45:19 -0700 Subject: [PATCH 37/50] Remove Python 2 encoding header from files (#4553) --- docs/conf.py | 2 -- scrapy/downloadermiddlewares/ajaxcrawl.py | 1 - scrapy/spiderloader.py | 1 - scrapy/templates/project/module/items.py.tmpl | 2 -- scrapy/templates/project/module/middlewares.py.tmpl | 2 -- scrapy/templates/project/module/pipelines.py.tmpl | 2 -- scrapy/templates/project/module/settings.py.tmpl | 2 -- scrapy/templates/spiders/basic.tmpl | 1 - scrapy/templates/spiders/crawl.tmpl | 1 - scrapy/templates/spiders/csvfeed.tmpl | 1 - scrapy/templates/spiders/xmlfeed.tmpl | 1 - scrapy/utils/log.py | 2 -- scrapy/utils/ssl.py | 2 -- tests/test_downloadermiddleware_redirect.py | 2 -- tests/test_downloadermiddleware_robotstxt.py | 1 - tests/test_http_response.py | 1 - tests/test_pipeline_crawl.py | 1 - tests/test_responsetypes.py | 1 - tests/test_utils_deprecate.py | 1 - tests/test_utils_iterators.py | 1 - tests/test_utils_log.py | 1 - tests/test_utils_url.py | 1 - 22 files changed, 30 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 813417bae..8ab38a090 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Scrapy documentation build configuration file, created by # sphinx-quickstart on Mon Nov 24 12:02:52 2008. # diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index ad7a81e6b..4e12a5044 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import re import logging diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 63da55718..db4193430 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import traceback import warnings from collections import defaultdict diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl index a12d08414..88a18331c 100644 --- a/scrapy/templates/project/module/items.py.tmpl +++ b/scrapy/templates/project/module/items.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Define here the models for your scraped items # # See documentation in: diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index b3e58ff94..6490f52a7 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Define here the models for your spider middleware # # See documentation in: diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index 4876526a9..ce0edd335 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index cb220eafc..a414b5fde 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Scrapy settings for $project_name project # # For simplicity, this file contains only settings considered important or diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 1cfe9cc9d..e9112bc95 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import scrapy diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 878425125..356496487 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index c2e4bacfe..cbcbe9e2c 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from scrapy.spiders import CSVFeedSpider diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index 863c9772f..5aa2aa8b0 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from scrapy.spiders import XMLFeedSpider diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 5998dc33b..203842fc8 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import logging import sys import warnings diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 6e81b33ff..c3c5e329b 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import OpenSSL import OpenSSL._util as pyOpenSSLutil diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 551e124ab..61c9eddbc 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import unittest from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index a1645ed96..b9452a0e7 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from unittest import mock from twisted.internet import reactor, error diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 522ec4875..43d6d936a 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import unittest from w3lib.encoding import resolve_encoding diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 24c516473..e2578a9c9 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os import shutil diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 8cdf7a176..9e63ac924 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import unittest from scrapy.responsetypes import responsetypes diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index b17e17f2f..adef66c1d 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import inspect import unittest from unittest import mock diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 46aaaecbc..69339256e 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os from twisted.trial import unittest diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 21100aeb8..25cd904bc 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import sys import logging import unittest diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 1f8388957..16e7449c9 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import unittest from scrapy.spiders import Spider From b852fff6f82e24c535c0dd9b4ef7fa78e7946497 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 8 May 2020 15:19:22 -0300 Subject: [PATCH 38/50] Style changes in link extractor --- scrapy/linkextractors/lxmlhtml.py | 52 ++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index ceb37c5f1..9ebf8e7c7 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -28,8 +28,9 @@ def _nons(tag): class LxmlParserLinkExtractor: - def __init__(self, tag="a", attr="href", process=None, unique=False, - strip=True, canonicalized=False): + def __init__( + self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False + ): self.scan_tag = tag if callable(tag) else lambda t: t == tag self.scan_attr = attr if callable(attr) else lambda a: a == attr self.process_attr = process if callable(process) else lambda v: v @@ -93,10 +94,23 @@ class LxmlParserLinkExtractor: class LxmlLinkExtractor(FilteringLinkExtractor): - def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=False, - unique=True, process_value=None, deny_extensions=None, restrict_css=(), - strip=True, restrict_text=None): + def __init__( + self, + allow=(), + deny=(), + allow_domains=(), + deny_domains=(), + restrict_xpaths=(), + tags=('a', 'area'), + attrs=('href',), + canonicalize=False, + unique=True, + process_value=None, + deny_extensions=None, + restrict_css=(), + strip=True, + restrict_text=None, + ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) lx = LxmlParserLinkExtractor( tag=lambda x: x in tags, @@ -106,12 +120,18 @@ class LxmlLinkExtractor(FilteringLinkExtractor): strip=strip, canonicalized=canonicalize ) - - super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, - allow_domains=allow_domains, deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, - canonicalize=canonicalize, deny_extensions=deny_extensions, - restrict_text=restrict_text) + super(LxmlLinkExtractor, self).__init__( + link_extractor=lx, + allow=allow, + deny=deny, + allow_domains=allow_domains, + deny_domains=deny_domains, + restrict_xpaths=restrict_xpaths, + restrict_css=restrict_css, + canonicalize=canonicalize, + deny_extensions=deny_extensions, + restrict_text=restrict_text, + ) def extract_links(self, response): """Returns a list of :class:`~scrapy.link.Link` objects from the @@ -124,9 +144,11 @@ class LxmlLinkExtractor(FilteringLinkExtractor): """ base_url = get_base_url(response) if self.restrict_xpaths: - docs = [subdoc - for x in self.restrict_xpaths - for subdoc in response.xpath(x)] + docs = [ + subdoc + for x in self.restrict_xpaths + for subdoc in response.xpath(x) + ] else: docs = [response.selector] all_links = [] From 3ebf2a0d82b0ae5b9e701972ba2a9e1420c6d2c7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 8 May 2020 15:17:33 -0300 Subject: [PATCH 39/50] Remove lambdas in link extractor --- scrapy/linkextractors/lxmlhtml.py | 28 +++++++++++++++++----------- tests/test_linkextractors.py | 5 +++++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 9ebf8e7c7..1615d44d7 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,6 +1,8 @@ """ Link extractor based on lxml.html """ +import operator +from functools import partial from urllib.parse import urljoin import lxml.etree as etree @@ -8,10 +10,10 @@ from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link +from scrapy.linkextractors import FilteringLinkExtractor from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url -from scrapy.linkextractors import FilteringLinkExtractor # from lxml/src/lxml/html/__init__.py @@ -27,20 +29,24 @@ def _nons(tag): return tag +def _identity(x): + return x + + +def _canonicalize_link_url(link): + return canonicalize_url(link.url, keep_fragments=True) + + class LxmlParserLinkExtractor: def __init__( self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False ): - self.scan_tag = tag if callable(tag) else lambda t: t == tag - self.scan_attr = attr if callable(attr) else lambda a: a == attr - self.process_attr = process if callable(process) else lambda v: v + self.scan_tag = tag if callable(tag) else partial(operator.eq, tag) + self.scan_attr = attr if callable(attr) else partial(operator.eq, attr) + self.process_attr = process if callable(process) else _identity self.unique = unique self.strip = strip - if canonicalized: - self.link_key = lambda link: link.url - else: - self.link_key = lambda link: canonicalize_url(link.url, - keep_fragments=True) + self.link_key = operator.attrgetter("url") if canonicalized else _canonicalize_link_url def _iter_links(self, document): for el in document.iter(etree.Element): @@ -113,8 +119,8 @@ class LxmlLinkExtractor(FilteringLinkExtractor): ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) lx = LxmlParserLinkExtractor( - tag=lambda x: x in tags, - attr=lambda x: x in attrs, + tag=partial(operator.contains, tags), + attr=partial(operator.contains, attrs), unique=unique, process=process_value, strip=strip, diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 68e8514ba..46d8c13af 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,3 +1,4 @@ +import pickle import re import unittest from warnings import catch_warnings @@ -462,6 +463,10 @@ class Base: Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False), ]) + def test_pickle_extractor(self): + lx = self.extractor_cls() + self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls) + class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = LxmlLinkExtractor From 81d0b2f61ac119efab4d5970bd235dbc288496ef Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 8 May 2020 16:23:53 -0300 Subject: [PATCH 40/50] Flake8: remove E111 --- pytest.ini | 4 ++-- scrapy/selector/unified.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pytest.ini b/pytest.ini index 604bbfe1d..20639baa3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -102,7 +102,7 @@ flake8-ignore = scrapy/pipelines/media.py E501 # scrapy/selector scrapy/selector/__init__.py F403 - scrapy/selector/unified.py E501 E111 + scrapy/selector/unified.py E501 # scrapy/settings scrapy/settings/__init__.py E501 scrapy/settings/default_settings.py E501 E114 E116 @@ -224,7 +224,7 @@ flake8-ignore = tests/test_spider.py E501 tests/test_spidermiddleware.py E501 tests/test_spidermiddleware_httperror.py E128 E501 E121 - tests/test_spidermiddleware_offsite.py E501 E128 E111 + tests/test_spidermiddleware_offsite.py E501 E128 tests/test_spidermiddleware_output_chain.py E501 tests/test_spidermiddleware_referer.py E501 F841 E501 E121 tests/test_squeues.py E501 E741 diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index a08955dc9..85a9bb526 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -65,9 +65,9 @@ class Selector(_ParselSelector, object_ref): selectorlist_cls = SelectorList def __init__(self, response=None, text=None, type=None, root=None, **kwargs): - if not(response is None or text is None): - raise ValueError('%s.__init__() received both response and text' - % self.__class__.__name__) + if response is not None and text is not None: + raise ValueError('%s.__init__() received both response and text' + % self.__class__.__name__) st = _st(response, type or self._default_type) From 1a157f2e26274455405b7e272bab5aede0fa59fa Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 8 May 2020 16:27:21 -0300 Subject: [PATCH 41/50] Flake8: remove E116 --- pytest.ini | 4 ++-- scrapy/pipelines/files.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pytest.ini b/pytest.ini index 20639baa3..295609435 100644 --- a/pytest.ini +++ b/pytest.ini @@ -97,7 +97,7 @@ flake8-ignore = scrapy/loader/processors.py E501 # scrapy/pipelines scrapy/pipelines/__init__.py E501 - scrapy/pipelines/files.py E116 E501 + scrapy/pipelines/files.py E501 scrapy/pipelines/images.py E501 scrapy/pipelines/media.py E501 # scrapy/selector @@ -105,7 +105,7 @@ flake8-ignore = scrapy/selector/unified.py E501 # scrapy/settings scrapy/settings/__init__.py E501 - scrapy/settings/default_settings.py E501 E114 E116 + scrapy/settings/default_settings.py E501 E114 scrapy/settings/deprecated.py E501 # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index a9066986b..cd3e29057 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -83,8 +83,7 @@ class S3FilesStore: AWS_USE_SSL = None AWS_VERIFY = None - POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in - # FilesPipeline.from_settings. + POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings HEADERS = { 'Cache-Control': 'max-age=172800', } From 83ce82f400d9686f7fbaa526f21f635fca8491de Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 8 May 2020 16:28:26 -0300 Subject: [PATCH 42/50] Flake8: remove E114 and E117 (unused) --- pytest.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 295609435..59cce9ac4 100644 --- a/pytest.ini +++ b/pytest.ini @@ -105,7 +105,7 @@ flake8-ignore = scrapy/selector/unified.py E501 # scrapy/settings scrapy/settings/__init__.py E501 - scrapy/settings/default_settings.py E501 E114 + scrapy/settings/default_settings.py E501 scrapy/settings/deprecated.py E501 # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 @@ -207,7 +207,7 @@ flake8-ignore = tests/test_item.py E128 F841 tests/test_link.py E501 tests/test_linkextractors.py E501 E128 - tests/test_loader.py E501 E741 E128 E117 + tests/test_loader.py E501 E741 E128 tests/test_logformatter.py E128 E501 tests/test_mail.py E128 E501 tests/test_middleware.py E501 E128 From c2c3054ac13838b89ad5062d6583cc52dd0ba317 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 8 May 2020 16:32:02 -0300 Subject: [PATCH 43/50] Flake8: remove E121 --- pytest.ini | 4 ++-- tests/test_spidermiddleware_httperror.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pytest.ini b/pytest.ini index 59cce9ac4..61139f7fe 100644 --- a/pytest.ini +++ b/pytest.ini @@ -223,10 +223,10 @@ flake8-ignore = tests/test_selector.py E501 tests/test_spider.py E501 tests/test_spidermiddleware.py E501 - tests/test_spidermiddleware_httperror.py E128 E501 E121 + tests/test_spidermiddleware_httperror.py E128 E501 tests/test_spidermiddleware_offsite.py E501 E128 tests/test_spidermiddleware_output_chain.py E501 - tests/test_spidermiddleware_referer.py E501 F841 E501 E121 + tests/test_spidermiddleware_referer.py E501 F841 E501 tests/test_squeues.py E501 E741 tests/test_utils_asyncio.py E501 tests/test_utils_conf.py E501 E128 diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 6b61df56f..29584f21b 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -21,10 +21,10 @@ class _HttpErrorSpider(MockServerSpider): def __init__(self, *args, **kwargs): super(_HttpErrorSpider, self).__init__(*args, **kwargs) self.start_urls = [ - self.mockserver.url("/status?n=200"), - self.mockserver.url("/status?n=404"), - self.mockserver.url("/status?n=402"), - self.mockserver.url("/status?n=500"), + self.mockserver.url("/status?n=200"), + self.mockserver.url("/status?n=404"), + self.mockserver.url("/status?n=402"), + self.mockserver.url("/status?n=500"), ] self.failed = set() self.skipped = set() From 6f8758624c8d3df7472948ce9805601f6037548a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 11 May 2020 13:50:34 -0300 Subject: [PATCH 44/50] Flake8: remove F841 --- pytest.ini | 22 ++++++++++------------ tests/pipelines.py | 2 +- tests/test_crawler.py | 6 +++--- tests/test_dependencies.py | 2 +- tests/test_extension_telnet.py | 2 -- tests/test_feedexport.py | 1 - tests/test_item.py | 8 ++++---- tests/test_pipeline_images.py | 2 +- tests/test_spidermiddleware_referer.py | 3 +-- tests/test_utils_defer.py | 4 ++-- tests/test_utils_deprecate.py | 4 ++-- tests/test_utils_signal.py | 2 +- 12 files changed, 26 insertions(+), 32 deletions(-) diff --git a/pytest.ini b/pytest.ini index 32fd76445..fee54dcbd 100644 --- a/pytest.ini +++ b/pytest.ini @@ -164,14 +164,13 @@ flake8-ignore = scrapy/robotstxt.py E501 scrapy/shell.py E501 scrapy/signalmanager.py E501 - scrapy/spiderloader.py F841 E501 + scrapy/spiderloader.py E501 scrapy/squeues.py E128 scrapy/squeues.py E501 scrapy/statscollectors.py E501 # tests tests/__init__.py E402 E501 tests/mockserver.py E501 - tests/pipelines.py F841 tests/spiders.py E501 tests/test_closespider.py E501 tests/test_command_fetch.py E501 @@ -180,8 +179,8 @@ flake8-ignore = tests/test_commands.py E128 E501 tests/test_contracts.py E501 E128 tests/test_crawl.py E501 E741 - tests/test_crawler.py F841 E501 - tests/test_dependencies.py F841 E501 + tests/test_crawler.py E501 + tests/test_dependencies.py E501 tests/test_downloader_handlers.py E128 E501 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 @@ -199,13 +198,12 @@ flake8-ignore = tests/test_dupefilters.py E501 E741 E128 tests/test_engine.py E501 E128 tests/test_exporters.py E501 E128 - tests/test_extension_telnet.py F841 - tests/test_feedexport.py E501 F841 + tests/test_feedexport.py E501 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 tests/test_http_request.py E402 E501 E128 E128 tests/test_http_response.py E501 E128 - tests/test_item.py E128 F841 + tests/test_item.py E128 tests/test_link.py E501 tests/test_linkextractors.py E501 E128 tests/test_loader.py E501 E741 E128 @@ -214,7 +212,7 @@ flake8-ignore = tests/test_middleware.py E501 E128 tests/test_pipeline_crawl.py E501 E128 tests/test_pipeline_files.py E501 - tests/test_pipeline_images.py F841 E501 + tests/test_pipeline_images.py E501 tests/test_pipeline_media.py E501 E741 E128 tests/test_proxy_connect.py E501 E741 tests/test_request_cb_kwargs.py E501 @@ -227,14 +225,14 @@ flake8-ignore = tests/test_spidermiddleware_httperror.py E128 E501 tests/test_spidermiddleware_offsite.py E501 E128 tests/test_spidermiddleware_output_chain.py E501 - tests/test_spidermiddleware_referer.py E501 F841 E501 + tests/test_spidermiddleware_referer.py E501 E501 tests/test_squeues.py E501 E741 tests/test_utils_asyncio.py E501 tests/test_utils_conf.py E501 E128 tests/test_utils_curl.py E501 tests/test_utils_datatypes.py E402 E501 - tests/test_utils_defer.py E501 F841 - tests/test_utils_deprecate.py F841 E501 + tests/test_utils_defer.py E501 + tests/test_utils_deprecate.py E501 tests/test_utils_http.py E501 E128 tests/test_utils_iterators.py E501 E128 tests/test_utils_log.py E741 @@ -242,7 +240,7 @@ flake8-ignore = tests/test_utils_reqser.py E501 E128 tests/test_utils_request.py E501 E128 tests/test_utils_response.py E501 - tests/test_utils_signal.py E741 F841 + tests/test_utils_signal.py E741 tests/test_utils_sitemap.py E128 E501 tests/test_utils_url.py E501 E501 tests/test_webclient.py E501 E128 E402 diff --git a/tests/pipelines.py b/tests/pipelines.py index cf677cc17..fed2af7d3 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -6,7 +6,7 @@ Some pipelines used for testing class ZeroDivisionErrorPipeline: def open_spider(self, spider): - a = 1 / 0 + 1 / 0 def process_item(self, item, spider): return item diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 9151278a5..ecc0cd7af 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -87,7 +87,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): class MySpider(scrapy.Spider): name = 'spider' - crawler = Crawler(MySpider, {}) + Crawler(MySpider, {}) assert get_scrapy_root_handler() is None def test_spider_custom_settings_log_level(self): @@ -240,13 +240,13 @@ class CrawlerRunnerHasSpider(unittest.TestCase): def test_crawler_runner_asyncio_enabled_true(self): if self.reactor_pytest == 'asyncio': - runner = CrawlerRunner(settings={ + CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): - runner = CrawlerRunner(settings={ + CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index a169acbe6..5d0a1d0c9 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -6,7 +6,7 @@ class ScrapyUtilsTest(unittest.TestCase): def test_required_openssl_version(self): try: module = import_module('OpenSSL') - except ImportError as ex: + except ImportError: raise unittest.SkipTest("OpenSSL is not available") if hasattr(module, '__version__'): diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 873a97248..1e716b94a 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -11,8 +11,6 @@ class TelnetExtensionTest(unittest.TestCase): def _get_console_and_portal(self, settings=None): crawler = get_crawler(settings_dict=settings) console = TelnetConsole(crawler) - username = console.username - password = console.password # This function has some side effects we don't need for this test console._get_telnet_vars = lambda: {} diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e02b0b840..cbc81bc35 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -715,7 +715,6 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def test_export_encoding(self): items = [dict({'foo': u'Test\xd6'})] - header = ['foo'] formats = { 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'), diff --git a/tests/test_item.py b/tests/test_item.py index 4017f6e84..58735cc5c 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -264,12 +264,12 @@ class ItemTest(unittest.TestCase): """Make sure the DictItem deprecation warning is not issued for Item""" with catch_warnings(record=True) as warnings: - item = Item() + Item() self.assertEqual(len(warnings), 0) class SubclassedItem(Item): pass - subclassed_item = SubclassedItem() + SubclassedItem() self.assertEqual(len(warnings), 0) @@ -321,13 +321,13 @@ class DictItemTest(unittest.TestCase): def test_deprecation_warning(self): with catch_warnings(record=True) as warnings: - dict_item = DictItem() + DictItem() self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) with catch_warnings(record=True) as warnings: class SubclassedDictItem(DictItem): pass - subclassed_dict_item = SubclassedDictItem() + SubclassedDictItem() self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 5018d6802..5ba03ff4c 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -15,7 +15,7 @@ from scrapy.utils.python import to_bytes skip = False try: from PIL import Image -except ImportError as e: +except ImportError: skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' else: encoders = set(('jpeg_encoder', 'jpeg_decoder')) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 41589177a..ca765518b 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -459,7 +459,6 @@ class TestRequestMetaSettingFallback(TestCase): target = 'http://www.example.com' for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]: - spider = Spider('foo') mw = RefererMiddleware(Settings(settings)) response = Response(origin, headers=response_headers) @@ -511,7 +510,7 @@ class TestSettingsPolicyByName(TestCase): def test_invalid_name(self): settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'}) with self.assertRaises(RuntimeError): - mw = RefererMiddleware(settings) + RefererMiddleware(settings) class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index a3b6e64f1..2d4b88121 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -64,7 +64,7 @@ class DeferUtilsTest(unittest.TestCase): gotexc = False try: yield process_chain([cb1, cb_fail, cb3], 'res', 'v1', 'v2') - except TypeError as e: + except TypeError: gotexc = True self.assertTrue(gotexc) @@ -104,7 +104,7 @@ class IterErrbackTest(unittest.TestCase): def iterbad(): for x in range(10): if x == 5: - a = 1 / 0 + 1 / 0 yield x errors = [] diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index adef66c1d..35d35b45d 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -25,7 +25,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): def test_no_warning_on_definition(self): with warnings.catch_warnings(record=True) as w: - Deprecated = create_deprecated_class('Deprecated', NewName) + create_deprecated_class('Deprecated', NewName) w = self._mywarnings(w) self.assertEqual(w, []) @@ -217,7 +217,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): def test_deprecate_a_class_with_custom_metaclass(self): Meta1 = type('Meta1', (type,), {}) New = Meta1('New', (), {}) - Deprecated = create_deprecated_class('Deprecated', New) + create_deprecated_class('Deprecated', New) def test_deprecate_subclass_of_deprecated_class(self): with warnings.catch_warnings(record=True) as w: diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index bb211dc60..c83c9398c 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -44,7 +44,7 @@ class SendCatchLogTest(unittest.TestCase): def error_handler(self, arg, handlers_called): handlers_called.add(self.error_handler) - a = 1 / 0 + 1 / 0 def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) From cb8140a42a27ede87b0880372024f2f1804618b8 Mon Sep 17 00:00:00 2001 From: nsirletti <40069643+nsirletti@users.noreply.github.com> Date: Mon, 11 May 2020 20:20:31 +0200 Subject: [PATCH 45/50] Deprecate Response.body_as_unicode() (#4555) Co-authored-by: Nicolas Sirletti --- docs/topics/request-response.rst | 5 ----- scrapy/http/response/text.py | 5 +++++ tests/test_http_response.py | 9 +++++++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 024f46466..15a83f453 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -834,11 +834,6 @@ TextResponse objects .. automethod:: TextResponse.follow_all - .. method:: TextResponse.body_as_unicode() - - The same as :attr:`text`, but available as a method. This method is - kept for backward compatibility; please prefer ``response.text``. - HtmlResponse objects -------------------- diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 2f0f3820c..5614e6e55 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -5,6 +5,7 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ +import warnings from contextlib import suppress from typing import Generator from urllib.parse import urljoin @@ -14,6 +15,7 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode, http_content_type_encoding, resolve_encoding) from w3lib.html import strip_html5_whitespace +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs, to_unicode @@ -61,6 +63,9 @@ class TextResponse(Response): def body_as_unicode(self): """Return body as unicode""" + warnings.warn('Response.body_as_unicode() is deprecated, ' + 'please use Response.text instead.', + ScrapyDeprecationWarning) return self.text @property diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 43d6d936a..2f73afe56 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,7 +1,9 @@ import unittest +from warnings import catch_warnings from w3lib.encoding import resolve_encoding +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import (Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers) from scrapy.selector import Selector @@ -660,6 +662,13 @@ class TextResponseTest(BaseResponseTest): with self.assertRaises(ValueError): response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]') + def test_body_as_unicode_deprecation_warning(self): + with catch_warnings(record=True) as warnings: + r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8') + self.assertEqual(r1.body_as_unicode(), u'Hello') + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + class HtmlResponseTest(TextResponseTest): From cf9be5344a89dd8e14f8241ec69de9c984ec1e05 Mon Sep 17 00:00:00 2001 From: willbeaufoy Date: Mon, 11 May 2020 19:35:25 +0100 Subject: [PATCH 46/50] Prevent create_instance() returning None (#4532) Currently create_instance() can return None if an extension is incorrectly implemented, but the extension will still show up as enabled in the logs. This can cause confusion, as in the linked bug. This change prevents this occurring by throwing an error if create_instance() will return None. --- scrapy/utils/misc.py | 15 ++++++++++++--- tests/test_utils_misc/__init__.py | 14 +++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 52cfba208..ab7cf9deb 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -137,17 +137,26 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): ``*args`` and ``**kwargs`` are forwarded to the constructors. Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``. + + Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an + extension has not been implemented correctly). """ if settings is None: if crawler is None: raise ValueError("Specify at least one of settings and crawler.") settings = crawler.settings if crawler and hasattr(objcls, 'from_crawler'): - return objcls.from_crawler(crawler, *args, **kwargs) + instance = objcls.from_crawler(crawler, *args, **kwargs) + method_name = 'from_crawler' elif hasattr(objcls, 'from_settings'): - return objcls.from_settings(settings, *args, **kwargs) + instance = objcls.from_settings(settings, *args, **kwargs) + method_name = 'from_settings' else: - return objcls(*args, **kwargs) + instance = objcls(*args, **kwargs) + method_name = '__new__' + if instance is None: + raise TypeError("%s.%s returned None" % (objcls.__qualname__, method_name)) + return instance @contextmanager diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 6f945cd01..015a0e5a2 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -114,8 +114,12 @@ class UtilsMiscTestCase(unittest.TestCase): # 2. with from_settings() constructor # 3. with from_crawler() constructor # 4. with from_settings() and from_crawler() constructor - spec_sets = ([], ['from_settings'], ['from_crawler'], - ['from_settings', 'from_crawler']) + spec_sets = ( + ['__qualname__'], + ['__qualname__', 'from_settings'], + ['__qualname__', 'from_crawler'], + ['__qualname__', 'from_settings', 'from_crawler'], + ) for specs in spec_sets: m = mock.MagicMock(spec_set=specs) _test_with_settings(m, settings) @@ -123,7 +127,7 @@ class UtilsMiscTestCase(unittest.TestCase): _test_with_crawler(m, settings, crawler) # Check adoption of crawler settings - m = mock.MagicMock(spec_set=['from_settings']) + m = mock.MagicMock(spec_set=['__qualname__', 'from_settings']) create_instance(m, None, crawler, *args, **kwargs) m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs) @@ -131,6 +135,10 @@ class UtilsMiscTestCase(unittest.TestCase): with self.assertRaises(ValueError): create_instance(m, None, None) + m.from_settings.return_value = None + with self.assertRaises(TypeError): + create_instance(m, settings, None) + def test_set_environ(self): assert os.environ.get('some_test_environ') is None with set_environ(some_test_environ='test_value'): From 97532a9144cc011e5a6ede84fa7d317c3af60131 Mon Sep 17 00:00:00 2001 From: Aditya Kumar Date: Tue, 12 May 2020 20:40:09 +0530 Subject: [PATCH 47/50] test(spiderloader): no duplicate spider names (#4560) --- tests/test_spiderloader/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index d8be6e277..b6fb27ffe 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -137,6 +137,11 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): msg = str(w[0].message) self.assertIn("several spiders with the same name", msg) self.assertIn("'spider3'", msg) + self.assertTrue(msg.count("'spider3'") == 2) + + self.assertNotIn("'spider1'", msg) + self.assertNotIn("'spider2'", msg) + self.assertNotIn("'spider4'", msg) spiders = set(spider_loader.list()) self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) @@ -156,7 +161,13 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): msg = str(w[0].message) self.assertIn("several spiders with the same name", msg) self.assertIn("'spider1'", msg) + self.assertTrue(msg.count("'spider1'") == 2) + self.assertIn("'spider2'", msg) + self.assertTrue(msg.count("'spider2'") == 2) + + self.assertNotIn("'spider3'", msg) + self.assertNotIn("'spider4'", msg) spiders = set(spider_loader.list()) self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) From 8d1269bcbc81fa0bb5a69068e07bdbcb0dba8889 Mon Sep 17 00:00:00 2001 From: Aditya Kumar Date: Wed, 13 May 2020 00:12:28 +0530 Subject: [PATCH 48/50] Cover chompjs in documentation (#4562) --- docs/topics/dynamic-content.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 3b85bfe8a..495111b56 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -184,6 +184,18 @@ data from it: >>> json.loads(json_data) {'field': 'value'} +- chompjs_ provides an API to parse JavaScript objects into a :class:`dict`. + + For example, if the JavaScript code contains + ``var data = {field: "value", secondField: "second value"};`` + you can extract that data as follows: + + >>> import chompjs + >>> javascript = response.css('script::text').get() + >>> data = chompjs.parse_js_object(javascript) + >>> data + {'field': 'value', 'secondField': 'second value'} + - Otherwise, use js2xml_ to convert the JavaScript code into an XML document that you can parse using :ref:`selectors `. @@ -241,6 +253,7 @@ along with `scrapy-selenium`_ for seamless integration. .. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29 +.. _chompjs: https://github.com/Nykakin/chompjs .. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets .. _curl: https://curl.haxx.se/ .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser From 8971878c1d4d95aff10a1e910688502de892662b Mon Sep 17 00:00:00 2001 From: marc Date: Wed, 13 May 2020 14:11:10 +0200 Subject: [PATCH 49/50] fix new detected flake8 cases --- scrapy/core/spidermw.py | 2 +- scrapy/utils/console.py | 2 ++ scrapy/utils/python.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 87d08cab7..35264a92b 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -19,7 +19,7 @@ def _isiterable(possible_iterator): def _fname(f): - return "%s.%s".format( + return "{}.{}".format( f.__self__.__class__.__name__, f.__func__.__name__ ) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index c7a2ace88..133261fd7 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -28,6 +28,7 @@ def _embed_ipython_shell(namespace={}, banner=''): def _embed_bpython_shell(namespace={}, banner=''): """Start a bpython shell""" import bpython + @wraps(_embed_bpython_shell) def wrapper(namespace=namespace, banner=''): bpython.embed(locals_=namespace, banner=banner) @@ -37,6 +38,7 @@ def _embed_bpython_shell(namespace={}, banner=''): def _embed_ptpython_shell(namespace={}, banner=''): """Start a ptpython shell""" import ptpython.repl + @wraps(_embed_ptpython_shell) def wrapper(namespace=namespace, banner=''): print(banner) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 3d02d9478..9c1f3c2fe 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -152,6 +152,7 @@ def memoizemethod_noargs(method): weak reference to its object """ cache = weakref.WeakKeyDictionary() + @wraps(method) def new_method(self, *args, **kwargs): if self not in cache: From df8a1d1c0108b15bebdad064d8b7bc61a894f062 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 8 May 2020 20:09:35 -0300 Subject: [PATCH 50/50] Flake8: Remove E741 --- pytest.ini | 18 +++++------ scrapy/shell.py | 5 ++- scrapy/spiders/sitemap.py | 3 +- scrapy/utils/log.py | 2 +- tests/test_command_version.py | 6 ++-- tests/test_crawl.py | 36 +++++++++++----------- tests/test_downloadermiddleware_cookies.py | 8 ++--- tests/test_dupefilters.py | 10 +++--- tests/test_pipeline_media.py | 10 +++--- tests/test_proxy_connect.py | 16 +++++----- tests/test_squeues.py | 24 +++++++-------- tests/test_utils_log.py | 24 ++++++--------- tests/test_utils_misc/__init__.py | 4 +-- tests/test_utils_signal.py | 12 ++++---- 14 files changed, 86 insertions(+), 92 deletions(-) diff --git a/pytest.ini b/pytest.ini index fee54dcbd..c141d49a4 100644 --- a/pytest.ini +++ b/pytest.ini @@ -178,13 +178,13 @@ flake8-ignore = tests/test_command_shell.py E501 E128 tests/test_commands.py E128 E501 tests/test_contracts.py E501 E128 - tests/test_crawl.py E501 E741 + tests/test_crawl.py E501 tests/test_crawler.py E501 tests/test_dependencies.py E501 tests/test_downloader_handlers.py E128 E501 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 - tests/test_downloadermiddleware_cookies.py E741 E501 E128 + tests/test_downloadermiddleware_cookies.py E501 E128 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 tests/test_downloadermiddleware_httpcache.py E501 @@ -195,7 +195,7 @@ flake8-ignore = tests/test_downloadermiddleware_retry.py E501 E128 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E501 E741 E128 + tests/test_dupefilters.py E501 E128 tests/test_engine.py E501 E128 tests/test_exporters.py E501 E128 tests/test_feedexport.py E501 @@ -206,15 +206,15 @@ flake8-ignore = tests/test_item.py E128 tests/test_link.py E501 tests/test_linkextractors.py E501 E128 - tests/test_loader.py E501 E741 E128 + tests/test_loader.py E128 E501 E741 tests/test_logformatter.py E128 E501 tests/test_mail.py E128 E501 tests/test_middleware.py E501 E128 tests/test_pipeline_crawl.py E501 E128 tests/test_pipeline_files.py E501 tests/test_pipeline_images.py E501 - tests/test_pipeline_media.py E501 E741 E128 - tests/test_proxy_connect.py E501 E741 + tests/test_pipeline_media.py E501 E128 + tests/test_proxy_connect.py E501 tests/test_request_cb_kwargs.py E501 tests/test_responsetypes.py E501 tests/test_robotstxt_interface.py E501 E501 @@ -225,8 +225,8 @@ flake8-ignore = tests/test_spidermiddleware_httperror.py E128 E501 tests/test_spidermiddleware_offsite.py E501 E128 tests/test_spidermiddleware_output_chain.py E501 - tests/test_spidermiddleware_referer.py E501 E501 - tests/test_squeues.py E501 E741 + tests/test_spidermiddleware_referer.py E501 + tests/test_squeues.py E501 tests/test_utils_asyncio.py E501 tests/test_utils_conf.py E501 E128 tests/test_utils_curl.py E501 @@ -235,12 +235,10 @@ flake8-ignore = tests/test_utils_deprecate.py E501 tests/test_utils_http.py E501 E128 tests/test_utils_iterators.py E501 E128 - tests/test_utils_log.py E741 tests/test_utils_python.py E501 tests/test_utils_reqser.py E501 E128 tests/test_utils_request.py E501 E128 tests/test_utils_response.py E501 - tests/test_utils_signal.py E741 tests/test_utils_sitemap.py E128 E501 tests/test_utils_url.py E501 E501 tests/test_webclient.py E501 E128 E402 diff --git a/scrapy/shell.py b/scrapy/shell.py index 08ce89481..2a3e13ddd 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -146,14 +146,13 @@ class Shell: b.append("Useful shortcuts:") if self.inthread: b.append(" fetch(url[, redirect=True]) " - "Fetch URL and update local objects " - "(by default, redirects are followed)") + "Fetch URL and update local objects (by default, redirects are followed)") b.append(" fetch(req) " "Fetch a scrapy.Request and update local objects ") b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") - return "\n".join("[s] %s" % l for l in b) + return "\n".join("[s] %s" % line for line in b) def _is_relevant(self, value): return isinstance(value, self.relevant_classes) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index d368c7108..c5360bfa7 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -96,5 +96,4 @@ def iterloc(it, alt=False): # Also consider alternate URLs (xhtml:link rel="alternate") if alt and 'alternate' in d: - for l in d['alternate']: - yield l + yield from d['alternate'] diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 203842fc8..6392862ee 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -37,7 +37,7 @@ class TopLevelFormatter(logging.Filter): self.loggers = loggers or [] def filter(self, record): - if any(record.name.startswith(l + '.') for l in self.loggers): + if any(record.name.startswith(logger + '.') for logger in self.loggers): record.name = record.name.split('.', 1)[0] return True diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 4ac7fb786..99c01c2b7 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -23,8 +23,10 @@ class VersionTest(ProcessTest, unittest.TestCase): def test_verbose_output(self): encoding = getattr(sys.stdout, 'encoding') or 'utf-8' _, out, _ = yield self.execute(['-v']) - headers = [l.partition(":")[0].strip() - for l in out.strip().decode(encoding).splitlines()] + headers = [ + line.partition(":")[0].strip() + for line in out.strip().decode(encoding).splitlines() + ] self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', 'cssselect', 'parsel', 'w3lib', 'Twisted', 'Python', 'pyOpenSSL', diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 4215ca56c..84f80d103 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -104,44 +104,44 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_retry_503(self): crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_retry_conn_failed(self): crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_retry_dns_error(self): crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: # try to fetch the homepage of a non-existent domain yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): - with LogCapture('scrapy', level=logging.ERROR) as l: + with LogCapture('scrapy', level=logging.ERROR) as log: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) - self.assertEqual(len(l.records), 1) - record = l.records[0] + self.assertEqual(len(log.records), 1) + record = log.records[0] self.assertIsNotNone(record.exc_info) self.assertIs(record.exc_info[0], ZeroDivisionError) @defer.inlineCallbacks def test_start_requests_bug_yielding(self): - with LogCapture('scrapy', level=logging.ERROR) as l: + with LogCapture('scrapy', level=logging.ERROR) as log: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) - self.assertEqual(len(l.records), 1) - record = l.records[0] + self.assertEqual(len(log.records), 1) + record = log.records[0] self.assertIsNotNone(record.exc_info) self.assertIs(record.exc_info[0], ZeroDivisionError) @@ -187,25 +187,25 @@ foo body with multiples lines '''}) crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver) - self.assertEqual(str(l).count("Got response 200"), 1) + self.assertEqual(str(log).count("Got response 200"), 1) @defer.inlineCallbacks def test_retry_conn_lost(self): # connection lost after receiving data crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_retry_conn_aborted(self): # connection lost before receiving data crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) def _assert_retried(self, log): self.assertEqual(str(log).count("Retrying"), 2) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index f86c50f50..d54434c8f 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -63,7 +63,7 @@ class CookiesMiddlewareTest(TestCase): mw = CookiesMiddleware.from_crawler(crawler) with LogCapture('scrapy.downloadermiddlewares.cookies', propagate=False, - level=logging.DEBUG) as l: + level=logging.DEBUG) as log: req = Request('http://scrapytest.org/') res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) @@ -71,7 +71,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request('http://scrapytest.org/sub1/') mw.process_request(req2, crawler.spider) - l.check( + log.check( ('scrapy.downloadermiddlewares.cookies', 'DEBUG', 'Received cookies from: <200 http://scrapytest.org/>\n' @@ -87,7 +87,7 @@ class CookiesMiddlewareTest(TestCase): mw = CookiesMiddleware.from_crawler(crawler) with LogCapture('scrapy.downloadermiddlewares.cookies', propagate=False, - level=logging.DEBUG) as l: + level=logging.DEBUG) as log: req = Request('http://scrapytest.org/') res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) @@ -95,7 +95,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request('http://scrapytest.org/sub1/') mw.process_request(req2, crawler.spider) - l.check() + log.check() def test_do_not_break_on_non_utf8_header(self): req = Request('http://scrapytest.org/') diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 7426107c1..ee1794f4f 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -160,7 +160,7 @@ class RFPDupeFilterTest(unittest.TestCase): shutil.rmtree(path) def test_log(self): - with LogCapture() as l: + with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': False, 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(SimpleSpider, settings_dict=settings) @@ -177,7 +177,7 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.log(r2, spider) assert crawler.stats.get_value('dupefilter/filtered') == 2 - l.check_present(('scrapy.dupefilters', 'DEBUG', + log.check_present(('scrapy.dupefilters', 'DEBUG', ('Filtered duplicate request: ' ' - no more duplicates will be shown' ' (see DUPEFILTER_DEBUG to show all duplicates)'))) @@ -185,7 +185,7 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close('finished') def test_log_debug(self): - with LogCapture() as l: + with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(SimpleSpider, settings_dict=settings) @@ -203,10 +203,10 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.log(r2, spider) assert crawler.stats.get_value('dupefilter/filtered') == 2 - l.check_present(('scrapy.dupefilters', 'DEBUG', + log.check_present(('scrapy.dupefilters', 'DEBUG', ('Filtered duplicate request: ' ' (referer: None)'))) - l.check_present(('scrapy.dupefilters', 'DEBUG', + log.check_present(('scrapy.dupefilters', 'DEBUG', ('Filtered duplicate request: ' ' (referer: http://scrapytest.org/INDEX.html)'))) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 949f0dea1..d7b0d32b2 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -63,21 +63,21 @@ class BaseMediaPipelineTestCase(unittest.TestCase): fail = Failure(Exception()) results = [(True, 1), (False, fail)] - with LogCapture() as l: + with LogCapture() as log: new_item = self.pipe.item_completed(results, item, self.info) assert new_item is item - assert len(l.records) == 1 - record = l.records[0] + assert len(log.records) == 1 + record = log.records[0] assert record.levelname == 'ERROR' self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail)) # disable failure logging and check again self.pipe.LOG_FAILED_RESULTS = False - with LogCapture() as l: + with LogCapture() as log: new_item = self.pipe.item_completed(results, item, self.info) assert new_item is item - assert len(l.records) == 0 + assert len(log.records) == 0 @inlineCallbacks def test_default_process_item(self): diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 188ec68dd..4763a5417 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -76,35 +76,35 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_connect_tunnel(self): crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) + self._assert_got_response_code(200, log) @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6) @defer.inlineCallbacks def test_https_connect_tunnel_error(self): crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl("https://localhost:99999/status?n=200") - self._assert_got_tunnel_error(l) + self._assert_got_tunnel_error(log) @defer.inlineCallbacks def test_https_tunnel_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) # The proxy returns a 407 error code but it does not reach the client; # he just sees a TunnelError. - self._assert_got_tunnel_error(l) + self._assert_got_tunnel_error(log) @defer.inlineCallbacks def test_https_tunnel_without_leak_proxy_authorization_header(self): request = Request(self.mockserver.url("/echo", is_secure=True)) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(seed=request) - self._assert_got_response_code(200, l) + self._assert_got_response_code(200, log) echo = json.loads(crawler.spider.meta['responses'][0].text) self.assertTrue('Proxy-Authorization' not in echo['headers']) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index d2cf9135f..becacce62 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -89,12 +89,12 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): def test_serialize_loader(self): q = self.queue() - l = TestLoader() - q.push(l) - l2 = q.pop() - assert isinstance(l2, TestLoader) - assert l2.default_item_class is TestItem - self.assertEqual(l2.name_out('x'), 'xx') + loader = TestLoader() + q.push(loader) + loader2 = q.pop() + assert isinstance(loader2, TestLoader) + assert loader2.default_item_class is TestItem + self.assertEqual(loader2.name_out('x'), 'xx') def test_serialize_request_recursive(self): q = self.queue() @@ -173,12 +173,12 @@ class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def test_serialize_loader(self): q = self.queue() - l = TestLoader() - q.push(l) - l2 = q.pop() - assert isinstance(l2, TestLoader) - assert l2.default_item_class is TestItem - self.assertEqual(l2.name_out('x'), 'xx') + loader = TestLoader() + q.push(loader) + loader2 = q.pop() + assert isinstance(loader2, TestLoader) + assert loader2.default_item_class is TestItem + self.assertEqual(loader2.name_out('x'), 'xx') def test_serialize_request_recursive(self): q = self.queue() diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 25cd904bc..535f56691 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -34,31 +34,27 @@ class TopLevelFormatterTest(unittest.TestCase): def test_top_level_logger(self): logger = logging.getLogger('test') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('test', 'WARNING', 'test log msg')) + log.check(('test', 'WARNING', 'test log msg')) def test_children_logger(self): logger = logging.getLogger('test.test1') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('test', 'WARNING', 'test log msg')) + log.check(('test', 'WARNING', 'test log msg')) def test_overlapping_name_logger(self): logger = logging.getLogger('test2') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('test2', 'WARNING', 'test log msg')) + log.check(('test2', 'WARNING', 'test log msg')) def test_different_name_logger(self): logger = logging.getLogger('different') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('different', 'WARNING', 'test log msg')) + log.check(('different', 'WARNING', 'test log msg')) class LogCounterHandlerTest(unittest.TestCase): @@ -107,6 +103,6 @@ class StreamLoggerTest(unittest.TestCase): sys.stdout = self.stdout def test_redirect(self): - with LogCapture() as l: + with LogCapture() as log: print('test log msg') - l.check(('test', 'ERROR', 'test log msg')) + log.check(('test', 'ERROR', 'test log msg')) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 015a0e5a2..28205e0d9 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -67,12 +67,12 @@ class UtilsMiscTestCase(unittest.TestCase): assert hasattr(arg_to_iter(100), '__iter__') assert hasattr(arg_to_iter('lala'), '__iter__') assert hasattr(arg_to_iter([1, 2, 3]), '__iter__') - assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__') + assert hasattr(arg_to_iter(c for c in 'abcd'), '__iter__') self.assertEqual(list(arg_to_iter(None)), []) self.assertEqual(list(arg_to_iter('lala')), ['lala']) self.assertEqual(list(arg_to_iter(100)), [100]) - self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c']) + self.assertEqual(list(arg_to_iter(c for c in 'abc')), ['a', 'b', 'c']) self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3]) self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}]) self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")]) diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index c83c9398c..b66588efb 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -20,7 +20,7 @@ class SendCatchLogTest(unittest.TestCase): dispatcher.connect(self.error_handler, signal=test_signal) dispatcher.connect(self.ok_handler, signal=test_signal) - with LogCapture() as l: + with LogCapture() as log: result = yield defer.maybeDeferred( self._get_result, test_signal, arg='test', handlers_called=handlers_called @@ -28,8 +28,8 @@ class SendCatchLogTest(unittest.TestCase): assert self.error_handler in handlers_called assert self.ok_handler in handlers_called - self.assertEqual(len(l.records), 1) - record = l.records[0] + self.assertEqual(len(log.records), 1) + record = log.records[0] self.assertIn('error_handler', record.getMessage()) self.assertEqual(record.levelname, 'ERROR') self.assertEqual(result[0][0], self.error_handler) @@ -95,8 +95,8 @@ class SendCatchLogTest2(unittest.TestCase): test_signal = object() dispatcher.connect(test_handler, test_signal) - with LogCapture() as l: + with LogCapture() as log: send_catch_log(test_signal) - self.assertEqual(len(l.records), 1) - self.assertIn("Cannot return deferreds from signal handler", str(l)) + self.assertEqual(len(log.records), 1) + self.assertIn("Cannot return deferreds from signal handler", str(log)) dispatcher.disconnect(test_handler, test_signal)