From 3b6f7ac9f2f5b48b9f2f3ce106d1205599d2164f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 22 Oct 2019 19:43:02 +0200 Subject: [PATCH 001/281] Use pylint --- .travis.yml | 2 + docs/utils/linkfix.py | 85 ++++++++++++++++++++++------------------- pylintrc | 88 +++++++++++++++++++++++++++++++++++++++++++ tox.ini | 14 +++++++ 4 files changed, 150 insertions(+), 39 deletions(-) create mode 100644 pylintrc diff --git a/.travis.yml b/.travis.yml index 0190a7f4d..28a19f4f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,8 @@ branches: - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: include: + - env: TOXENV=pylint + python: 3.7 - env: TOXENV=py27 python: 2.7 - env: TOXENV=py27-pinned diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 6290adbe2..9acfc3b23 100755 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -14,50 +14,57 @@ Author: dufferzafar import re -# Used for remembering the file (and its contents) -# so we don't have to open the same file again. -_filename = None -_contents = None -# A regex that matches standard linkcheck output lines -line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') +def main(): -# Read lines from the linkcheck output file -try: - with open("build/linkcheck/output.txt") as out: - output_lines = out.readlines() -except IOError: - print("linkcheck output not found; please run linkcheck first.") - exit(1) + # Used for remembering the file (and its contents) + # so we don't have to open the same file again. + _filename = None + _contents = None -# For every line, fix the respective file -for line in output_lines: - match = re.match(line_re, line) + # A regex that matches standard linkcheck output lines + line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') - if match: - newfilename = match.group(1) - errortype = match.group(2) + # Read lines from the linkcheck output file + try: + with open("build/linkcheck/output.txt") as out: + output_lines = out.readlines() + except IOError: + print("linkcheck output not found; please run linkcheck first.") + exit(1) - # Broken links can't be fixed and - # I am not sure what do with the local ones. - if errortype.lower() in ["broken", "local"]: - print("Not Fixed: " + line) + # For every line, fix the respective file + for line in output_lines: + match = re.match(line_re, line) + + if match: + newfilename = match.group(1) + errortype = match.group(2) + + # Broken links can't be fixed and + # I am not sure what do with the local ones. + if errortype.lower() in ["broken", "local"]: + print("Not Fixed: " + line) + else: + # If this is a new file + if newfilename != _filename: + + # Update the previous file + if _filename: + with open(_filename, "w") as _file: + _file.write(_contents) + + _filename = newfilename + + # Read the new file to memory + with open(_filename) as _file: + _contents = _file.read() + + _contents = _contents.replace(match.group(3), match.group(4)) else: - # If this is a new file - if newfilename != _filename: + # We don't understand what the current line means! + print("Not Understood: " + line) - # Update the previous file - if _filename: - with open(_filename, "w") as _file: - _file.write(_contents) - _filename = newfilename - - # Read the new file to memory - with open(_filename) as _file: - _contents = _file.read() - - _contents = _contents.replace(match.group(3), match.group(4)) - else: - # We don't understand what the current line means! - print("Not Understood: " + line) +if __name__ == '__main__': + main() diff --git a/pylintrc b/pylintrc new file mode 100644 index 000000000..b83bc9f82 --- /dev/null +++ b/pylintrc @@ -0,0 +1,88 @@ +[MASTER] +persistent=no +jobs=1 # >1 hides results + +[MESSAGES CONTROL] +disable=abstract-method, + anomalous-backslash-in-string, + arguments-differ, + attribute-defined-outside-init, + bad-classmethod-argument, + bad-continuation, + bad-indentation, + bad-mcs-classmethod-argument, + bad-whitespace, + broad-except, + c-extension-no-member, + catching-non-exception, + cell-var-from-loop, + comparison-with-callable, + consider-using-in, + cyclic-import, + dangerous-default-value, + deprecated-method, + deprecated-module, + duplicate-code, # https://github.com/PyCQA/pylint/issues/214 + eval-used, + expression-not-assigned, + fixme, + function-redefined, + global-statement, + import-error, + import-outside-toplevel, + inconsistent-return-statements, + inherit-non-class, + invalid-name, + keyword-arg-before-vararg, + line-too-long, + logging-format-interpolation, + logging-not-lazy, + lost-exception, + method-hidden, + missing-docstring, + missing-final-newline, + multiple-imports, + multiple-statements, + no-else-continue, + no-else-raise, + no-else-return, + no-init, + no-member, + no-method-argument, + no-name-in-module, + no-self-argument, + no-self-use, + pointless-string-statement, + protected-access, + redefined-argument-from-local, + redefined-builtin, + redefined-outer-name, + reimported, + signature-differs, + super-init-not-called, + superfluous-parens, + too-few-public-methods, + too-many-ancestors, + too-many-arguments, + too-many-branches, + too-many-function-args, + too-many-instance-attributes, + too-many-locals, + too-many-return-statements, + trailing-newlines, + trailing-whitespace, + unexpected-special-method-signature, + ungrouped-imports, + unidiomatic-typecheck, + unnecessary-comprehension, + unnecessary-pass, + unsubscriptable-object, + unused-argument, + unused-import, + unused-variable, + unused-wildcard-import, + used-before-assignment, + useless-object-inheritance, # Required for Python 2 support + wildcard-import, + wrong-import-order, + wrong-import-position diff --git a/tox.ini b/tox.ini index ffe7360d3..e7d366fe9 100644 --- a/tox.ini +++ b/tox.ini @@ -98,6 +98,20 @@ deps = {[testenv:py35]deps} commands = py.test {posargs:scrapy tests} +[testenv:pylint] +basepython = python3.7 +deps = + {[testenv:py35]deps} + # Optional dependencies + boto + reppy + robotexclusionrulesparser + # Test dependencies + pylint + +commands = + pylint scrapy + [docs] changedir = docs deps = From 02577f55a0586bc3e6c13a4a3ea572c7eefc82b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 24 Oct 2019 13:25:11 +0200 Subject: [PATCH 002/281] Have PyLint cover all Python files in the repository --- pylintrc | 19 +++++++++++++++++++ tox.ini | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/pylintrc b/pylintrc index b83bc9f82..ca3ea1c57 100644 --- a/pylintrc +++ b/pylintrc @@ -11,13 +11,18 @@ disable=abstract-method, bad-continuation, bad-indentation, bad-mcs-classmethod-argument, + bad-super-call, bad-whitespace, + blacklisted-name, broad-except, c-extension-no-member, catching-non-exception, cell-var-from-loop, comparison-with-callable, + consider-iterating-dictionary, consider-using-in, + consider-using-set-comprehension, + consider-using-sys-exit, cyclic-import, dangerous-default-value, deprecated-method, @@ -30,6 +35,7 @@ disable=abstract-method, global-statement, import-error, import-outside-toplevel, + import-self, inconsistent-return-statements, inherit-non-class, invalid-name, @@ -39,6 +45,7 @@ disable=abstract-method, logging-not-lazy, lost-exception, method-hidden, + misplaced-comparison-constant, missing-docstring, missing-final-newline, multiple-imports, @@ -52,6 +59,9 @@ disable=abstract-method, no-name-in-module, no-self-argument, no-self-use, + no-value-for-parameter, + not-callable, + pointless-statement, pointless-string-statement, protected-access, redefined-argument-from-local, @@ -59,6 +69,7 @@ disable=abstract-method, redefined-outer-name, reimported, signature-differs, + singleton-comparison, super-init-not-called, superfluous-parens, too-few-public-methods, @@ -67,15 +78,21 @@ disable=abstract-method, too-many-branches, too-many-function-args, too-many-instance-attributes, + too-many-lines, too-many-locals, + too-many-public-methods, too-many-return-statements, trailing-newlines, trailing-whitespace, + unbalanced-tuple-unpacking, + undefined-variable, unexpected-special-method-signature, ungrouped-imports, unidiomatic-typecheck, unnecessary-comprehension, + unnecessary-lambda, unnecessary-pass, + unreachable, unsubscriptable-object, unused-argument, unused-import, @@ -83,6 +100,8 @@ disable=abstract-method, unused-wildcard-import, used-before-assignment, useless-object-inheritance, # Required for Python 2 support + useless-return, + useless-super-delegation, wildcard-import, wrong-import-order, wrong-import-position diff --git a/tox.ini b/tox.ini index e7d366fe9..428571ef2 100644 --- a/tox.ini +++ b/tox.ini @@ -110,7 +110,7 @@ deps = pylint commands = - pylint scrapy + pylint conftest.py docs extras scrapy setup.py tests [docs] changedir = docs From 7a7d13b1122dac397ee0bb8edd4e6fd61665e232 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 23 Nov 2019 19:04:02 -0300 Subject: [PATCH 003/281] Rename LogFormatter.error to item_error --- scrapy/core/scraper.py | 2 +- scrapy/logformatter.py | 6 +++--- tests/test_logformatter.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index db463f989..c5bb48ea6 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -231,7 +231,7 @@ class Scraper(object): signal=signals.item_dropped, item=item, response=response, spider=spider, exception=output.value) else: - logkws = self.logformatter.error(item, ex, response, spider) + logkws = self.logformatter.item_error(item, ex, response, spider) logger.log(*logformatter_adapter(logkws), extra={'spider': spider}, exc_info=failure_to_exc_info(output)) return self.signals.send_catch_log_deferred( diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 5189d7cfa..79c752da4 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -8,7 +8,7 @@ from scrapy.utils.request import referer_str SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s" DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s" CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s" -ERRORMSG = u"'Error processing %(item)s'" +ITEMERRORMSG = u"'Error processing %(item)s'" class LogFormatter(object): @@ -93,11 +93,11 @@ class LogFormatter(object): } } - def error(self, item, exception, response, spider): + def item_error(self, item, exception, response, spider): """Logs a message when an item causes an error while it is passing through the item pipeline.""" return { 'level': logging.ERROR, - 'msg': ERRORMSG, + 'msg': ITEMERRORMSG, 'args': { 'item': item, } diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index d0b23a8c4..f2f8c0464 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -63,13 +63,13 @@ class LogFormatterTestCase(unittest.TestCase): assert all(isinstance(x, six.text_type) for x in lines) self.assertEqual(lines, [u"Dropped: \u2018", '{}']) - def test_error(self): + def test_item_error(self): # In practice, the complete traceback is shown by passing the # 'exc_info' argument to the logging function item = {'key': 'value'} exception = Exception() response = Response("http://www.example.com") - logkws = self.formatter.error(item, exception, response, self.spider) + logkws = self.formatter.item_error(item, exception, response, self.spider) logline = logkws['msg'] % logkws['args'] self.assertEqual(logline, u"'Error processing {'key': 'value'}'") From facb9265421ead8afb839323af2e18f81dda560b Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 23 Nov 2019 19:16:41 -0300 Subject: [PATCH 004/281] Remove quotes from item_error message --- scrapy/logformatter.py | 8 ++++---- tests/test_logformatter.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 79c752da4..9e038160f 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -5,10 +5,10 @@ from twisted.python.failure import Failure from scrapy.utils.request import referer_str -SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s" -DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s" -CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s" -ITEMERRORMSG = u"'Error processing %(item)s'" +SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s" +DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s" +CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s" +ITEMERRORMSG = "Error processing %(item)s" class LogFormatter(object): diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index f2f8c0464..990927f71 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -71,7 +71,7 @@ class LogFormatterTestCase(unittest.TestCase): response = Response("http://www.example.com") logkws = self.formatter.item_error(item, exception, response, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, u"'Error processing {'key': 'value'}'") + self.assertEqual(logline, u"Error processing {'key': 'value'}") def test_scraped(self): item = CustomItem() From 4756e7c587880997a54d9abf94b9a4c0b5bab71c Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 23 Nov 2019 19:33:29 -0300 Subject: [PATCH 005/281] LogFormatter.spider_error --- scrapy/core/scraper.py | 8 ++++---- scrapy/logformatter.py | 12 ++++++++++++ tests/test_logformatter.py | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index c5bb48ea6..21820e988 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -16,7 +16,7 @@ from scrapy import signals from scrapy.http import Request, Response from scrapy.item import BaseItem from scrapy.core.spidermw import SpiderMiddlewareManager -from scrapy.utils.request import referer_str + logger = logging.getLogger(__name__) @@ -152,9 +152,9 @@ class Scraper(object): if isinstance(exc, CloseSpider): self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') return - logger.error( - "Spider error processing %(request)s (referer: %(referer)s)", - {'request': request, 'referer': referer_str(request)}, + logkws = self.logformatter.spider_error(_failure, request, response, spider) + logger.log( + *logformatter_adapter(logkws), exc_info=failure_to_exc_info(_failure), extra={'spider': spider} ) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 9e038160f..d87f685d5 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -9,6 +9,7 @@ SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s" DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s" CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s" ITEMERRORMSG = "Error processing %(item)s" +SPIDERERRORMSG = "Spider error processing %(request)s (referer: %(referer)s)" class LogFormatter(object): @@ -103,6 +104,17 @@ class LogFormatter(object): } } + def spider_error(self, failure, request, response, spider): + """Logs an error message from a spider.""" + return { + 'level': logging.ERROR, + 'msg': SPIDERERRORMSG, + 'args': { + 'request': request, + 'referer': referer_str(request), + } + } + @classmethod def from_crawler(cls, crawler): return cls() diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 990927f71..47d2747c2 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -2,6 +2,7 @@ import unittest from testfixtures import LogCapture from twisted.internet import defer +from twisted.python.failure import Failure from twisted.trial.unittest import TestCase as TwistedTestCase import six @@ -73,6 +74,19 @@ class LogFormatterTestCase(unittest.TestCase): logline = logkws['msg'] % logkws['args'] self.assertEqual(logline, u"Error processing {'key': 'value'}") + def test_spider_error(self): + # In practice, the complete traceback is shown by passing the + # 'exc_info' argument to the logging function + failure = Failure(Exception()) + request = Request("http://www.example.com", headers={'Referer': 'http://example.org'}) + response = Response("http://www.example.com", request=request) + logkws = self.formatter.spider_error(failure, request, response, self.spider) + logline = logkws['msg'] % logkws['args'] + self.assertEqual( + logline, + "Spider error processing (referer: http://example.org)" + ) + def test_scraped(self): item = CustomItem() item['name'] = u'\xa3' From 03af8885ff475dc47a3de89517b1a5d627bd49c4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 23 Nov 2019 20:02:44 -0300 Subject: [PATCH 006/281] LogFormatter.download_error --- scrapy/core/scraper.py | 22 +++++++++++++--------- scrapy/logformatter.py | 16 ++++++++++++++++ tests/test_logformatter.py | 18 ++++++++++++++++++ 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 21820e988..427969f30 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -200,19 +200,23 @@ class Scraper(object): """Log and silence errors that come from the engine (typically download errors that got propagated thru here) """ - if (isinstance(download_failure, Failure) and - not download_failure.check(IgnoreRequest)): + if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest): if download_failure.frames: - logger.error('Error downloading %(request)s', - {'request': request}, - exc_info=failure_to_exc_info(download_failure), - extra={'spider': spider}) + logkws = self.logformatter.download_error(download_failure, request, spider) + logger.log( + *logformatter_adapter(logkws), + extra={'spider': spider}, + exc_info=failure_to_exc_info(download_failure), + ) else: errmsg = download_failure.getErrorMessage() if errmsg: - logger.error('Error downloading %(request)s: %(errmsg)s', - {'request': request, 'errmsg': errmsg}, - extra={'spider': spider}) + logkws = self.logformatter.download_error( + download_failure, request, spider, errmsg) + logger.log( + *logformatter_adapter(logkws), + extra={'spider': spider}, + ) if spider_failure is not download_failure: return spider_failure diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index d87f685d5..99bd5cfac 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -10,6 +10,8 @@ DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s" CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s" ITEMERRORMSG = "Error processing %(item)s" SPIDERERRORMSG = "Spider error processing %(request)s (referer: %(referer)s)" +DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s" +DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s" class LogFormatter(object): @@ -115,6 +117,20 @@ class LogFormatter(object): } } + def download_error(self, failure, request, spider, errmsg=None): + """Logs a download error message from a spider (typically coming from the engine).""" + args = {'request': request} + if errmsg: + msg = DOWNLOADERRORMSG_LONG + args['errmsg'] = errmsg + else: + msg = DOWNLOADERRORMSG_SHORT + return { + 'level': logging.ERROR, + 'msg': msg, + 'args': args, + } + @classmethod def from_crawler(cls, crawler): return cls() diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 47d2747c2..697ac1d15 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -87,6 +87,24 @@ class LogFormatterTestCase(unittest.TestCase): "Spider error processing (referer: http://example.org)" ) + def test_download_error_short(self): + # In practice, the complete traceback is shown by passing the + # 'exc_info' argument to the logging function + failure = Failure(Exception()) + request = Request("http://www.example.com") + logkws = self.formatter.download_error(failure, request, self.spider) + logline = logkws['msg'] % logkws['args'] + self.assertEqual(logline, "Error downloading ") + + def test_download_error_long(self): + # In practice, the complete traceback is shown by passing the + # 'exc_info' argument to the logging function + failure = Failure(Exception()) + request = Request("http://www.example.com") + logkws = self.formatter.download_error(failure, request, self.spider, "Some message") + logline = logkws['msg'] % logkws['args'] + self.assertEqual(logline, "Error downloading : Some message") + def test_scraped(self): item = CustomItem() item['name'] = u'\xa3' From 17e648182332a1d231383c7416e08e89280cb1d0 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 27 Nov 2019 18:42:42 -0300 Subject: [PATCH 007/281] [Docs] Fix Twisted links --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index eab366efd..a79f3a8cb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -279,7 +279,7 @@ intersphinx_mapping = { 'python': ('https://docs.python.org/3', None), 'sphinx': ('https://www.sphinx-doc.org/en/master', None), 'tox': ('https://tox.readthedocs.io/en/latest', None), - 'twisted': ('https://twistedmatrix.com/documents/current', None), + 'twisted': ('https://twistedmatrix.com/documents/current/api', None), } From 048cd74ae594f449ba97d07c927d7640f32a6770 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 27 Nov 2019 19:16:18 -0300 Subject: [PATCH 008/281] Add separate mapping for Twisted API docs --- docs/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index a79f3a8cb..40e69c8ac 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -279,7 +279,8 @@ intersphinx_mapping = { 'python': ('https://docs.python.org/3', None), 'sphinx': ('https://www.sphinx-doc.org/en/master', None), 'tox': ('https://tox.readthedocs.io/en/latest', None), - 'twisted': ('https://twistedmatrix.com/documents/current/api', None), + 'twisted': ('https://twistedmatrix.com/documents/current', None), + 'twistedapi': ('https://twistedmatrix.com/documents/current/api', None), } From c7f9b955bdf2405fce58907b0395abce2400a66d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 19 Dec 2019 12:44:52 +0100 Subject: [PATCH 009/281] Pylint: ignore not-an-iterable --- pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/pylintrc b/pylintrc index ca3ea1c57..c52a4c2d0 100644 --- a/pylintrc +++ b/pylintrc @@ -60,6 +60,7 @@ disable=abstract-method, no-self-argument, no-self-use, no-value-for-parameter, + not-an-iterable, not-callable, pointless-statement, pointless-string-statement, From 3faef2d08277f43659400470ec408ee29400018a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 12 Sep 2019 20:10:58 +0500 Subject: [PATCH 010/281] Add async def support to signal handlers that already supported Deferreds. --- scrapy/utils/defer.py | 17 +++++++++++++++++ scrapy/utils/signal.py | 6 ++++-- tests/test_utils_signal.py | 26 +++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index bbd5ebe52..a2c24e5fb 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -140,3 +140,20 @@ def deferred_from_coro(o): # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor return defer.Deferred.fromFuture(asyncio.ensure_future(o)) return o + + +def maybeDeferred_coro(f, *args, **kw): + """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """ + try: + result = f(*args, **kw) + except: # noqa: E722 + return defer.fail(failure.Failure(captureVars=defer.Deferred.debug)) + + if isinstance(result, defer.Deferred): + return result + elif _isfuture(result) or inspect.isawaitable(result): + return deferred_from_coro(result) + elif isinstance(result, failure.Failure): + return defer.fail(result) + else: + return defer.succeed(result) diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index de00bac49..60c561da6 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -2,12 +2,14 @@ import logging -from twisted.internet.defer import maybeDeferred, DeferredList, Deferred +from twisted.internet.defer import DeferredList, Deferred from twisted.python.failure import Failure from pydispatch.dispatcher import Any, Anonymous, liveReceivers, \ getAllReceivers, disconnect from pydispatch.robustapply import robustApply + +from scrapy.utils.defer import maybeDeferred_coro from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) @@ -61,7 +63,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): spider = named.get('spider', None) dfds = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - d = maybeDeferred(robustApply, receiver, signal=signal, sender=sender, + d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, *arguments, **named) d.addErrback(logerror, receiver) d.addBoth(lambda result: (receiver, result)) diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 16b7c5c68..e5f6f0ed4 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,3 +1,6 @@ +import asyncio + +from pytest import mark from testfixtures import LogCapture from twisted.trial import unittest from twisted.python.failure import Failure @@ -5,6 +8,7 @@ from twisted.internet import defer, reactor from pydispatch import dispatcher from scrapy.utils.signal import send_catch_log, send_catch_log_deferred +from scrapy.utils.test import get_from_asyncio_queue class SendCatchLogTest(unittest.TestCase): @@ -54,7 +58,7 @@ class SendCatchLogDeferredTest(SendCatchLogTest): return send_catch_log_deferred(signal, *a, **kw) -class SendCatchLogDeferredTest2(SendCatchLogTest): +class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) @@ -63,8 +67,24 @@ class SendCatchLogDeferredTest2(SendCatchLogTest): reactor.callLater(0, d.callback, "OK") return d - def _get_result(self, signal, *a, **kw): - return send_catch_log_deferred(signal, *a, **kw) + +class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): + + async def ok_handler(self, arg, handlers_called): + handlers_called.add(self.ok_handler) + assert arg == 'test' + await defer.succeed(42) + return "OK" + + +@mark.only_asyncio() +class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): + + async def ok_handler(self, arg, handlers_called): + handlers_called.add(self.ok_handler) + assert arg == 'test' + await asyncio.sleep(0.2) + return await get_from_asyncio_queue("OK") class SendCatchLogTest2(unittest.TestCase): From 6ce1ad31071326d22387f8444abefd6f3e18ed86 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 10 Jan 2020 04:20:37 -0300 Subject: [PATCH 011/281] [test] Spider middleware: catch exceptions right after the spider callback --- tests/test_spidermiddleware_output_chain.py | 50 +++++++++++++++++++-- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 739cf1c2d..b19a74609 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -1,10 +1,10 @@ - from testfixtures import LogCapture -from twisted.trial.unittest import TestCase from twisted.internet import defer +from twisted.trial.unittest import TestCase -from scrapy import Spider, Request +from scrapy import Request, Spider from scrapy.utils.test import get_crawler + from tests.mockserver import MockServer @@ -74,7 +74,7 @@ class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback name = 'ProcessSpiderInputSpiderWithErrback' def start_requests(self): - yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse, errback=self.errback) + yield Request(self.mockserver.url('/status?n=200'), self.parse, errback=self.errback) def errback(self, failure): self.logger.info('Got a Failure on the Request errback') @@ -100,6 +100,17 @@ class GeneratorCallbackSpider(Spider): raise ImportError() +# ================================================================================ +# (2.1) exceptions from a spider callback (generator, middleware right after callback) +class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider): + name = 'GeneratorCallbackSpiderMiddlewareRightAfterSpider' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + __name__ + '.LogExceptionMiddleware': 100000, + }, + } + + # ================================================================================ # (3) exceptions from a spider callback (not a generator) class NotGeneratorCallbackSpider(Spider): @@ -117,6 +128,17 @@ class NotGeneratorCallbackSpider(Spider): return [{'test': 1}, {'test': 1/0}] +# ================================================================================ +# (3.1) exceptions from a spider callback (not a generator, middleware right after callback) +class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackSpider): + name = 'NotGeneratorCallbackSpiderMiddlewareRightAfterSpider' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + __name__ + '.LogExceptionMiddleware': 100000, + }, + } + + # ================================================================================ # (4) exceptions from a middleware process_spider_output method (generator) class GeneratorOutputChainSpider(Spider): @@ -320,6 +342,16 @@ class TestSpiderMiddleware(TestCase): self.assertIn("Middleware: ImportError exception caught", str(log2)) self.assertIn("'item_scraped_count': 2", str(log2)) + @defer.inlineCallbacks + def test_generator_callback_right_after_callback(self): + """ + (2.1) Special case of (2): Exceptions should be caught + even if the middleware is placed right after the spider + """ + log21 = yield self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider) + self.assertIn("Middleware: ImportError exception caught", str(log21)) + self.assertIn("'item_scraped_count': 2", str(log21)) + @defer.inlineCallbacks def test_not_a_generator_callback(self): """ @@ -330,6 +362,16 @@ class TestSpiderMiddleware(TestCase): self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) self.assertNotIn("item_scraped_count", str(log3)) + @defer.inlineCallbacks + def test_not_a_generator_callback_right_after_callback(self): + """ + (3.1) Special case of (3): Exceptions should be caught + even if the middleware is placed right after the spider + """ + log31 = yield self.crawl_log(NotGeneratorCallbackSpiderMiddlewareRightAfterSpider) + self.assertIn("Middleware: ZeroDivisionError exception caught", str(log31)) + self.assertNotIn("item_scraped_count", str(log31)) + @defer.inlineCallbacks def test_generator_output_chain(self): """ From c088c04f449c3383b7867c25b387f13949b1d6c0 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 10 Jan 2020 04:20:55 -0300 Subject: [PATCH 012/281] Spider middleware: catch exceptions right after the spider callback --- scrapy/core/spidermw.py | 71 +++++++++++++++++++++++++---------------- scrapy/utils/python.py | 1 + 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 097a374bf..180a0b1fe 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,13 +3,14 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ -from itertools import chain, islice +from itertools import islice from twisted.python.failure import Failure + from scrapy.exceptions import _InvalidOutput from scrapy.middleware import MiddlewareManager -from scrapy.utils.defer import mustbe_deferred from scrapy.utils.conf import build_component_list +from scrapy.utils.defer import mustbe_deferred from scrapy.utils.python import MutableChain @@ -17,6 +18,13 @@ def _isiterable(possible_iterator): return hasattr(possible_iterator, '__iter__') +def _fname(f): + return "%s.%s".format( + f.__self__.__class__.__name__, + f.__func__.__name__ + ) + + class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -31,27 +39,36 @@ class SpiderMiddlewareManager(MiddlewareManager): self.methods['process_spider_input'].append(mw.process_spider_input) if hasattr(mw, 'process_start_requests'): self.methods['process_start_requests'].appendleft(mw.process_start_requests) - self.methods['process_spider_output'].appendleft(getattr(mw, 'process_spider_output', None)) - self.methods['process_spider_exception'].appendleft(getattr(mw, 'process_spider_exception', None)) + process_spider_output = getattr(mw, 'process_spider_output', None) + self.methods['process_spider_output'].appendleft(process_spider_output) + process_spider_exception = getattr(mw, 'process_spider_exception', None) + self.methods['process_spider_exception'].appendleft(process_spider_exception) def scrape_response(self, scrape_func, response, request, spider): - fname = lambda f: '%s.%s' % ( - f.__self__.__class__.__name__, - f.__func__.__name__) def process_spider_input(response): for method in self.methods['process_spider_input']: try: result = method(response=response, spider=spider) if result is not None: - raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' - .format(fname(method), type(result))) + msg = "Middleware {} must return None or raise an exception, got {}" + raise _InvalidOutput(msg.format(_fname(method), type(result))) except _InvalidOutput: raise except Exception: return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) + def _evaluate_iterable(iterable, method_index, recover_to): + try: + for r in iterable: + yield r + except Exception as ex: + exception_result = process_spider_exception(Failure(ex), method_index) + if isinstance(exception_result, Failure): + raise + recover_to.extend(exception_result) + def process_spider_exception(_failure, start_index=0): exception = _failure.value # don't handle _InvalidOutput exception @@ -69,8 +86,8 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' - .format(fname(method), type(result))) + msg = "Middleware {} must return None or an iterable, got {}" + raise _InvalidOutput(msg.format(_fname(method), type(result))) return _failure def process_spider_output(result, start_index=0): @@ -78,38 +95,36 @@ class SpiderMiddlewareManager(MiddlewareManager): # chain, they went through it already from the process_spider_exception method recovered = MutableChain() - def evaluate_iterable(iterable, index): - try: - for r in iterable: - yield r - except Exception as ex: - exception_result = process_spider_exception(Failure(ex), index+1) - if isinstance(exception_result, Failure): - raise - recovered.extend(exception_result) - method_list = islice(self.methods['process_spider_output'], start_index, None) for method_index, method in enumerate(method_list, start=start_index): if method is None: continue - # the following might fail directly if the output value is not a generator try: + # might fail directly if the output value is not a generator result = method(response=response, result=result, spider=spider) except Exception as ex: exception_result = process_spider_exception(Failure(ex), method_index+1) if isinstance(exception_result, Failure): raise return exception_result - if _isiterable(result): - result = evaluate_iterable(result, method_index) else: - raise _InvalidOutput('Middleware {} must return an iterable, got {}' - .format(fname(method), type(result))) + if _isiterable(result): + result = _evaluate_iterable(result, method_index+1, recovered) + else: + msg = "Middleware {} must return an iterable, got {}" + raise _InvalidOutput(msg.format(_fname(method), type(result))) - return chain(result, recovered) + return MutableChain(result, recovered) + + def process_callback_output(result): + if isinstance(result, Failure): + return process_spider_exception(result) + recovered = MutableChain() + result = _evaluate_iterable(result, 0, recovered) + return MutableChain(process_spider_output(result), recovered) dfd = mustbe_deferred(process_spider_input, response) - dfd.addCallbacks(callback=process_spider_output, errback=process_spider_exception) + dfd.addCallbacks(callback=process_callback_output, errback=process_callback_output) return dfd def process_start_requests(self, start_requests, spider): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 8d829c5a5..875650f3e 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -375,6 +375,7 @@ class MutableChain(object): """ Thin wrapper around itertools.chain, allowing to add iterables "in-place" """ + def __init__(self, *args): self.data = chain(*args) From d6e928f47209396d0f6c0b155eb07115c4855c93 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 10 Jan 2020 04:40:03 -0300 Subject: [PATCH 013/281] Remove object as base class for MutableChain Plus some minor styling adjustments --- scrapy/utils/python.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 875650f3e..e5582cc18 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -1,15 +1,15 @@ """ This module contains essential stuff that should've come with Python itself ;) """ +import errno import gc +import inspect import os import re -import inspect +import sys import weakref -import errno from functools import partial, wraps from itertools import chain -import sys from scrapy.utils.decorators import deprecated @@ -371,7 +371,7 @@ else: gc.collect() -class MutableChain(object): +class MutableChain: """ Thin wrapper around itertools.chain, allowing to add iterables "in-place" """ From 9770ca35fb494503298270902569ed897a365d32 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 10 Jan 2020 18:45:39 -0300 Subject: [PATCH 014/281] Spider middleware: simplify deferred errback handling --- scrapy/core/spidermw.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 180a0b1fe..ed02b306b 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -117,14 +117,12 @@ class SpiderMiddlewareManager(MiddlewareManager): return MutableChain(result, recovered) def process_callback_output(result): - if isinstance(result, Failure): - return process_spider_exception(result) recovered = MutableChain() result = _evaluate_iterable(result, 0, recovered) return MutableChain(process_spider_output(result), recovered) dfd = mustbe_deferred(process_spider_input, response) - dfd.addCallbacks(callback=process_callback_output, errback=process_callback_output) + dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception) return dfd def process_start_requests(self, start_requests, spider): From 03241aa4a66f9b0e8be4dc104807011f546840e8 Mon Sep 17 00:00:00 2001 From: abhishekh2001 <53903855+abhishekh2001@users.noreply.github.com> Date: Wed, 15 Jan 2020 08:54:25 +0400 Subject: [PATCH 015/281] Fixed artwork/README formatting --- artwork/README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/artwork/README.rst b/artwork/README.rst index 92f6ecb7e..8a1028cde 100644 --- a/artwork/README.rst +++ b/artwork/README.rst @@ -1,5 +1,4 @@ -:orphan: - +============== Scrapy artwork ============== From 80925ab845b7f55be97d9bb91015ceee90efc333 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 5 Aug 2019 11:39:07 -0300 Subject: [PATCH 016/281] Get server IP address for HTTP/1.1 responses --- docs/topics/request-response.rst | 12 +++++++++- scrapy/core/downloader/__init__.py | 2 +- scrapy/core/downloader/handlers/http11.py | 18 ++++++++++----- scrapy/http/response/__init__.py | 5 +++-- tests/test_crawl.py | 27 +++++++++++++++++++++++ 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 8997a7f19..a4cc1a7d7 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -34,7 +34,7 @@ Request objects :type url: string :param callback: the function that will be called with the response of this - request (once its downloaded) as its first parameter. For more information + request (once it's downloaded) as its first parameter. For more information see :ref:`topics-request-response-ref-request-callback-arguments` below. If a Request doesn't specify a callback, the spider's :meth:`~scrapy.spiders.Spider.parse` method will be used. @@ -611,6 +611,12 @@ Response objects This represents the :class:`Request` that generated this response. :type request: :class:`Request` object + :param ip_address: The IP address of the server from which the Response originated. + :type ip_address: :class:`ipaddress.IPv4Address` object + + .. FIXME: Add ipaddress.IPv6Address once it's supported + + .. attribute:: Response.url A string containing the URL of the response. @@ -679,6 +685,10 @@ Response objects they're shown on the string representation of the Response (`__str__` method) which is used by the engine for logging. + .. attribute:: Response.ip_address + + The IP address of the server from which the Response originated. + .. method:: Response.copy() Returns a new Response which is a copy of this Response. diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 157dc3418..11c9dd908 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -172,7 +172,7 @@ class Downloader(object): return response dfd.addCallback(_downloaded) - # 3. After response arrives, remove the request from transferring + # 3. After response arrives, remove the request from transferring # state to free up the transferring slot so it can be used by the # following requests (perhaps those which came from the downloader # middleware itself) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 5a5f6cf0a..b690f439f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -4,6 +4,7 @@ import logging import re import warnings from io import BytesIO +from ipaddress import ip_address from time import time from urllib.parse import urldefrag @@ -382,7 +383,7 @@ class ScrapyAgent(object): def _cb_bodyready(self, txresponse, request): # deliverBody hangs for responses without body if txresponse.length == 0: - return txresponse, b'', None + return txresponse, b'', None, None maxsize = request.meta.get('download_maxsize', self._maxsize) warnsize = request.meta.get('download_warnsize', self._warnsize) @@ -418,11 +419,11 @@ class ScrapyAgent(object): return d def _cb_bodydone(self, result, request, url): - txresponse, body, flags = result + txresponse, body, flags, ip_address = result status = int(txresponse.code) headers = Headers(txresponse.headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url, body=body) - return respcls(url=url, status=status, headers=headers, body=body, flags=flags) + return respcls(url=url, status=status, headers=headers, body=body, flags=flags, ip_address=ip_address) @implementer(IBodyProducer) @@ -456,6 +457,11 @@ class _ResponseReader(protocol.Protocol): self._fail_on_dataloss_warned = False self._reached_warnsize = False self._bytes_received = 0 + self._ip_address = None + + def connectionMade(self): + if self._ip_address is None: + self._ip_address = ip_address(self.transport._producer.getPeer().host) def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. @@ -488,16 +494,16 @@ class _ResponseReader(protocol.Protocol): body = self._bodybuf.getvalue() if reason.check(ResponseDone): - self._finished.callback((self._txresponse, body, None)) + self._finished.callback((self._txresponse, body, None, self._ip_address)) return if reason.check(PotentialDataLoss): - self._finished.callback((self._txresponse, body, ['partial'])) + self._finished.callback((self._txresponse, body, ['partial'], self._ip_address)) return if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons): if not self._fail_on_dataloss: - self._finished.callback((self._txresponse, body, ['dataloss'])) + self._finished.callback((self._txresponse, body, ['dataloss'], self._ip_address)) return elif not self._fail_on_dataloss_warned: diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f92d0901c..ca5ecc02c 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -17,13 +17,14 @@ from scrapy.utils.trackref import object_ref class Response(object_ref): - def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None): + def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, ip_address=None): self.headers = Headers(headers or {}) self.status = int(status) self._set_body(body) self._set_url(url) self.request = request self.flags = [] if flags is None else list(flags) + self.ip_address = ip_address @property def meta(self): @@ -76,7 +77,7 @@ class Response(object_ref): """Create a new Response with the same attributes except for those given new values. """ - for x in ['url', 'status', 'headers', 'body', 'request', 'flags']: + for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'ip_address']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f433fcea6..6281160ae 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,5 +1,7 @@ import json import logging +from ipaddress import IPv4Address +from urllib.parse import urlparse from testfixtures import LogCapture from twisted.internet import defer @@ -308,3 +310,28 @@ with multiples lines self.assertIn("[callback] status 201", str(log)) self.assertIn("[errback] status 404", str(log)) self.assertIn("[errback] status 500", str(log)) + + @defer.inlineCallbacks + def test_dns_server_ip_address(self): + from socket import gethostbyname + + crawler = self.runner.create_crawler(SingleRequestSpider) + url = 'https://example.org' + yield crawler.crawl(seed=url) + ip_address = crawler.spider.meta['responses'][0].ip_address + self.assertIsInstance(ip_address, IPv4Address) + self.assertEqual(str(ip_address), gethostbyname(urlparse(url).netloc)) + + crawler = self.runner.create_crawler(SingleRequestSpider) + url = self.mockserver.url('/status?n=200') + yield crawler.crawl(seed=url, mockserver=self.mockserver) + ip_address = crawler.spider.meta['responses'][0].ip_address + self.assertIsNone(ip_address) + + crawler = self.runner.create_crawler(SingleRequestSpider) + url = self.mockserver.url('/echo?body=test') + expected_netloc, _ = urlparse(url).netloc.split(':') + yield crawler.crawl(seed=url, mockserver=self.mockserver) + ip_address = crawler.spider.meta['responses'][0].ip_address + self.assertIsInstance(ip_address, IPv4Address) + self.assertEqual(str(ip_address), gethostbyname(expected_netloc)) From e8da7e296691d2b4eb63e2a442bb600e03e5766f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 26 Jan 2020 17:53:39 -0300 Subject: [PATCH 017/281] Test DNS resolution using CrawlerProcess --- tests/CrawlerProcess/ip_address.py | 51 ++++++++++++++++++++++++++++++ tests/test_crawl.py | 10 +----- tests/test_crawler.py | 8 +++++ 3 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 tests/CrawlerProcess/ip_address.py diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.py new file mode 100644 index 000000000..6b069cc90 --- /dev/null +++ b/tests/CrawlerProcess/ip_address.py @@ -0,0 +1,51 @@ +from urllib.parse import urlparse + +from twisted.internet import defer +from twisted.internet.base import ThreadedResolver +from twisted.internet.interfaces import IResolverSimple +from zope.interface.declarations import implementer + +from scrapy import Spider, Request +from scrapy.crawler import CrawlerProcess + +from tests.mockserver import MockServer + + +@implementer(IResolverSimple) +class MockThreadedResolver(ThreadedResolver): + """ + Resolves all names to localhost + """ + + @classmethod + def from_crawler(cls, crawler, reactor): + return cls(reactor) + + def install_on_reactor(self,): + self.reactor.installResolver(self) + + def getHostByName(self, name, timeout=None): + return defer.succeed("127.0.0.1") + + +class LocalhostSpider(Spider): + name = "localhost_spider" + + def start_requests(self): + yield Request(self.url) + + def parse(self, response): + netloc = urlparse(response.url).netloc + self.logger.info("Host: %s" % netloc.split(":")[0]) + self.logger.info("Type: %s" % type(response.ip_address)) + self.logger.info("IP address: %s" % response.ip_address) + + +with MockServer() as mockserver: + settings = {"DNS_RESOLVER": __name__ + ".MockThreadedResolver"} + process = CrawlerProcess(settings) + + port = urlparse(mockserver.http_address).port + url = "http://not.a.real.domain:{port}/echo?body=test".format(port=port) + process.crawl(LocalhostSpider, url=url) + process.start() diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 6281160ae..9896058dc 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,6 +1,7 @@ import json import logging from ipaddress import IPv4Address +from socket import gethostbyname from urllib.parse import urlparse from testfixtures import LogCapture @@ -313,15 +314,6 @@ with multiples lines @defer.inlineCallbacks def test_dns_server_ip_address(self): - from socket import gethostbyname - - crawler = self.runner.create_crawler(SingleRequestSpider) - url = 'https://example.org' - yield crawler.crawl(seed=url) - ip_address = crawler.spider.meta['responses'][0].ip_address - self.assertIsInstance(ip_address, IPv4Address) - self.assertEqual(str(ip_address), gethostbyname(urlparse(url).netloc)) - crawler = self.runner.create_crawler(SingleRequestSpider) url = self.mockserver.url('/status?n=200') yield crawler.crawl(seed=url, mockserver=self.mockserver) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0ce0674de..dfc1cf448 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -107,6 +107,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): def test_spider_custom_settings_log_level(self): log_file = self.mktemp() + class MySpider(scrapy.Spider): name = 'spider' custom_settings = { @@ -323,3 +324,10 @@ class CrawlerProcessSubprocess(unittest.TestCase): "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log, "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, ])) + + def test_response_ip_address(self): + log = self.run_script("ip_address.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Host: not.a.real.domain", log) + self.assertIn("Type: ", log) + self.assertIn("IP address: 127.0.0.1", log) From 8529dff41d3d2f6c81ee58c60b16dd9f2b8f72b4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 26 Jan 2020 18:00:56 -0300 Subject: [PATCH 018/281] Update docs regarding Response.ip_address and IPv6 --- docs/topics/request-response.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a4cc1a7d7..17eb63064 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -612,10 +612,7 @@ Response objects :type request: :class:`Request` object :param ip_address: The IP address of the server from which the Response originated. - :type ip_address: :class:`ipaddress.IPv4Address` object - - .. FIXME: Add ipaddress.IPv6Address once it's supported - + :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` .. attribute:: Response.url From 72b8613ee9827af031862bd84f1bea9acefcbebe Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 27 Nov 2019 14:46:20 -0300 Subject: [PATCH 019/281] bytes_received signal (no tests) --- docs/topics/signals.rst | 45 ++++++++++++++++------- scrapy/core/downloader/handlers/http11.py | 25 +++++++++++-- scrapy/signals.py | 1 + 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3f29aa323..6efb73abb 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -73,7 +73,7 @@ engine_started Sent when the Scrapy engine has started crawling. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. .. note:: This signal may be fired *after* the :signal:`spider_opened` signal, depending on how the spider was started. So **don't** rely on this signal @@ -88,7 +88,7 @@ engine_stopped Sent when the Scrapy engine is stopped (for example, when a crawling process has finished). - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. item_scraped ------------ @@ -99,7 +99,7 @@ item_scraped Sent when an item has been scraped, after it has passed all the :ref:`topics-item-pipeline` stages (without being dropped). - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param item: the item scraped :type item: dict or :class:`~scrapy.item.Item` object @@ -119,7 +119,7 @@ item_dropped Sent after an item has been dropped from the :ref:`topics-item-pipeline` when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param item: the item dropped from the :ref:`topics-item-pipeline` :type item: dict or :class:`~scrapy.item.Item` object @@ -144,7 +144,7 @@ item_error Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises an exception), except :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param item: the item dropped from the :ref:`topics-item-pipeline` :type item: dict or :class:`~scrapy.item.Item` object @@ -158,6 +158,23 @@ item_error :param failure: the exception raised :type failure: twisted.python.failure.Failure +bytes_received +-------------- + +.. signal:: bytes_received +.. function:: bytes_received(data, request) + + Sent by the HTTP 1.1 download handler when a group of bytes is + received for a specific request. + + This signal does not support returning deferreds from its handlers. + + :param data: the data received by the download handler + :type spider: :class:`bytes` object + + :param request: the request that generated the response + :type request: :class:`~scrapy.http.Request` object + spider_closed ------------- @@ -167,7 +184,7 @@ spider_closed Sent after a spider has been closed. This can be used to release per-spider resources reserved on :signal:`spider_opened`. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param spider: the spider which has been closed :type spider: :class:`~scrapy.spiders.Spider` object @@ -191,7 +208,7 @@ spider_opened reserve per-spider resources, but can be used for any task that needs to be performed when a spider is opened. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param spider: the spider which has been opened :type spider: :class:`~scrapy.spiders.Spider` object @@ -215,7 +232,7 @@ spider_idle You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to prevent the spider from being closed. - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param spider: the spider which has gone idle :type spider: :class:`~scrapy.spiders.Spider` object @@ -234,7 +251,7 @@ spider_error Sent when a spider callback generates an error (ie. raises an exception). - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param failure: the exception raised :type failure: twisted.python.failure.Failure @@ -254,7 +271,7 @@ request_scheduled Sent when the engine schedules a :class:`~scrapy.http.Request`, to be downloaded later. - The signal does not support returning deferreds from their handlers. + The signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler :type request: :class:`~scrapy.http.Request` object @@ -271,7 +288,7 @@ request_dropped Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be downloaded later, is rejected by the scheduler. - The signal does not support returning deferreds from their handlers. + The signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler :type request: :class:`~scrapy.http.Request` object @@ -287,7 +304,7 @@ request_reached_downloader Sent when a :class:`~scrapy.http.Request` reached downloader. - The signal does not support returning deferreds from their handlers. + The signal does not support returning deferreds from its handlers. :param request: the request that reached downloader :type request: :class:`~scrapy.http.Request` object @@ -304,7 +321,7 @@ response_received Sent when the engine receives a new :class:`~scrapy.http.Response` from the downloader. - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param response: the response received :type response: :class:`~scrapy.http.Response` object @@ -323,7 +340,7 @@ response_downloaded Sent by the downloader right after a ``HTTPResponse`` is downloaded. - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param response: the response downloaded :type response: :class:`~scrapy.http.Response` object diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 5a5f6cf0a..92c3d5f5c 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -16,6 +16,7 @@ from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from zope.interface import implementer +from scrapy import signals from scrapy.core.downloader.tls import openssl_methods from scrapy.core.downloader.webclient import _parse from scrapy.exceptions import ScrapyDeprecationWarning @@ -32,6 +33,7 @@ class HTTP11DownloadHandler: lazy = False def __init__(self, settings, crawler=None): + self.crawler = crawler self._pool = HTTPConnectionPool(reactor, persistent=True) self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self._pool._factory.noisy = False @@ -76,6 +78,7 @@ class HTTP11DownloadHandler: maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), fail_on_dataloss=self._fail_on_dataloss, + crawler=self.crawler, ) return agent.download_request(request) @@ -272,7 +275,7 @@ class ScrapyAgent(object): _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, - maxsize=0, warnsize=0, fail_on_dataloss=True): + maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress @@ -281,6 +284,7 @@ class ScrapyAgent(object): self._warnsize = warnsize self._fail_on_dataloss = fail_on_dataloss self._txresponse = None + self._crawler = crawler def _get_agent(self, request, timeout): bindaddress = request.meta.get('bindaddress') or self._bindAddress @@ -409,7 +413,15 @@ class ScrapyAgent(object): d = defer.Deferred(_cancel) txresponse.deliverBody( - _ResponseReader(d, txresponse, request, maxsize, warnsize, fail_on_dataloss) + _ResponseReader( + d, + txresponse, + request, + maxsize, + warnsize, + fail_on_dataloss, + self._crawler, + ) ) # save response for timeouts @@ -445,7 +457,7 @@ class _RequestBodyProducer(object): class _ResponseReader(protocol.Protocol): - def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss): + def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler): self._finished = finished self._txresponse = txresponse self._request = request @@ -456,6 +468,7 @@ class _ResponseReader(protocol.Protocol): self._fail_on_dataloss_warned = False self._reached_warnsize = False self._bytes_received = 0 + self._crawler = crawler def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. @@ -465,6 +478,12 @@ class _ResponseReader(protocol.Protocol): self._bodybuf.write(bodyBytes) self._bytes_received += len(bodyBytes) + self._crawler.signals.send_catch_log( + signal=signals.bytes_received, + data=bodyBytes, + request=self._request, + ) + if self._maxsize and self._bytes_received > self._maxsize: logger.error("Received (%(bytes)s) bytes larger than download " "max size (%(maxsize)s) in request %(request)s.", diff --git a/scrapy/signals.py b/scrapy/signals.py index 6b9125302..590421893 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -16,6 +16,7 @@ request_dropped = object() request_reached_downloader = object() response_received = object() response_downloaded = object() +bytes_received = object() item_scraped = object() item_dropped = object() item_error = object() From cab449b1952020b86fbe2915a537150fc885c567 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 29 Nov 2019 11:37:40 -0300 Subject: [PATCH 020/281] Typo fix --- tests/test_engine.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index 25dee7c1f..9d68836cc 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -107,7 +107,7 @@ class CrawlerRun(object): self.reqreached = [] self.itemerror = [] self.itemresp = [] - self.signals_catched = {} + self.signals_caught = {} self.spider_class = spider_class def run(self): @@ -172,7 +172,7 @@ class CrawlerRun(object): signalargs = kwargs.copy() sig = signalargs.pop('signal') signalargs.pop('sender', None) - self.signals_catched[sig] = signalargs + self.signals_caught[sig] = signalargs class EngineTest(unittest.TestCase): @@ -186,7 +186,7 @@ class EngineTest(unittest.TestCase): self._assert_scheduled_requests(urls_to_visit=8) self._assert_downloaded_responses() self._assert_scraped_items() - self._assert_signals_catched() + self._assert_signals_caught() @defer.inlineCallbacks def test_crawler_dupefilter(self): @@ -263,19 +263,19 @@ class EngineTest(unittest.TestCase): self.assertEqual('Item 2 name', item['name']) self.assertEqual('200', item['price']) - def _assert_signals_catched(self): - assert signals.engine_started in self.run.signals_catched - assert signals.engine_stopped in self.run.signals_catched - assert signals.spider_opened in self.run.signals_catched - assert signals.spider_idle in self.run.signals_catched - assert signals.spider_closed in self.run.signals_catched + def _assert_signals_caught(self): + assert signals.engine_started in self.run.signals_caught + assert signals.engine_stopped in self.run.signals_caught + assert signals.spider_opened in self.run.signals_caught + assert signals.spider_idle in self.run.signals_caught + assert signals.spider_closed in self.run.signals_caught self.assertEqual({'spider': self.run.spider}, - self.run.signals_catched[signals.spider_opened]) + self.run.signals_caught[signals.spider_opened]) self.assertEqual({'spider': self.run.spider}, - self.run.signals_catched[signals.spider_idle]) + self.run.signals_caught[signals.spider_idle]) self.assertEqual({'spider': self.run.spider, 'reason': 'finished'}, - self.run.signals_catched[signals.spider_closed]) + self.run.signals_caught[signals.spider_closed]) @defer.inlineCallbacks def test_close_downloader(self): From bda37e38bd53d5aae691b56d4136fbff99f78158 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 29 Nov 2019 12:02:27 -0300 Subject: [PATCH 021/281] [Tests] bytes_received signal --- tests/test_engine.py | 53 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index 9d68836cc..b63c7e232 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -13,22 +13,24 @@ module with the ``runserver`` argument:: import os import re import sys +from collections import defaultdict from urllib.parse import urlparse from twisted.internet import reactor, defer -from twisted.web import server, static, util from twisted.trial import unittest +from twisted.web import server, static, util +from pydispatch import dispatcher from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.utils.test import get_crawler -from pydispatch import dispatcher -from tests import tests_datadir -from scrapy.spiders import Spider +from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor -from scrapy.http import Request +from scrapy.spiders import Spider from scrapy.utils.signal import disconnect_all +from scrapy.utils.test import get_crawler + +from tests import tests_datadir, get_testdata class TestItem(Item): @@ -107,6 +109,7 @@ class CrawlerRun(object): self.reqreached = [] self.itemerror = [] self.itemresp = [] + self.bytes = defaultdict(lambda: b"") self.signals_caught = {} self.spider_class = spider_class @@ -124,6 +127,7 @@ class CrawlerRun(object): self.crawler = get_crawler(self.spider_class) self.crawler.signals.connect(self.item_scraped, signals.item_scraped) self.crawler.signals.connect(self.item_error, signals.item_error) + self.crawler.signals.connect(self.bytes_received, signals.bytes_received) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.request_dropped, signals.request_dropped) self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader) @@ -155,6 +159,9 @@ class CrawlerRun(object): def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) + def bytes_received(self, data, request): + self.bytes[request] += data + def request_scheduled(self, request, spider): self.reqplug.append((request, spider)) @@ -187,6 +194,7 @@ class EngineTest(unittest.TestCase): self._assert_downloaded_responses() self._assert_scraped_items() self._assert_signals_caught() + self._assert_bytes_received() @defer.inlineCallbacks def test_crawler_dupefilter(self): @@ -263,6 +271,39 @@ class EngineTest(unittest.TestCase): self.assertEqual('Item 2 name', item['name']) self.assertEqual('200', item['price']) + def _assert_bytes_received(self): + self.assertEqual(8, len(self.run.bytes)) + for request, data in self.run.bytes.items(): + if self.run.getpath(request.url) == "/": + self.assertEqual(data, get_testdata("test_site", "index.html")) + elif self.run.getpath(request.url) == "/item1.html": + self.assertEqual(data, get_testdata("test_site", "item1.html")) + elif self.run.getpath(request.url) == "/item2.html": + self.assertEqual(data, get_testdata("test_site", "item2.html")) + elif self.run.getpath(request.url) == "/redirected": + self.assertEqual(data, b"Redirected here") + elif self.run.getpath(request.url) == '/redirect': + self.assertEqual(data, + b"\n\n" + b" \n" + b" \n" + b" \n" + b" \n" + b" click here\n" + b" \n" + b"\n" + ) + elif self.run.getpath(request.url) == "/tem999.html": + self.assertEqual(data, + b"\n\n" + b" 404 - No Such Resource\n" + b" \n" + b"

No Such Resource

\n" + b"

File not found.

\n" + b" \n" + b"\n" + ) + def _assert_signals_caught(self): assert signals.engine_started in self.run.signals_caught assert signals.engine_stopped in self.run.signals_caught From 89483ce9f709e230ee5ff9050d206430d2d17c9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 3 Dec 2019 12:06:08 +0100 Subject: [PATCH 022/281] Fix Flake8 issues --- tests/test_engine.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index b63c7e232..c0769c992 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -283,7 +283,8 @@ class EngineTest(unittest.TestCase): elif self.run.getpath(request.url) == "/redirected": self.assertEqual(data, b"Redirected here") elif self.run.getpath(request.url) == '/redirect': - self.assertEqual(data, + self.assertEqual( + data, b"\n\n" b" \n" b" \n" @@ -294,7 +295,8 @@ class EngineTest(unittest.TestCase): b"\n" ) elif self.run.getpath(request.url) == "/tem999.html": - self.assertEqual(data, + self.assertEqual( + data, b"\n\n" b" 404 - No Such Resource\n" b" \n" From c9d36522302ab73552d804137a3625552275a771 Mon Sep 17 00:00:00 2001 From: "Matsievskiy S.V" Date: Mon, 27 Jan 2020 18:24:57 +0300 Subject: [PATCH 023/281] add zsh -h autocomplete option --- extras/scrapy_zsh_completion | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion index e995947cb..33f46eda8 100644 --- a/extras/scrapy_zsh_completion +++ b/extras/scrapy_zsh_completion @@ -1,11 +1,12 @@ #compdef scrapy _scrapy() { local context state state_descr line + local ret=1 typeset -A opt_args _arguments \ - "(- 1 *)--help[Help]" \ + "(- 1 *)"{-h,--help}"[Help]" \ "1: :->command" \ - "*:: :->args" + "*:: :->args" && ret=0 case $state in command) @@ -134,6 +135,8 @@ _scrapy() { esac ;; esac + + return ret } _scrapy_cmds() { From dbe20a863ff63dce937b2d3b159782d8268e6838 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 27 Jan 2020 12:21:18 -0300 Subject: [PATCH 024/281] bytes_received signal: send spider argument --- docs/topics/signals.rst | 5 ++++- scrapy/core/downloader/handlers/http11.py | 1 + tests/test_engine.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 6efb73abb..3e70ca067 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -162,7 +162,7 @@ bytes_received -------------- .. signal:: bytes_received -.. function:: bytes_received(data, request) +.. function:: bytes_received(data, request, spider) Sent by the HTTP 1.1 download handler when a group of bytes is received for a specific request. @@ -175,6 +175,9 @@ bytes_received :param request: the request that generated the response :type request: :class:`~scrapy.http.Request` object + :param spider: the spider associated with the response + :type spider: :class:`~scrapy.spiders.Spider` object + spider_closed ------------- diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 92c3d5f5c..c53c9bb2d 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -482,6 +482,7 @@ class _ResponseReader(protocol.Protocol): signal=signals.bytes_received, data=bodyBytes, request=self._request, + spider=self._crawler.spider, ) if self._maxsize and self._bytes_received > self._maxsize: diff --git a/tests/test_engine.py b/tests/test_engine.py index c0769c992..57cc89ba3 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -159,7 +159,7 @@ class CrawlerRun(object): def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) - def bytes_received(self, data, request): + def bytes_received(self, data, request, spider): self.bytes[request] += data def request_scheduled(self, request, spider): From 613fd41f44d1455f9c9369087958674f3fdfcc8d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 27 Jan 2020 12:30:26 -0300 Subject: [PATCH 025/281] bytes_received signal: improve test performance --- tests/test_engine.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index 57cc89ba3..bb475958e 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -109,7 +109,7 @@ class CrawlerRun(object): self.reqreached = [] self.itemerror = [] self.itemresp = [] - self.bytes = defaultdict(lambda: b"") + self.bytes = defaultdict(lambda: list()) self.signals_caught = {} self.spider_class = spider_class @@ -160,7 +160,7 @@ class CrawlerRun(object): self.itemresp.append((item, response)) def bytes_received(self, data, request, spider): - self.bytes[request] += data + self.bytes[request].append(data) def request_scheduled(self, request, spider): self.reqplug.append((request, spider)) @@ -274,17 +274,18 @@ class EngineTest(unittest.TestCase): def _assert_bytes_received(self): self.assertEqual(8, len(self.run.bytes)) for request, data in self.run.bytes.items(): + joined_data = b"".join(data) if self.run.getpath(request.url) == "/": - self.assertEqual(data, get_testdata("test_site", "index.html")) + self.assertEqual(joined_data, get_testdata("test_site", "index.html")) elif self.run.getpath(request.url) == "/item1.html": - self.assertEqual(data, get_testdata("test_site", "item1.html")) + self.assertEqual(joined_data, get_testdata("test_site", "item1.html")) elif self.run.getpath(request.url) == "/item2.html": - self.assertEqual(data, get_testdata("test_site", "item2.html")) + self.assertEqual(joined_data, get_testdata("test_site", "item2.html")) elif self.run.getpath(request.url) == "/redirected": - self.assertEqual(data, b"Redirected here") + self.assertEqual(joined_data, b"Redirected here") elif self.run.getpath(request.url) == '/redirect': self.assertEqual( - data, + joined_data, b"\n\n" b" \n" b" \n" @@ -296,7 +297,7 @@ class EngineTest(unittest.TestCase): ) elif self.run.getpath(request.url) == "/tem999.html": self.assertEqual( - data, + joined_data, b"\n\n" b" 404 - No Such Resource\n" b" \n" From 4ffd18fb11ff89863569b8b4de44241e3ca2f86e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 27 Jan 2020 13:29:33 -0300 Subject: [PATCH 026/281] [docs] Mention that signals.bytes_received could be fired multiple times --- docs/topics/signals.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3e70ca067..f490911f3 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -165,7 +165,8 @@ bytes_received .. function:: bytes_received(data, request, spider) Sent by the HTTP 1.1 download handler when a group of bytes is - received for a specific request. + received for a specific request. This signal might be fired + multiple times for the same request. This signal does not support returning deferreds from its handlers. From ad4477d335bee8b10bc3bbca969defddd9b316f8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 27 Jan 2020 14:16:43 -0300 Subject: [PATCH 027/281] Remove unnecessary else --- scrapy/core/spidermw.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index ed02b306b..8b36cbb04 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -107,12 +107,11 @@ class SpiderMiddlewareManager(MiddlewareManager): if isinstance(exception_result, Failure): raise return exception_result + if _isiterable(result): + result = _evaluate_iterable(result, method_index+1, recovered) else: - if _isiterable(result): - result = _evaluate_iterable(result, method_index+1, recovered) - else: - msg = "Middleware {} must return an iterable, got {}" - raise _InvalidOutput(msg.format(_fname(method), type(result))) + msg = "Middleware {} must return an iterable, got {}" + raise _InvalidOutput(msg.format(_fname(method), type(result))) return MutableChain(result, recovered) From 752e8f7018cbfac9cbdf486046d6bd8171cca0e8 Mon Sep 17 00:00:00 2001 From: Daniel Kimsey Date: Sun, 26 Jan 2020 13:21:31 -0600 Subject: [PATCH 028/281] FilesPipeline.file_path has optional arguments Documented signature doesn't match the actual interface in [files.py](https://github.com/scrapy/scrapy/blob/master/scrapy/pipelines/files.py#L520). Specifically, it looks like it may be [called](https://github.com/scrapy/scrapy/blob/master/scrapy/pipelines/files.py#L422) without a response value. I found this when I was implementing the pipeline with the signature `file_path(self, request, response, info)` and the following error was being return in my results : [(False, )] Scrapy==1.8.0 --- docs/topics/media-pipeline.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 1e0e0f18f..67a0bfdba 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -410,7 +410,7 @@ See here the methods that you can override in your custom Files Pipeline: .. class:: FilesPipeline - .. method:: file_path(request, response, info) + .. method:: file_path(self, request, response=None, info=None) This method is called once per downloaded item. It returns the download path of the file originating from the specified @@ -434,7 +434,7 @@ See here the methods that you can override in your custom Files Pipeline: class MyFilesPipeline(FilesPipeline): - def file_path(self, request, response, info): + def file_path(self, request, response=None, info=None): return 'files/' + os.path.basename(urlparse(request.url).path) By default the :meth:`file_path` method returns @@ -524,7 +524,7 @@ See here the methods that you can override in your custom Images Pipeline: The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`, customizing the field names and adding custom behavior for images. - .. method:: file_path(request, response, info) + .. method:: file_path(self, request, response=None, info=None) This method is called once per downloaded item. It returns the download path of the file originating from the specified @@ -548,7 +548,7 @@ See here the methods that you can override in your custom Images Pipeline: class MyImagesPipeline(ImagesPipeline): - def file_path(self, request, response, info): + def file_path(self, request, response=None, info=None): return 'files/' + os.path.basename(urlparse(request.url).path) By default the :meth:`file_path` method returns From 2c9643d38cc076c4d2032efd994fda4cfcc9f88a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 29 Jan 2020 14:11:56 -0300 Subject: [PATCH 029/281] Test: bytes_received signal fired multiple times --- tests/test_engine.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index bb475958e..3c5cc403b 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -12,6 +12,7 @@ module with the ``runserver`` argument:: import os import re +import string import sys from collections import defaultdict from urllib.parse import urlparse @@ -90,6 +91,7 @@ def start_test_site(debug=False): r = static.File(root_dir) r.putChild(b"redirect", util.Redirect(b"/redirected")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) + r.putChild(b"random", static.Data(string.ascii_letters.encode("utf8") * 2**14, "text/plain")) port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") if debug: @@ -117,8 +119,12 @@ class CrawlerRun(object): self.port = start_test_site() self.portno = self.port.getHost().port - start_urls = [self.geturl("/"), self.geturl("/redirect"), - self.geturl("/redirect")] # a duplicate + start_urls = [ + self.geturl("/"), + self.geturl("/redirect"), + self.geturl("/redirect"), # duplicate + self.geturl("/random"), + ] for name, signal in vars(signals).items(): if not name.startswith('_'): @@ -190,7 +196,7 @@ class EngineTest(unittest.TestCase): self.run = CrawlerRun(spider) yield self.run.run() self._assert_visited_urls() - self._assert_scheduled_requests(urls_to_visit=8) + self._assert_scheduled_requests(urls_to_visit=9) self._assert_downloaded_responses() self._assert_scraped_items() self._assert_signals_caught() @@ -200,7 +206,7 @@ class EngineTest(unittest.TestCase): def test_crawler_dupefilter(self): self.run = CrawlerRun(TestDupeFilterSpider) yield self.run.run() - self._assert_scheduled_requests(urls_to_visit=7) + self._assert_scheduled_requests(urls_to_visit=8) self._assert_dropped_requests() @defer.inlineCallbacks @@ -237,8 +243,8 @@ class EngineTest(unittest.TestCase): def _assert_downloaded_responses(self): # response tests - self.assertEqual(8, len(self.run.respplug)) - self.assertEqual(8, len(self.run.reqreached)) + self.assertEqual(9, len(self.run.respplug)) + self.assertEqual(9, len(self.run.reqreached)) for response, _ in self.run.respplug: if self.run.getpath(response.url) == '/item999.html': @@ -272,7 +278,7 @@ class EngineTest(unittest.TestCase): self.assertEqual('200', item['price']) def _assert_bytes_received(self): - self.assertEqual(8, len(self.run.bytes)) + self.assertEqual(9, len(self.run.bytes)) for request, data in self.run.bytes.items(): joined_data = b"".join(data) if self.run.getpath(request.url) == "/": @@ -306,6 +312,8 @@ class EngineTest(unittest.TestCase): b" \n" b"\n" ) + elif self.run.getpath(request.url) == "/random": + self.assertTrue(len(data) > 1) # signal was fired multiple times def _assert_signals_caught(self): assert signals.engine_started in self.run.signals_caught From a499f38b14d16338d20084c0dcb24528a1f1f22f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 29 Jan 2020 14:35:17 -0300 Subject: [PATCH 030/281] Remove object parent class --- scrapy/core/downloader/handlers/http11.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c53c9bb2d..6f1bd9ad6 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -268,7 +268,7 @@ class ScrapyProxyAgent(Agent): ) -class ScrapyAgent(object): +class ScrapyAgent: _Agent = Agent _ProxyAgent = ScrapyProxyAgent @@ -438,7 +438,7 @@ class ScrapyAgent(object): @implementer(IBodyProducer) -class _RequestBodyProducer(object): +class _RequestBodyProducer: def __init__(self, body): self.body = body From 6f02a8dccb95373f22bac18c08d9fda8169dcb02 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 29 Jan 2020 14:53:23 -0300 Subject: [PATCH 031/281] Add source parameter to bytes_received signal --- docs/topics/signals.rst | 12 ++++++++---- scrapy/core/downloader/handlers/http11.py | 18 +++++++++++++----- scrapy/core/downloader/handlers/s3.py | 1 + tests/test_downloader_handlers.py | 3 +++ tests/test_engine.py | 5 ++++- 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index f490911f3..3a15bf95c 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -162,11 +162,11 @@ bytes_received -------------- .. signal:: bytes_received -.. function:: bytes_received(data, request, spider) +.. function:: bytes_received(data, request, spider, source) - Sent by the HTTP 1.1 download handler when a group of bytes is - received for a specific request. This signal might be fired - multiple times for the same request. + Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is + received for a specific request. This signal might be fired multiple + times for the same request, with partial data each time. This signal does not support returning deferreds from its handlers. @@ -179,6 +179,10 @@ bytes_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.spiders.Spider` object + :param source: a string to identify which handler sent the signal + (current values could be "http11" or "s3") + :type source: :class:`str` object + spider_closed ------------- diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 6f1bd9ad6..49c9eacac 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -32,8 +32,9 @@ logger = logging.getLogger(__name__) class HTTP11DownloadHandler: lazy = False - def __init__(self, settings, crawler=None): + def __init__(self, settings, crawler=None, source="http11"): self.crawler = crawler + self.source = source self._pool = HTTPConnectionPool(reactor, persistent=True) self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self._pool._factory.noisy = False @@ -67,8 +68,8 @@ class HTTP11DownloadHandler: self._disconnect_timeout = 1 @classmethod - def from_crawler(cls, crawler): - return cls(crawler.settings, crawler) + def from_crawler(cls, crawler, **kwargs): + return cls(crawler.settings, crawler, **kwargs) def download_request(self, request, spider): """Return a deferred for the HTTP download""" @@ -79,6 +80,7 @@ class HTTP11DownloadHandler: warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), fail_on_dataloss=self._fail_on_dataloss, crawler=self.crawler, + source=self.source, ) return agent.download_request(request) @@ -275,7 +277,7 @@ class ScrapyAgent: _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, - maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None): + maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None, source=None): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress @@ -285,6 +287,7 @@ class ScrapyAgent: self._fail_on_dataloss = fail_on_dataloss self._txresponse = None self._crawler = crawler + self._source = source def _get_agent(self, request, timeout): bindaddress = request.meta.get('bindaddress') or self._bindAddress @@ -421,6 +424,7 @@ class ScrapyAgent: warnsize, fail_on_dataloss, self._crawler, + self._source, ) ) @@ -457,7 +461,9 @@ class _RequestBodyProducer: class _ResponseReader(protocol.Protocol): - def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler): + def __init__( + self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler, source + ): self._finished = finished self._txresponse = txresponse self._request = request @@ -469,6 +475,7 @@ class _ResponseReader(protocol.Protocol): self._reached_warnsize = False self._bytes_received = 0 self._crawler = crawler + self._source = source def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. @@ -483,6 +490,7 @@ class _ResponseReader(protocol.Protocol): data=bodyBytes, request=self._request, spider=self._crawler.spider, + source=self._source, ) if self._maxsize and self._bytes_received > self._maxsize: diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 40a1fa48e..2366b6394 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -73,6 +73,7 @@ class S3DownloadHandler: objcls=httpdownloadhandler, settings=settings, crawler=crawler, + source="s3", ) self._download_http = _http_handler.download_request diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 8d95d7cac..22a813647 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -730,6 +730,9 @@ class Http11ProxyTestCase(HttpProxyTestCase): class HttpDownloadHandlerMock: + def __init__(self, *args, **kwargs): + pass + def download_request(self, request, spider): return request diff --git a/tests/test_engine.py b/tests/test_engine.py index 3c5cc403b..c83a23b55 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -112,6 +112,7 @@ class CrawlerRun(object): self.itemerror = [] self.itemresp = [] self.bytes = defaultdict(lambda: list()) + self.bytes_source = set() self.signals_caught = {} self.spider_class = spider_class @@ -165,8 +166,9 @@ class CrawlerRun(object): def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) - def bytes_received(self, data, request, spider): + def bytes_received(self, data, request, spider, source): self.bytes[request].append(data) + self.bytes_source.add(source) def request_scheduled(self, request, spider): self.reqplug.append((request, spider)) @@ -279,6 +281,7 @@ class EngineTest(unittest.TestCase): def _assert_bytes_received(self): self.assertEqual(9, len(self.run.bytes)) + self.assertEqual(self.run.bytes_source, set(["http11"])) for request, data in self.run.bytes.items(): joined_data = b"".join(data) if self.run.getpath(request.url) == "/": From a2ae380efcaa5a3419a4f6a35541ae0fb71a2e7f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 13:23:52 -0300 Subject: [PATCH 032/281] Remove unnecessary commas --- scrapy/resolver.py | 2 +- tests/CrawlerProcess/ip_address.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 554a3a14d..f69894b1e 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -29,7 +29,7 @@ class CachingThreadedResolver(ThreadedResolver): cache_size = 0 return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT')) - def install_on_reactor(self,): + def install_on_reactor(self): self.reactor.installResolver(self) def getHostByName(self, name, timeout=None): diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.py index 6b069cc90..949e97172 100644 --- a/tests/CrawlerProcess/ip_address.py +++ b/tests/CrawlerProcess/ip_address.py @@ -21,7 +21,7 @@ class MockThreadedResolver(ThreadedResolver): def from_crawler(cls, crawler, reactor): return cls(reactor) - def install_on_reactor(self,): + def install_on_reactor(self): self.reactor.installResolver(self) def getHostByName(self, name, timeout=None): From bb8f7dc609382153df79774ad9d8f6d33d064279 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 14:50:14 -0300 Subject: [PATCH 033/281] Mock DNS server --- tests/mockserver.py | 90 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 22 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index a45277db9..585741f1b 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,3 +1,4 @@ +import argparse import json import os import random @@ -6,18 +7,19 @@ from subprocess import Popen, PIPE from urllib.parse import urlencode from OpenSSL import SSL -from twisted.web.server import Site, NOT_DONE_YET -from twisted.web.resource import Resource +from twisted.internet import defer, reactor, ssl +from twisted.internet.task import deferLater +from twisted.names import dns, error +from twisted.names.server import DNSServerFactory +from twisted.web.resource import EncodingResourceWrapper, Resource +from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site from twisted.web.static import File from twisted.web.test.test_webclient import PayloadResource -from twisted.web.server import GzipEncoderFactory -from twisted.web.resource import EncodingResourceWrapper from twisted.web.util import redirectTo -from twisted.internet import reactor, ssl -from twisted.internet.task import deferLater from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3 +from scrapy.utils.test import get_testenv def getarg(request, name, default=None, type=None): @@ -198,12 +200,10 @@ class Root(Resource): return b'Scrapy mock HTTP server\n' -class MockServer(): +class MockServer: def __enter__(self): - from scrapy.utils.test import get_testenv - - self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'], + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'http'], stdout=PIPE, env=get_testenv()) http_address = self.proc.stdout.readline().strip().decode('ascii') https_address = self.proc.stdout.readline().strip().decode('ascii') @@ -224,6 +224,37 @@ class MockServer(): return host + path +class MockDNSResolver: + """ + Implements twisted.internet.interfaces.IResolver partially + """ + + def _resolve(self, name): + record = dns.Record_A(address=b"127.0.0.1") + answer = dns.RRHeader(name=name, payload=record) + return [answer], [], [] + + def query(self, query, timeout=None): + if query.type == dns.A: + return defer.succeed(self._resolve(query.name.name)) + return defer.fail(error.DomainError()) + + def lookupAllRecords(self, name, timeout=None): + return defer.succeed(self._resolve(name)) + + +class MockDNSServer(): + + def __enter__(self): + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', 'dns'], + stdout=PIPE, env=get_testenv()) + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.proc.kill() + self.proc.communicate() + + def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None): factory = ssl.DefaultOpenSSLContextFactory( os.path.join(os.path.dirname(__file__), keyfile), @@ -238,19 +269,34 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c if __name__ == "__main__": - root = Root() - factory = Site(root) - httpPort = reactor.listenTCP(0, factory) - contextFactory = ssl_context_factory() - httpsPort = reactor.listenSSL(0, factory, contextFactory) + parser = argparse.ArgumentParser() + parser.add_argument("-t", "--type", type=str, choices=("http", "dns"), default="http") + args = parser.parse_args() - def print_listening(): - httpHost = httpPort.getHost() - httpsHost = httpsPort.getHost() - httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port) - httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port) - print(httpAddress) - print(httpsAddress) + if args.type == "http": + root = Root() + factory = Site(root) + httpPort = reactor.listenTCP(0, factory) + contextFactory = ssl_context_factory() + httpsPort = reactor.listenSSL(0, factory, contextFactory) + + def print_listening(): + httpHost = httpPort.getHost() + httpsHost = httpsPort.getHost() + httpAddress = "http://%s:%d" % (httpHost.host, httpHost.port) + httpsAddress = "https://%s:%d" % (httpsHost.host, httpsHost.port) + print(httpAddress) + print(httpsAddress) + + elif args.type == "dns": + clients = [MockDNSResolver()] + factory = DNSServerFactory(clients=clients) + protocol = dns.DNSDatagramProtocol(controller=factory) + reactor.listenUDP(10053, protocol) + reactor.listenTCP(10053, factory) + + def print_listening(): + print("DNS server running on port 10053") reactor.callWhenRunning(print_listening) reactor.run() From 4851efdfb0885a40a44a2834c6c69d0104326801 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 14:50:54 -0300 Subject: [PATCH 034/281] Flake8 adjustments --- tests/mockserver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 585741f1b..67139534e 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -257,9 +257,9 @@ class MockDNSServer(): def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None): factory = ssl.DefaultOpenSSLContextFactory( - os.path.join(os.path.dirname(__file__), keyfile), - os.path.join(os.path.dirname(__file__), certfile), - ) + os.path.join(os.path.dirname(__file__), keyfile), + os.path.join(os.path.dirname(__file__), certfile), + ) if cipher_string: ctx = factory.getContext() # disabling TLS1.2+ because it unconditionally enables some strong ciphers From e0ef8ad2d6f958de6ce04cd7756e142efeb1a6a2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 15:52:15 -0300 Subject: [PATCH 035/281] CrawlerRunner test for Response.ip_address --- tests/CrawlerProcess/ip_address.py | 51 ------------------------------ tests/CrawlerRunner/ip_address.py | 37 ++++++++++++++++++++++ tests/mockserver.py | 11 ++++--- tests/test_crawler.py | 20 ++++++++---- 4 files changed, 57 insertions(+), 62 deletions(-) delete mode 100644 tests/CrawlerProcess/ip_address.py create mode 100644 tests/CrawlerRunner/ip_address.py diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.py deleted file mode 100644 index 949e97172..000000000 --- a/tests/CrawlerProcess/ip_address.py +++ /dev/null @@ -1,51 +0,0 @@ -from urllib.parse import urlparse - -from twisted.internet import defer -from twisted.internet.base import ThreadedResolver -from twisted.internet.interfaces import IResolverSimple -from zope.interface.declarations import implementer - -from scrapy import Spider, Request -from scrapy.crawler import CrawlerProcess - -from tests.mockserver import MockServer - - -@implementer(IResolverSimple) -class MockThreadedResolver(ThreadedResolver): - """ - Resolves all names to localhost - """ - - @classmethod - def from_crawler(cls, crawler, reactor): - return cls(reactor) - - def install_on_reactor(self): - self.reactor.installResolver(self) - - def getHostByName(self, name, timeout=None): - return defer.succeed("127.0.0.1") - - -class LocalhostSpider(Spider): - name = "localhost_spider" - - def start_requests(self): - yield Request(self.url) - - def parse(self, response): - netloc = urlparse(response.url).netloc - self.logger.info("Host: %s" % netloc.split(":")[0]) - self.logger.info("Type: %s" % type(response.ip_address)) - self.logger.info("IP address: %s" % response.ip_address) - - -with MockServer() as mockserver: - settings = {"DNS_RESOLVER": __name__ + ".MockThreadedResolver"} - process = CrawlerProcess(settings) - - port = urlparse(mockserver.http_address).port - url = "http://not.a.real.domain:{port}/echo?body=test".format(port=port) - process.crawl(LocalhostSpider, url=url) - process.start() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py new file mode 100644 index 000000000..5a71536d8 --- /dev/null +++ b/tests/CrawlerRunner/ip_address.py @@ -0,0 +1,37 @@ +from urllib.parse import urlparse + +from twisted.internet import reactor +from twisted.names.client import createResolver + +from scrapy import Spider, Request +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + +from tests.mockserver import MockServer, MockDNSServer + + +class LocalhostSpider(Spider): + name = "localhost_spider" + + def start_requests(self): + yield Request(self.url) + + def parse(self, response): + netloc = urlparse(response.url).netloc + self.logger.info("Host: %s" % netloc.split(":")[0]) + self.logger.info("Type: %s" % type(response.ip_address)) + self.logger.info("IP address: %s" % response.ip_address) + + +with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: + port = urlparse(mock_http_server.http_address).port + url = "http://not.a.real.domain:{port}/echo".format(port=port) + + servers = [(mock_dns_server.host, mock_dns_server.port)] + reactor.installResolver(createResolver(servers=servers)) + + configure_logging() + runner = CrawlerRunner() + d = runner.crawl(LocalhostSpider, url=url) + d.addBoth(lambda _: reactor.stop()) + reactor.run() diff --git a/tests/mockserver.py b/tests/mockserver.py index 67139534e..08a81418c 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -246,8 +246,11 @@ class MockDNSResolver: class MockDNSServer(): def __enter__(self): - self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', 'dns'], + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'], stdout=PIPE, env=get_testenv()) + host, port = self.proc.stdout.readline().strip().decode('ascii').split(":") + self.host = host + self.port = int(port) return self def __exit__(self, exc_type, exc_value, traceback): @@ -292,11 +295,11 @@ if __name__ == "__main__": clients = [MockDNSResolver()] factory = DNSServerFactory(clients=clients) protocol = dns.DNSDatagramProtocol(controller=factory) - reactor.listenUDP(10053, protocol) - reactor.listenTCP(10053, factory) + listener = reactor.listenUDP(0, protocol) def print_listening(): - print("DNS server running on port 10053") + host = listener.getHost() + print("%s:%s" % (host.host, host.port)) reactor.callWhenRunning(print_listening) reactor.run() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index dfc1cf448..5d381c368 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -281,9 +281,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertNotIn("Asyncio reactor is installed", str(log)) -class CrawlerProcessSubprocess(unittest.TestCase): - script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess') - +class ScriptRunnerMixin: def run_script(self, script_name): script_path = os.path.join(self.script_dir, script_name) args = (sys.executable, script_path) @@ -292,6 +290,10 @@ class CrawlerProcessSubprocess(unittest.TestCase): stdout, stderr = p.communicate() return stderr.decode('utf-8') + +class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): + script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess') + def test_simple(self): log = self.run_script('simple.py') self.assertIn('Spider closed (finished)', log) @@ -325,9 +327,13 @@ class CrawlerProcessSubprocess(unittest.TestCase): "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, ])) + +class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): + script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') + def test_response_ip_address(self): log = self.run_script("ip_address.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn("Host: not.a.real.domain", log) - self.assertIn("Type: ", log) - self.assertIn("IP address: 127.0.0.1", log) + self.assertIn("INFO: Spider closed (finished)", log) + self.assertIn("INFO: Host: not.a.real.domain", log) + self.assertIn("INFO: Type: ", log) + self.assertIn("INFO: IP address: 127.0.0.1", log) From 13670f0397ba8dcec3dceb1852bad5751406d19d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 16:16:43 -0300 Subject: [PATCH 036/281] Ignore tests/CrawlerRunner directory --- conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index c0de09909..55294feca 100644 --- a/conftest.py +++ b/conftest.py @@ -11,7 +11,8 @@ collect_ignore = [ # not a test, but looks like a test "scrapy/utils/testsite.py", # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess - *_py_files("tests/CrawlerProcess") + *_py_files("tests/CrawlerProcess"), + *_py_files("tests/CrawlerRunner"), ] for line in open('tests/ignores.txt'): From ad70497416527c3d882a64f7803e73155f3fa1da Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 4 Feb 2020 13:30:13 -0300 Subject: [PATCH 037/281] Remove unnecessary parentheses in class definition --- tests/mockserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 08a81418c..30d9bc0e8 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -243,7 +243,7 @@ class MockDNSResolver: return defer.succeed(self._resolve(name)) -class MockDNSServer(): +class MockDNSServer: def __enter__(self): self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'], From fbea370c58c1d82b52fd9c1f7d3a6cee94477c7a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 5 Feb 2020 01:35:13 -0300 Subject: [PATCH 038/281] Rename function parameter --- scrapy/core/spidermw.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 8b36cbb04..dd9b3c376 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -59,12 +59,12 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def _evaluate_iterable(iterable, method_index, recover_to): + def _evaluate_iterable(iterable, exception_processor_index, recover_to): try: for r in iterable: yield r except Exception as ex: - exception_result = process_spider_exception(Failure(ex), method_index) + exception_result = process_spider_exception(Failure(ex), exception_processor_index) if isinstance(exception_result, Failure): raise recover_to.extend(exception_result) From 898bc00811aac9d3e38d1863b95a10c2e8effb02 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 5 Feb 2020 11:31:27 +0000 Subject: [PATCH 039/281] new signal --- scrapy/signals.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/signals.py b/scrapy/signals.py index 6b9125302..cd7ed7fb1 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -14,6 +14,7 @@ spider_error = object() request_scheduled = object() request_dropped = object() request_reached_downloader = object() +request_left_downloader = object() response_received = object() response_downloaded = object() item_scraped = object() From ae04174884eeb777d7b3caceed52bf522944ceb1 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 5 Feb 2020 11:32:31 +0000 Subject: [PATCH 040/281] emit new signal --- scrapy/core/downloader/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 157dc3418..5a2fdadf5 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -181,6 +181,9 @@ class Downloader(object): def finish_transferring(_): slot.transferring.remove(request) self._process_queue(spider, slot) + self.signals.send_catch_log(signal=signals.request_left_downloader, + request=request, + spider=spider) return _ return dfd.addBoth(finish_transferring) From 9916f6e556f9d4a41ea86d4a73687af1a40e43ba Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 5 Feb 2020 11:32:54 +0000 Subject: [PATCH 041/281] tests for new signal --- tests/test_request_left.py | 59 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tests/test_request_left.py diff --git a/tests/test_request_left.py b/tests/test_request_left.py new file mode 100644 index 000000000..ddeca0499 --- /dev/null +++ b/tests/test_request_left.py @@ -0,0 +1,59 @@ +from twisted.internet import defer +from twisted.trial.unittest import TestCase +from scrapy.signals import request_left_downloader +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer + +class SignalCatcherSpider(Spider): + name = 'signal_catcher' + + def __init__(self, crawler, url, *args, **kwargs): + super(SignalCatcherSpider, self).__init__(*args, **kwargs) + crawler.signals.connect(self.on_response_download, + signal=request_left_downloader) + self.catched_times = 0 + self.start_urls = [url] + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = cls(crawler, *args, **kwargs) + return spider + + def on_response_download(self, request, spider): + self.catched_times = self.catched_times + 1 + + +class TestCatching(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_success(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl(self.mockserver.url("/status?n=200")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_timeout(self): + crawler = get_crawler(SignalCatcherSpider, + {'DOWNLOAD_TIMEOUT': 0.1}) + yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_disconnect(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl(self.mockserver.url("/drop")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_noconnect(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl('http://thereisdefinetelynosuchdomain.com') + self.assertEqual(crawler.spider.catched_times, 1) From aab39f63412b4b7a0ae2713446859d6d8103e5f7 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 5 Feb 2020 11:35:03 +0000 Subject: [PATCH 042/281] docummentation for new signal --- docs/topics/signals.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3f29aa323..7fa5bc030 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -295,6 +295,23 @@ request_reached_downloader :param spider: the spider that yielded the request :type spider: :class:`~scrapy.spiders.Spider` object +request_left_downloader +--------------------------- + +.. signal:: request_left_downloader +.. function:: request_left_downloader(request, spider) + + Sent when a :class:`~scrapy.http.Request` left downloader even in case of + failure. + + The signal does not support returning deferreds from their handlers. + + :param request: the request that reached downloader + :type request: :class:`~scrapy.http.Request` object + + :param spider: the spider that yielded the request + :type spider: :class:`~scrapy.spiders.Spider` object + response_received ----------------- From 3769f75386104c1a3072894b302d3c3239ff8c37 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 5 Feb 2020 12:08:08 +0000 Subject: [PATCH 043/281] pep8 E302 --- tests/test_request_left.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_request_left.py b/tests/test_request_left.py index ddeca0499..5d271190d 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -5,6 +5,7 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer + class SignalCatcherSpider(Spider): name = 'signal_catcher' From 11941c324431e1f8822e64fb03d163b9e721eaa5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 5 Feb 2020 13:27:54 -0300 Subject: [PATCH 044/281] Remove elusive six occurrence from tox.ini --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index b62100026..b1babc7fd 100644 --- a/tox.ini +++ b/tox.ini @@ -56,7 +56,6 @@ deps = pyOpenSSL==16.2.0 queuelib==1.4.2 service_identity==16.0.0 - six==1.10.0 Twisted==17.9.0 w3lib==1.17.0 zope.interface==4.1.3 From c2cca368213013f9dc9d4569b926da4d213f20a5 Mon Sep 17 00:00:00 2001 From: Respawnz <47511522+Respawnz@users.noreply.github.com> Date: Thu, 6 Feb 2020 05:39:15 +0800 Subject: [PATCH 045/281] typo --- docs/topics/developer-tools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index bf14643be..1fedf91df 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -132,7 +132,7 @@ a use case: Say you want to find the ``Next`` button on the page. Type ``Next`` into the search bar on the top right of the `Inspector`. You should get two results. -The first is a ``li`` tag with the ``class="text"``, the second the text +The first is a ``li`` tag with the ``class="next"``, the second the text of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``. If you hover over the tag, you'll see the button highlighted. From here we could easily create a :ref:`Link Extractor ` to From 576663e5a778646aa4ef870641fa677ea94d21f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 6 Feb 2020 10:43:20 +0100 Subject: [PATCH 046/281] Make METAREFRESH_IGNORE_TAGS an empty list by default --- docs/topics/downloader-middleware.rst | 2 +- scrapy/settings/default_settings.py | 2 +- tests/test_downloadermiddleware_redirect.py | 16 +++++++++------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 8a760e53b..3ec6e0c17 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -868,7 +868,7 @@ Whether the Meta Refresh middleware will be enabled. METAREFRESH_IGNORE_TAGS ^^^^^^^^^^^^^^^^^^^^^^^ -Default: ``['script', 'noscript']`` +Default: ``[]`` Meta tags within these tags are ignored. diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c10dc1a1c..1a7d35b13 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -225,7 +225,7 @@ MEMUSAGE_NOTIFY_MAIL = [] MEMUSAGE_WARNING_MB = 0 METAREFRESH_ENABLED = True -METAREFRESH_IGNORE_TAGS = ['script', 'noscript'] +METAREFRESH_IGNORE_TAGS = [] METAREFRESH_MAXDELAY = 100 NEWSPIDER_MODULE = '' diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index e7faf14a7..e0f145d0e 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -300,19 +300,21 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): body = ('''''') rsp = HtmlResponse(req.url, body=body.encode()) - response = self.mw.process_response(req, rsp, self.spider) - assert isinstance(response, Response) + req2 = self.mw.process_response(req, rsp, self.spider) + assert isinstance(req2, Request) + self.assertEqual(req2.url, 'http://example.org/newpage') - def test_ignore_tags_empty_list(self): - crawler = get_crawler(Spider, {'METAREFRESH_IGNORE_TAGS': []}) + def test_ignore_tags_1_x_list(self): + """Test that Scrapy 1.x behavior remains possible""" + settings = {'METAREFRESH_IGNORE_TAGS': ['script', 'noscript']} + crawler = get_crawler(Spider, settings) mw = MetaRefreshMiddleware.from_crawler(crawler) req = Request(url='http://example.org') body = ('''''') rsp = HtmlResponse(req.url, body=body.encode()) - req2 = mw.process_response(req, rsp, self.spider) - assert isinstance(req2, Request) - self.assertEqual(req2.url, 'http://example.org/newpage') + response = mw.process_response(req, rsp, self.spider) + assert isinstance(response, Response) if __name__ == "__main__": unittest.main() From 6733f4d976150e0e5352d4ae9697880ae60ad638 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 6 Feb 2020 18:40:42 +0500 Subject: [PATCH 047/281] Update docs/topics/signals.rst Co-Authored-By: elacuesta --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 7fa5bc030..47be6b603 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -301,7 +301,7 @@ request_left_downloader .. signal:: request_left_downloader .. function:: request_left_downloader(request, spider) - Sent when a :class:`~scrapy.http.Request` left downloader even in case of + Sent when a :class:`~scrapy.http.Request` leaves the downloader even in case of failure. The signal does not support returning deferreds from their handlers. From 4a91a5427df4846ed9fa11612cfeb9e31f34a1c8 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 6 Feb 2020 13:44:51 +0000 Subject: [PATCH 048/281] fix typo --- tests/test_request_left.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 5d271190d..8256d1c92 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -13,7 +13,7 @@ class SignalCatcherSpider(Spider): super(SignalCatcherSpider, self).__init__(*args, **kwargs) crawler.signals.connect(self.on_response_download, signal=request_left_downloader) - self.catched_times = 0 + self.caught_times = 0 self.start_urls = [url] @classmethod @@ -22,7 +22,7 @@ class SignalCatcherSpider(Spider): return spider def on_response_download(self, request, spider): - self.catched_times = self.catched_times + 1 + self.caught_times = self.caught_times + 1 class TestCatching(TestCase): @@ -38,23 +38,23 @@ class TestCatching(TestCase): def test_success(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl(self.mockserver.url("/status?n=200")) - self.assertEqual(crawler.spider.catched_times, 1) + self.assertEqual(crawler.spider.caught_times, 1) @defer.inlineCallbacks def test_timeout(self): crawler = get_crawler(SignalCatcherSpider, {'DOWNLOAD_TIMEOUT': 0.1}) yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) - self.assertEqual(crawler.spider.catched_times, 1) + self.assertEqual(crawler.spider.caught_times, 1) @defer.inlineCallbacks def test_disconnect(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl(self.mockserver.url("/drop")) - self.assertEqual(crawler.spider.catched_times, 1) + self.assertEqual(crawler.spider.caught_times, 1) @defer.inlineCallbacks def test_noconnect(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl('http://thereisdefinetelynosuchdomain.com') - self.assertEqual(crawler.spider.catched_times, 1) + self.assertEqual(crawler.spider.caught_times, 1) From 4be19e443e9c101a248c21509ae8000ce500d51a Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 6 Feb 2020 13:46:23 +0000 Subject: [PATCH 049/281] name signla catcher in accord with signal name --- tests/test_request_left.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 8256d1c92..5cfef8e7d 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -11,7 +11,7 @@ class SignalCatcherSpider(Spider): def __init__(self, crawler, url, *args, **kwargs): super(SignalCatcherSpider, self).__init__(*args, **kwargs) - crawler.signals.connect(self.on_response_download, + crawler.signals.connect(self.on_request_left, signal=request_left_downloader) self.caught_times = 0 self.start_urls = [url] @@ -21,7 +21,7 @@ class SignalCatcherSpider(Spider): spider = cls(crawler, *args, **kwargs) return spider - def on_response_download(self, request, spider): + def on_request_left(self, request, spider): self.caught_times = self.caught_times + 1 From 489ffcda5143a2ef28d4cbcf5418babd963f2b0f Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 6 Feb 2020 22:39:00 +0500 Subject: [PATCH 050/281] Add a test for an async item_scraped handler. --- tests/test_signals.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tests/test_signals.py diff --git a/tests/test_signals.py b/tests/test_signals.py new file mode 100644 index 000000000..001e798e5 --- /dev/null +++ b/tests/test_signals.py @@ -0,0 +1,39 @@ +from twisted.internet import defer +from twisted.trial import unittest + +from scrapy import signals, Request, Spider +from scrapy.utils.test import get_crawler + +from tests.mockserver import MockServer + + +class ItemSpider(Spider): + name = 'itemspider' + + def start_requests(self): + for _ in range(10): + yield Request(self.mockserver.url('/status?n=200'), + dont_filter=True) + + def parse(self, response): + return {'field': 42} + + +class AsyncSignalTestCase(unittest.TestCase): + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self.items = [] + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + async def _on_item_scraped(self, item): + self.items.append(item) + + @defer.inlineCallbacks + def test_simple_pipeline(self): + crawler = get_crawler(ItemSpider) + crawler.signals.connect(self._on_item_scraped, signals.item_scraped) + yield crawler.crawl(mockserver=self.mockserver) + self.assertEqual(len(self.items), 10) From 35dafef7f106dbf1c022d997b4e29e3eee84de7c Mon Sep 17 00:00:00 2001 From: elacuesta Date: Thu, 6 Feb 2020 14:42:34 -0300 Subject: [PATCH 051/281] Specify Twisted reactor (TWISTED_REACTOR setting) (#4294) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add the ability to install a specific reactor * Add docs for the TWISTED_REACTOR setting * Add tests for the TWISTED_REACTOR setting * Update asyncio reactor test * Ignore W503 globally W503 is not PEP8-compliant: https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b * Line length adjustment * Adjust asyncio reactor tests * Merge ASYNCIO_ENABLED and TWISTED_REACTOR settings * More docs about TWISTED_REACTOR * Fix asyncio reactor test * Docs: fix title * Reword docs * Check the TWISTED_REACTOR setting outside of the installing function * Remove unrelated change * Update scrapy/utils/log.py Co-Authored-By: Adrián Chaves * Update docs/topics/settings.rst Co-Authored-By: Adrián Chaves * Update docs/topics/settings.rst Co-Authored-By: Adrián Chaves Co-authored-by: Adrián Chaves --- docs/faq.rst | 11 +++++ docs/topics/broad-crawls.rst | 7 +++ docs/topics/settings.rst | 45 +++++++++--------- pytest.ini | 7 +-- scrapy/crawler.py | 19 ++++---- scrapy/settings/default_settings.py | 4 +- scrapy/utils/asyncio.py | 17 ------- scrapy/utils/defer.py | 4 +- scrapy/utils/log.py | 11 ++--- scrapy/utils/reactor.py | 35 +++++++++++++- .../asyncio_enabled_no_reactor.py | 3 +- .../CrawlerProcess/asyncio_enabled_reactor.py | 3 +- .../CrawlerProcess/twisted_reactor_asyncio.py | 13 +++++ tests/CrawlerProcess/twisted_reactor_poll.py | 13 +++++ .../CrawlerProcess/twisted_reactor_select.py | 13 +++++ tests/test_commands.py | 10 ++-- tests/test_crawler.py | 47 ++++++++++++++----- tests/test_utils_asyncio.py | 4 +- 18 files changed, 182 insertions(+), 84 deletions(-) delete mode 100644 scrapy/utils/asyncio.py create mode 100644 tests/CrawlerProcess/twisted_reactor_asyncio.py create mode 100644 tests/CrawlerProcess/twisted_reactor_poll.py create mode 100644 tests/CrawlerProcess/twisted_reactor_select.py diff --git a/docs/faq.rst b/docs/faq.rst index b65908012..f72e4cf01 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -361,6 +361,17 @@ Note that by doing so, you lose the ability to set a specific timeout for DNS re (the value of the :setting:`DNS_TIMEOUT` setting is ignored). +.. _faq-specific-reactor: + +How to deal with ``: filedescriptor out of range in select()`` exceptions? +---------------------------------------------------------------------------------------------- + +This issue `has been reported`_ to appear when running broad crawls in macOS, where the default +Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a +different reactor is possible by using the :setting:`TWISTED_REACTOR` setting. + + +.. _has been reported: https://github.com/scrapy/scrapy/issues/2905 .. _user agents: https://en.wikipedia.org/wiki/User_agent .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 1ab08d949..4922694ee 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -211,3 +211,10 @@ If your broad crawl shows a high memory usage, in addition to :ref:`crawling in BFO order ` and :ref:`lowering concurrency ` you should :ref:`debug your memory leaks `. + + +Install a specific Twisted reactor +================================== + +If the crawl is exceeding the system's capabilities, you might want to try +installing a specific Twisted reactor, via the :setting:`TWISTED_REACTOR` setting. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4b770d249..fa63a5807 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -160,27 +160,6 @@ to any particular component. In that case the module of that component will be shown, typically an extension, middleware or pipeline. It also means that the component must be enabled in order for the setting to have any effect. -.. setting:: ASYNCIO_REACTOR - -ASYNCIO_REACTOR ---------------- - -Default: ``False`` - -Whether to install and require the Twisted reactor that uses the asyncio loop. - -When this option is set to ``True``, Scrapy will require -:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. It will -install this reactor if no reactor is installed yet, such as when using the -``scrapy`` script or :class:`~scrapy.crawler.CrawlerProcess`. If you are using -:class:`~scrapy.crawler.CrawlerRunner`, you need to install the correct reactor -manually. If a different reactor is installed outside Scrapy, it will raise an -exception. - -The default value for this option is currently ``False`` to maintain backward -compatibility and avoid possible problems caused by using a different Twisted -reactor. - .. setting:: AWS_ACCESS_KEY_ID AWS_ACCESS_KEY_ID @@ -1463,6 +1442,30 @@ command. The project name must not conflict with the name of custom files or directories in the ``project`` subdirectory. +.. setting:: TWISTED_REACTOR + +TWISTED_REACTOR +--------------- + +Default: ``None`` + +Import path of a given Twisted reactor, for instance: +:class:`twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + +Scrapy will install this reactor if no other is installed yet, such as when +the ``scrapy`` CLI program is invoked or when using the +:class:`~scrapy.crawler.CrawlerProcess` class. If you are using the +:class:`~scrapy.crawler.CrawlerRunner` class, you need to install the correct +reactor manually. An exception will be raised if the installation fails. + +The default value for this option is currently ``None``, which means that Scrapy +will not attempt to install any specific reactor, and the default one defined by +Twisted for the current platform will be used. This is to maintain backward +compatibility and avoid possible problems caused by using a non-default reactor. + +For additional information, please see +:doc:`core/howto/choosing-reactor`. + .. setting:: URLLENGTH_LIMIT diff --git a/pytest.ini b/pytest.ini index bae68cd3a..552829d4e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -21,6 +21,7 @@ twisted = 1 markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed flake8-ignore = + W503 # Files that are only meant to provide top-level imports are expected not # to use any of their imports: scrapy/core/downloader/handlers/http.py F401 @@ -109,7 +110,7 @@ flake8-ignore = # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 scrapy/spidermiddlewares/offsite.py E501 - scrapy/spidermiddlewares/referer.py E501 E129 W503 W504 + scrapy/spidermiddlewares/referer.py E501 E129 W504 scrapy/spidermiddlewares/urllength.py E501 # scrapy/spiders scrapy/spiders/__init__.py E501 E402 @@ -129,13 +130,13 @@ flake8-ignore = scrapy/utils/http.py F403 E226 scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 E701 - scrapy/utils/log.py E128 W503 + scrapy/utils/log.py E128 E501 scrapy/utils/markup.py F403 scrapy/utils/misc.py E501 E226 scrapy/utils/multipart.py F403 scrapy/utils/project.py E501 scrapy/utils/python.py E501 - scrapy/utils/reactor.py E226 + scrapy/utils/reactor.py E226 E501 scrapy/utils/reqser.py E501 scrapy/utils/request.py E127 E501 scrapy/utils/response.py E501 E128 diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 35c6b7716..49b8e4511 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -13,7 +13,6 @@ from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.settings import overridden_settings, Settings from scrapy.signalmanager import SignalManager -from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed from scrapy.utils.log import ( configure_logging, get_scrapy_root_handler, @@ -23,6 +22,7 @@ from scrapy.utils.log import ( ) from scrapy.utils.misc import create_instance, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names +from scrapy.utils.reactor import install_reactor, verify_installed_reactor logger = logging.getLogger(__name__) @@ -138,7 +138,7 @@ class CrawlerRunner: self._crawlers = set() self._active = set() self.bootstrap_failed = False - self._handle_asyncio_reactor() + self._handle_twisted_reactor() @property def spiders(self): @@ -232,10 +232,9 @@ class CrawlerRunner: while self._active: yield defer.DeferredList(self._active) - def _handle_asyncio_reactor(self): - if self.settings.getbool('ASYNCIO_REACTOR') and not is_asyncio_reactor_installed(): - raise Exception("ASYNCIO_REACTOR is on but the Twisted asyncio " - "reactor is not installed.") + def _handle_twisted_reactor(self): + if self.settings.get("TWISTED_REACTOR"): + verify_installed_reactor(self.settings["TWISTED_REACTOR"]) class CrawlerProcess(CrawlerRunner): @@ -324,10 +323,10 @@ class CrawlerProcess(CrawlerRunner): except RuntimeError: # raised if already stopped or in shutdown stage pass - def _handle_asyncio_reactor(self): - if self.settings.getbool('ASYNCIO_REACTOR'): - install_asyncio_reactor() - super()._handle_asyncio_reactor() + def _handle_twisted_reactor(self): + if self.settings.get("TWISTED_REACTOR"): + install_reactor(self.settings["TWISTED_REACTOR"]) + super()._handle_twisted_reactor() def _get_spider_loader(settings): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c10dc1a1c..c80833710 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -19,8 +19,6 @@ from os.path import join, abspath, dirname AJAXCRAWL_ENABLED = False -ASYNCIO_REACTOR = False - AUTOTHROTTLE_ENABLED = False AUTOTHROTTLE_DEBUG = False AUTOTHROTTLE_MAX_DELAY = 60.0 @@ -291,6 +289,8 @@ TELNETCONSOLE_HOST = '127.0.0.1' TELNETCONSOLE_USERNAME = 'scrapy' TELNETCONSOLE_PASSWORD = None +TWISTED_REACTOR = None + SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { 'scrapy.contracts.default.UrlContract': 1, diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py deleted file mode 100644 index 917973de2..000000000 --- a/scrapy/utils/asyncio.py +++ /dev/null @@ -1,17 +0,0 @@ -import asyncio -from contextlib import suppress - -from twisted.internet import asyncioreactor -from twisted.internet.error import ReactorAlreadyInstalledError - - -def install_asyncio_reactor(): - """ Tries to install AsyncioSelectorReactor - """ - with suppress(ReactorAlreadyInstalledError): - asyncioreactor.install(asyncio.get_event_loop()) - - -def is_asyncio_reactor_installed(): - from twisted.internet import reactor - return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 62b43a96c..6a21a4903 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -2,14 +2,14 @@ Helper functions for dealing with Twisted deferreds """ import asyncio -from functools import wraps import inspect +from functools import wraps from twisted.internet import defer, task from twisted.python import failure from scrapy.exceptions import IgnoreRequest -from scrapy.utils.asyncio import is_asyncio_reactor_installed +from scrapy.utils.reactor import is_asyncio_reactor_installed def defer_fail(_failure): diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index e4cf0196b..afef2c93f 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -1,17 +1,16 @@ # -*- coding: utf-8 -*- -import sys import logging +import sys import warnings from logging.config import dictConfig -from twisted.python.failure import Failure from twisted.python import log as twisted_log +from twisted.python.failure import Failure import scrapy -from scrapy.settings import Settings from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.asyncio import is_asyncio_reactor_installed +from scrapy.settings import Settings from scrapy.utils.versions import scrapy_components_versions @@ -149,8 +148,8 @@ def log_scrapy_info(settings): {'versions': ", ".join("%s %s" % (name, version) for name, version in scrapy_components_versions() if name != "Scrapy")}) - if is_asyncio_reactor_installed(): - logger.debug("Asyncio reactor is installed") + from twisted.internet import reactor + logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) class StreamLogger(object): diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index b98fff6ec..80f52a4ef 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,4 +1,9 @@ -from twisted.internet import error +import asyncio +from contextlib import suppress + +from twisted.internet import asyncioreactor, error + +from scrapy.utils.misc import load_object def listen_tcp(portrange, host, factory): @@ -42,3 +47,31 @@ class CallLaterOnce(object): def __call__(self): self._call = None return self._func(*self._a, **self._kw) + + +def install_reactor(reactor_path): + reactor_class = load_object(reactor_path) + if reactor_class is asyncioreactor.AsyncioSelectorReactor: + with suppress(error.ReactorAlreadyInstalledError): + asyncioreactor.install(asyncio.get_event_loop()) + else: + *module, _ = reactor_path.split(".") + installer_path = module + ["install"] + installer = load_object(".".join(installer_path)) + with suppress(error.ReactorAlreadyInstalledError): + installer() + + +def verify_installed_reactor(reactor_path): + from twisted.internet import reactor + reactor_class = load_object(reactor_path) + if not isinstance(reactor, reactor_class): + msg = "The installed reactor ({}.{}) does not match the requested one ({})".format( + reactor.__module__, reactor.__class__.__name__, reactor_path + ) + raise Exception(msg) + + +def is_asyncio_reactor_installed(): + from twisted.internet import reactor + return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index db1b75931..d1e4a7bb5 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -10,8 +10,7 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess(settings={ - 'ASYNCIO_REACTOR': True, + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) - process.crawl(NoRequestsSpider) process.start() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index cec3c9c25..8568bd8b8 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -15,8 +15,7 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess(settings={ - 'ASYNCIO_REACTOR': True, + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) - process.crawl(NoRequestsSpider) process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_asyncio.py b/tests/CrawlerProcess/twisted_reactor_asyncio.py new file mode 100644 index 000000000..c6cbf949b --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_asyncio.py @@ -0,0 +1,13 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", +}) +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_poll.py b/tests/CrawlerProcess/twisted_reactor_poll.py new file mode 100644 index 000000000..27063260b --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_poll.py @@ -0,0 +1,13 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class PollReactorSpider(scrapy.Spider): + name = 'poll_reactor' + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor", +}) +process.crawl(PollReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_select.py b/tests/CrawlerProcess/twisted_reactor_select.py new file mode 100644 index 000000000..9af8ceb4d --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_select.py @@ -0,0 +1,13 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class SelectReactorSpider(scrapy.Spider): + name = 'epoll_reactor' + + +process = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", +}) +process.crawl(SelectReactorSpider) +process.start() diff --git a/tests/test_commands.py b/tests/test_commands.py index 6024af71c..3612b70c9 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -296,12 +296,14 @@ class BadSpider(scrapy.Spider): self.assertIn("badspider.py", log) def test_asyncio_enabled_true(self): - log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=True']) - self.assertIn("DEBUG: Asyncio reactor is installed", log) + log = self.get_log(self.debug_log_spider, args=[ + '-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor' + ]) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) def test_asyncio_enabled_false(self): - log = self.get_log(self.debug_log_spider, args=['-s', 'ASYNCIO_REACTOR=False']) - self.assertNotIn("DEBUG: Asyncio reactor is installed", log) + log = self.get_log(self.debug_log_spider, args=[]) + self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) class BenchCommandTest(CommandTest): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0ce0674de..f8fa26def 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -254,30 +254,38 @@ class CrawlerRunnerHasSpider(unittest.TestCase): def test_crawler_runner_asyncio_enabled_true(self): if self.reactor_pytest == 'asyncio': - runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True}) + runner = CrawlerRunner(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + }) else: - msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed" + msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): - runner = CrawlerRunner(settings={'ASYNCIO_REACTOR': True}) + runner = CrawlerRunner(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + }) @defer.inlineCallbacks def test_crawler_process_asyncio_enabled_true(self): with LogCapture(level=logging.DEBUG) as log: if self.reactor_pytest == 'asyncio': - runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True}) + runner = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + }) yield runner.crawl(NoRequestsSpider) - self.assertIn("Asyncio reactor is installed", str(log)) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) else: - msg = "ASYNCIO_REACTOR is on but the Twisted asyncio reactor is not installed" + msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): - runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': True}) + runner = CrawlerProcess(settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + }) @defer.inlineCallbacks def test_crawler_process_asyncio_enabled_false(self): - runner = CrawlerProcess(settings={'ASYNCIO_REACTOR': False}) + runner = CrawlerProcess(settings={"TWISTED_REACTOR": None}) with LogCapture(level=logging.DEBUG) as log: yield runner.crawl(NoRequestsSpider) - self.assertNotIn("Asyncio reactor is installed", str(log)) + self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) class CrawlerProcessSubprocess(unittest.TestCase): @@ -294,17 +302,17 @@ class CrawlerProcessSubprocess(unittest.TestCase): def test_simple(self): log = self.run_script('simple.py') self.assertIn('Spider closed (finished)', log) - self.assertNotIn("DEBUG: Asyncio reactor is installed", log) + self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) def test_asyncio_enabled_no_reactor(self): log = self.run_script('asyncio_enabled_no_reactor.py') self.assertIn('Spider closed (finished)', log) - self.assertIn("DEBUG: Asyncio reactor is installed", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) def test_asyncio_enabled_reactor(self): log = self.run_script('asyncio_enabled_reactor.py') self.assertIn('Spider closed (finished)', log) - self.assertIn("DEBUG: Asyncio reactor is installed", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) def test_ipv6_default_name_resolver(self): log = self.run_script('default_name_resolver.py') @@ -323,3 +331,18 @@ class CrawlerProcessSubprocess(unittest.TestCase): "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log, "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, ])) + + def test_reactor_select(self): + log = self.run_script("twisted_reactor_select.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) + + def test_reactor_poll(self): + log = self.run_script("twisted_reactor_poll.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log) + + def test_reactor_asyncio(self): + log = self.run_script("twisted_reactor_asyncio.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 44acc24af..295323e4d 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -2,7 +2,7 @@ from unittest import TestCase from pytest import mark -from scrapy.utils.asyncio import is_asyncio_reactor_installed, install_asyncio_reactor +from scrapy.utils.reactor import is_asyncio_reactor_installed, install_reactor @mark.usefixtures('reactor_pytest') @@ -14,4 +14,4 @@ class AsyncioTest(TestCase): def test_install_asyncio_reactor(self): # this should do nothing - install_asyncio_reactor() + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") From 3263441fbcec8f46d363926d9106572cb0ecac5e Mon Sep 17 00:00:00 2001 From: Lane Shaw Date: Thu, 6 Feb 2020 16:14:40 -0500 Subject: [PATCH 052/281] Update RFPDupeFilter line separator for correct universal newlines mode usage (#4283) --- scrapy/dupefilters.py | 2 +- tests/test_dupefilters.py | 48 +++++++++++++++++++++++++++++++-------- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index ea6a4cfc3..a36c8304f 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -49,7 +49,7 @@ class RFPDupeFilter(BaseDupeFilter): return True self.fingerprints.add(fp) if self.file: - self.file.write(fp + os.linesep) + self.file.write(fp + '\n') def request_fingerprint(self, request): return request_fingerprint(request) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 0546558bc..88ce9627f 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -2,6 +2,8 @@ import hashlib import tempfile import unittest import shutil +import os +import sys from testfixtures import LogCapture from scrapy.dupefilters import RFPDupeFilter @@ -84,17 +86,21 @@ class RFPDupeFilterTest(unittest.TestCase): path = tempfile.mkdtemp() try: df = RFPDupeFilter(path) - df.open() - assert not df.request_seen(r1) - assert df.request_seen(r1) - df.close('finished') + try: + df.open() + assert not df.request_seen(r1) + assert df.request_seen(r1) + finally: + df.close('finished') df2 = RFPDupeFilter(path) - df2.open() - assert df2.request_seen(r1) - assert not df2.request_seen(r2) - assert df2.request_seen(r2) - df2.close('finished') + try: + df2.open() + assert df2.request_seen(r1) + assert not df2.request_seen(r2) + assert df2.request_seen(r2) + finally: + df2.close('finished') finally: shutil.rmtree(path) @@ -129,6 +135,30 @@ class RFPDupeFilterTest(unittest.TestCase): case_insensitive_dupefilter.close('finished') + def test_seenreq_newlines(self): + """ Checks against adding duplicate \r to + line endings on Windows platforms. """ + + r1 = Request('http://scrapytest.org/1') + + path = tempfile.mkdtemp() + try: + df = RFPDupeFilter(path) + df.open() + df.request_seen(r1) + df.close('finished') + + with open(os.path.join(path, 'requests.seen'), 'rb') as seen_file: + line = next(seen_file).decode() + assert not line.endswith('\r\r\n') + if sys.platform == 'win32': + assert line.endswith('\r\n') + else: + assert line.endswith('\n') + + finally: + shutil.rmtree(path) + def test_log(self): with LogCapture() as l: settings = {'DUPEFILTER_DEBUG': False, From 4f31c3ce017db2b4f69949a81efd56fee60ee32d Mon Sep 17 00:00:00 2001 From: Joy Bhalla Date: Fri, 7 Feb 2020 02:51:33 +0530 Subject: [PATCH 053/281] Document a backward incompatibility that may affect custom schedulers (#4274) --- docs/news.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 6d0d4b4ee..e4b985c77 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -288,6 +288,13 @@ Backward-incompatible changes :class:`~scrapy.http.Request` objects instead of arbitrary Python data structures. +* An additional ``crawler`` parameter has been added to the ``__init__`` method + of the :class:`scrapy.core.scheduler.Scheduler` class. + Custom scheduler subclasses which don't accept arbitrary parameters in + their ``__init__`` method might break because of this change. + + For more information, refer to the documentation for the :setting:`SCHEDULER` setting. + See also :ref:`1.7-deprecation-removals` below. From 84b55b73646acca71461366ef98a1a501331f5d8 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 7 Feb 2020 11:07:35 +0500 Subject: [PATCH 054/281] Update docs/topics/signals.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 47be6b603..60d9ce2bc 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -306,7 +306,7 @@ request_left_downloader The signal does not support returning deferreds from their handlers. - :param request: the request that reached downloader + :param request: the request that reached the downloader :type request: :class:`~scrapy.http.Request` object :param spider: the spider that yielded the request From 2f83f3e2cb3497e89d42533b8f20f8398a696f46 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 7 Feb 2020 11:07:43 +0500 Subject: [PATCH 055/281] Update docs/topics/signals.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 60d9ce2bc..49475c1af 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -301,7 +301,7 @@ request_left_downloader .. signal:: request_left_downloader .. function:: request_left_downloader(request, spider) - Sent when a :class:`~scrapy.http.Request` leaves the downloader even in case of + Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of failure. The signal does not support returning deferreds from their handlers. From 8817b9e8e92f01147e7e44dd767165766093f408 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 7 Feb 2020 11:07:53 +0500 Subject: [PATCH 056/281] Update docs/topics/signals.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 49475c1af..a7d60e9cb 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -296,7 +296,7 @@ request_reached_downloader :type spider: :class:`~scrapy.spiders.Spider` object request_left_downloader ---------------------------- +----------------------- .. signal:: request_left_downloader .. function:: request_left_downloader(request, spider) From 153b78e53f5c0f4630d09d560e111ca68f357905 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 7 Feb 2020 11:08:55 +0500 Subject: [PATCH 057/281] Update docs/topics/signals.rst Co-Authored-By: elacuesta --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index a7d60e9cb..886d1b866 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -304,7 +304,7 @@ request_left_downloader Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of failure. - The signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param request: the request that reached the downloader :type request: :class:`~scrapy.http.Request` object From 31f6c7112fe8efce2105983b8350b1dabdce7a1c Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 7 Feb 2020 17:14:52 +0500 Subject: [PATCH 058/281] Add a test for an async callbacks that returns requests. --- tests/spiders.py | 18 ++++++++++++++++++ tests/test_crawl.py | 14 ++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/spiders.py b/tests/spiders.py index 3b1ee94b8..284c77829 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -117,6 +117,24 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider): return [{'id': 1}, {'id': 2}] +class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): + + name = 'asyncdef_asyncio_reqs_return' + + async def parse(self, response): + await asyncio.sleep(0.2) + req_id = response.meta.get('req_id', 0) + status = await get_from_asyncio_queue(response.status) + self.logger.info("Got response %d, req_id %d" % (status, req_id)) + if req_id > 0: + return + reqs = [] + for i in range(1, 3): + req = Request(self.start_urls[0], dont_filter=True, meta={'req_id': i}) + reqs.append(req) + return reqs + + class ItemSpider(FollowAllSpider): name = 'item' diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 85005eba4..b4b5bac1c 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -13,7 +13,8 @@ from scrapy.utils.python import to_unicode from tests.mockserver import MockServer from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback, - AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider) + AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider, + AsyncDefAsyncioReqsReturnSpider) class CrawlTestCase(TestCase): @@ -330,7 +331,7 @@ with multiples lines @mark.only_asyncio() @defer.inlineCallbacks - def test_async_def_asyncio_parse_list(self): + def test_async_def_asyncio_parse_items_list(self): items = [] def _on_item_scraped(item): @@ -343,3 +344,12 @@ with multiples lines self.assertIn("Got response 200", str(log)) self.assertIn({'id': 1}, items) self.assertIn({'id': 2}, items) + + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_asyncio_parse_reqs_list(self): + crawler = self.runner.create_crawler(AsyncDefAsyncioReqsReturnSpider) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + for req_id in range(3): + self.assertIn("Got response 200, req_id %d" % req_id, str(log)) From 7323780c97e69b560bf9a4bd7e6ccd60fb2b8f13 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 31 Dec 2019 16:15:41 +0500 Subject: [PATCH 059/281] Support yield in async def callbacks. --- conftest.py | 4 ++- scrapy/utils/py36.py | 10 ++++++++ scrapy/utils/spider.py | 8 ++++++ tests/py36/_test_crawl.py | 50 ++++++++++++++++++++++++++++++++++++++ tests/test_crawl.py | 51 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 scrapy/utils/py36.py create mode 100644 tests/py36/_test_crawl.py diff --git a/conftest.py b/conftest.py index c0de09909..be5fbabf4 100644 --- a/conftest.py +++ b/conftest.py @@ -11,7 +11,9 @@ collect_ignore = [ # not a test, but looks like a test "scrapy/utils/testsite.py", # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess - *_py_files("tests/CrawlerProcess") + *_py_files("tests/CrawlerProcess"), + # Py36-only parts of respective tests + *_py_files("tests/py36"), ] for line in open('tests/ignores.txt'): diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py new file mode 100644 index 000000000..c8c24076e --- /dev/null +++ b/scrapy/utils/py36.py @@ -0,0 +1,10 @@ +""" +Helpers using Python 3.6+ syntax (ignore SyntaxError on import). +""" + + +async def collect_asyncgen(result): + results = [] + async for x in result: + results.append(x) + return results diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 72775df5c..4e2a4d1bc 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,12 +4,20 @@ import inspect from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter +try: + from scrapy.utils.py36 import collect_asyncgen +except SyntaxError: + collect_asyncgen = None logger = logging.getLogger(__name__) def iterate_spider_output(result): + if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result): + d = deferred_from_coro(collect_asyncgen(result)) + d.addCallback(iterate_spider_output) + return d return arg_to_iter(deferred_from_coro(result)) diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py new file mode 100644 index 000000000..74c7daf53 --- /dev/null +++ b/tests/py36/_test_crawl.py @@ -0,0 +1,50 @@ +import asyncio + +from scrapy import Request +from tests.spiders import SimpleSpider + + +class AsyncDefAsyncioGenSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen' + + async def parse(self, response): + await asyncio.sleep(0.2) + yield {'foo': 42} + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioGenLoopSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen_loop' + + async def parse(self, response): + for i in range(10): + await asyncio.sleep(0.1) + yield {'foo': i} + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioGenComplexSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen_complex' + initial_reqs = 4 + following_reqs = 3 + depth = 2 + + def _get_req(self, index): + return Request(self.mockserver.url("/status?n=200&request=%d" % index), + meta={'index': index}) + + def start_requests(self): + for i in range(self.initial_reqs): + yield self._get_req(i) + + async def parse(self, response): + index = response.meta['index'] + yield {'index': index} + if index < 10 ** self.depth: + for new_index in range(10 * index, 10 * index + self.following_reqs): + yield self._get_req(new_index) + await asyncio.sleep(0.1) + yield {'index': index + 5} diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 85005eba4..856068465 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,5 +1,6 @@ import json import logging +import sys from pytest import mark from testfixtures import LogCapture @@ -343,3 +344,53 @@ with multiples lines self.assertIn("Got response 200", str(log)) self.assertIn({'id': 1}, items) self.assertIn({'id': 2}, items) + + @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_asyncgen_parse(self): + from tests.py36._test_crawl import AsyncDefAsyncioGenSpider + crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + self.assertIn("Got response 200", str(log)) + itemcount = crawler.stats.get_value('item_scraped_count') + self.assertEqual(itemcount, 1) + + @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_asyncgen_parse_loop(self): + items = [] + + def _on_item_scraped(item): + items.append(item) + + from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider + crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + self.assertIn("Got response 200", str(log)) + itemcount = crawler.stats.get_value('item_scraped_count') + self.assertEqual(itemcount, 10) + for i in range(10): + self.assertIn({'foo': i}, items) + + @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_asyncgen_parse_complex(self): + items = [] + + def _on_item_scraped(item): + items.append(item) + + from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider + crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + yield crawler.crawl(mockserver=self.mockserver) + itemcount = crawler.stats.get_value('item_scraped_count') + self.assertEqual(itemcount, 80) + for i in [0, 3, 21, 22, 207, 311]: # some random items + self.assertIn({'index': i}, items) From 59653ebac609dc11c9d3b29624972d2aaddd5541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 7 Feb 2020 21:07:57 +0100 Subject: [PATCH 060/281] Update installation instructions regarding Python 3 and virtual environments --- docs/intro/install.rst | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 178be723c..49968437c 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -81,35 +81,18 @@ Python packages can be installed either globally (a.k.a system wide), or in user-space. We do not recommend installing Scrapy system wide. Instead, we recommend that you install Scrapy within a so-called -"virtual environment" (`virtualenv`_). -Virtualenvs allow you to not conflict with already-installed Python +"virtual environment" (:mod:`venv`). +Virtual environments allow you to not conflict with already-installed Python system packages (which could break some of your system tools and scripts), and still install packages normally with ``pip`` (without ``sudo`` and the likes). -To get started with virtual environments, see `virtualenv installation instructions`_. -To install it globally (having it globally installed actually helps here), -it should be a matter of running:: +See :ref:`tut-venv` on how to create your virtual environment. - $ [sudo] pip install virtualenv - -Check this `user guide`_ on how to create your virtualenv. - -.. note:: - If you use Linux or OS X, `virtualenvwrapper`_ is a handy tool to create virtualenvs. - -Once you have created a virtualenv, you can install Scrapy inside it with ``pip``, +Once you have created a virtual environment, you can install Scrapy inside it with ``pip``, just like any other Python package. (See :ref:`platform-specific guides ` below for non-Python dependencies that you may need to install beforehand). -Python virtualenvs can be created to use Python 2 by default, or Python 3 by default. As Scrapy -only supports Python 3, make sure you created a Python 3 virtualenv. - -.. _virtualenv: https://virtualenv.pypa.io -.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/ -.. _virtualenvwrapper: https://virtualenvwrapper.readthedocs.io/en/latest/install.html -.. _user guide: https://virtualenv.pypa.io/en/stable/userguide/ - .. _intro-install-platform-notes: @@ -205,15 +188,12 @@ solutions: brew update; brew upgrade python -* *(Optional)* Install Scrapy inside an isolated python environment. +* *(Optional)* :ref:`Install Scrapy inside a Python virtual environment + `. This method is a workaround for the above OS X issue, but it's an overall good practice for managing dependencies and can complement the first method. - `virtualenv`_ is a tool you can use to create virtual environments in python. - We recommended reading a tutorial like - http://docs.python-guide.org/en/latest/dev/virtualenvs/ to get started. - After any of these workarounds you should be able to install Scrapy:: pip install Scrapy From 35723d76c0c07575309810e776305e9ea22fc18d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 7 Feb 2020 22:59:53 +0100 Subject: [PATCH 061/281] Use canonicalize_url in link extraction --- scrapy/linkextractors/lxmlhtml.py | 4 ++-- tests/test_linkextractors.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index fdfa92370..da525d52e 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -9,7 +9,7 @@ from w3lib.url import canonicalize_url from scrapy.link import Link from scrapy.utils.misc import arg_to_iter, rel_has_nofollow -from scrapy.utils.python import unique as unique_list, to_unicode +from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url from scrapy.linkextractors import FilteringLinkExtractor @@ -66,7 +66,7 @@ class LxmlParserLinkExtractor(object): url = self.process_attr(attr_val) if url is None: continue - url = to_unicode(url, encoding=response_encoding) + url = canonicalize_url(url, encoding=response_encoding) # to fix relative links after process_value url = urljoin(response_url, url) link = Link(url, _collect_string_content(el) or u'', diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 38fb8fb4a..e9d6c0abe 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -2,8 +2,6 @@ import re import unittest from warnings import catch_warnings -import pytest - from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link @@ -214,7 +212,7 @@ class Base: response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15') links = self.extractor_cls(restrict_xpaths='//p').extract_links(response) self.assertEqual(links, - [Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')]) + [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')]) def test_restrict_xpaths_concat_in_handle_data(self): """html entities cause SGMLParser to call handle_data hook twice""" @@ -506,7 +504,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), ]) - @pytest.mark.xfail def test_restrict_xpaths_with_html_entities(self): super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() From a64fa2f0866c10594f1e5cf00a0161f9fea1eb62 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 10 Feb 2020 10:16:05 -0300 Subject: [PATCH 062/281] Keyword arguments when creating a _ResponseReader --- scrapy/core/downloader/handlers/http11.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 49c9eacac..7a1a77b23 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -417,14 +417,14 @@ class ScrapyAgent: d = defer.Deferred(_cancel) txresponse.deliverBody( _ResponseReader( - d, - txresponse, - request, - maxsize, - warnsize, - fail_on_dataloss, - self._crawler, - self._source, + finished=d, + txresponse=txresponse, + request=request, + maxsize=maxsize, + warnsize=warnsize, + fail_on_dataloss=fail_on_dataloss, + crawler=self._crawler, + source=self._source, ) ) From 122ce6d6fb3861d99ba2f2810b2370056bae1190 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 10 Feb 2020 10:20:26 -0300 Subject: [PATCH 063/281] Check bytes are received in order (bytes_received signal) --- tests/test_engine.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_engine.py b/tests/test_engine.py index c83a23b55..0d970928b 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -12,7 +12,6 @@ module with the ``runserver`` argument:: import os import re -import string import sys from collections import defaultdict from urllib.parse import urlparse @@ -91,7 +90,8 @@ def start_test_site(debug=False): r = static.File(root_dir) r.putChild(b"redirect", util.Redirect(b"/redirected")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) - r.putChild(b"random", static.Data(string.ascii_letters.encode("utf8") * 2**14, "text/plain")) + numbers = [str(x).encode("utf8") for x in range(2**14)] + r.putChild(b"numbers", static.Data(b"".join(numbers), "text/plain")) port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") if debug: @@ -124,7 +124,7 @@ class CrawlerRun(object): self.geturl("/"), self.geturl("/redirect"), self.geturl("/redirect"), # duplicate - self.geturl("/random"), + self.geturl("/numbers"), ] for name, signal in vars(signals).items(): @@ -315,8 +315,12 @@ class EngineTest(unittest.TestCase): b" \n" b"\n" ) - elif self.run.getpath(request.url) == "/random": - self.assertTrue(len(data) > 1) # signal was fired multiple times + elif self.run.getpath(request.url) == "/numbers": + # signal was fired multiple times + self.assertTrue(len(data) > 1) + # bytes were received in order + numbers = [str(x).encode("utf8") for x in range(2**14)] + self.assertEqual(joined_data, b"".join(numbers)) def _assert_signals_caught(self): assert signals.engine_started in self.run.signals_caught From 42b4e9b3372ce3f9da57c7512b31a3c455b8a161 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 10 Feb 2020 11:23:38 -0300 Subject: [PATCH 064/281] Reword signal docs --- docs/topics/signals.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3a15bf95c..dfb87cef3 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -279,7 +279,7 @@ request_scheduled Sent when the engine schedules a :class:`~scrapy.http.Request`, to be downloaded later. - The signal does not support returning deferreds from its handlers. + This signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler :type request: :class:`~scrapy.http.Request` object @@ -296,7 +296,7 @@ request_dropped Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be downloaded later, is rejected by the scheduler. - The signal does not support returning deferreds from its handlers. + This signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler :type request: :class:`~scrapy.http.Request` object @@ -312,7 +312,7 @@ request_reached_downloader Sent when a :class:`~scrapy.http.Request` reached downloader. - The signal does not support returning deferreds from its handlers. + This signal does not support returning deferreds from its handlers. :param request: the request that reached downloader :type request: :class:`~scrapy.http.Request` object From 13ba9bc629cb0a77ebaca36a10a0a4984d7cce68 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 10 Feb 2020 12:29:39 -0300 Subject: [PATCH 065/281] Note about Response.ip_address --- docs/topics/request-response.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 17eb63064..89e570028 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -685,6 +685,8 @@ Response objects .. attribute:: Response.ip_address The IP address of the server from which the Response originated. + This attribute is currently only populated by the HTTP 1.1 download + handler, i.e. for ``http(s)`` responses. .. method:: Response.copy() From 4626e90df8ba4a945bb9cd6be47a915788e76f23 Mon Sep 17 00:00:00 2001 From: Abhishek Pratap Singh <35230163+Prime-5@users.noreply.github.com> Date: Mon, 10 Feb 2020 18:48:31 +0000 Subject: [PATCH 066/281] Allow updating flags in follow and follow_all (#4279) --- scrapy/http/response/__init__.py | 7 +++++-- scrapy/http/response/text.py | 6 ++++-- tests/test_http_response.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f92d0901c..027fbac6f 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -107,7 +107,7 @@ class Response(object_ref): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, cb_kwargs=None): + dont_filter=False, errback=None, cb_kwargs=None, flags=None): # type: (...) -> Request """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -124,6 +124,7 @@ class Response(object_ref): elif url is None: raise ValueError("url can't be None") url = self.urljoin(url) + return Request( url=url, callback=callback, @@ -137,11 +138,12 @@ class Response(object_ref): dont_filter=dont_filter, errback=errback, cb_kwargs=cb_kwargs, + flags=flags, ) def follow_all(self, urls, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, cb_kwargs=None): + dont_filter=False, errback=None, cb_kwargs=None, flags=None): # type: (...) -> Generator[Request, None, None] """ Return an iterable of :class:`~.Request` instances to follow all links @@ -169,6 +171,7 @@ class Response(object_ref): dont_filter=dont_filter, errback=errback, cb_kwargs=cb_kwargs, + flags=flags, ) for url in urls ) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 09049c157..33a485328 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -121,7 +121,7 @@ class TextResponse(Response): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding=None, priority=0, - dont_filter=False, errback=None, cb_kwargs=None): + dont_filter=False, errback=None, cb_kwargs=None, flags=None): # type: (...) -> Request """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -157,11 +157,12 @@ class TextResponse(Response): dont_filter=dont_filter, errback=errback, cb_kwargs=cb_kwargs, + flags=flags, ) def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding=None, priority=0, - dont_filter=False, errback=None, cb_kwargs=None, + dont_filter=False, errback=None, cb_kwargs=None, flags=None, css=None, xpath=None): # type: (...) -> Generator[Request, None, None] """ @@ -214,6 +215,7 @@ class TextResponse(Response): dont_filter=dont_filter, errback=errback, cb_kwargs=cb_kwargs, + flags=flags, ) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 4c1b2afc3..ff487cfa3 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -166,6 +166,10 @@ class BaseResponseTest(unittest.TestCase): def test_follow_whitespace_link(self): self._assert_followed_url(Link('http://example.com/foo '), 'http://example.com/foo%20') + def test_follow_flags(self): + res = self.response_class('http://example.com/') + fol = res.follow('http://example.com/', flags=['cached', 'allowed']) + self.assertEqual(fol.flags, ['cached', 'allowed']) # Response.follow_all @@ -232,6 +236,17 @@ class BaseResponseTest(unittest.TestCase): expected = [u.replace(' ', '%20') for u in absolute] self._assert_followed_all_urls(links, expected) + def test_follow_all_flags(self): + re = self.response_class('http://www.example.com/') + urls = [ + 'http://www.example.com/', + 'http://www.example.com/2', + 'http://www.example.com/foo', + ] + fol = re.follow_all(urls, flags=['cached', 'allowed']) + for req in fol: + self.assertEqual(req.flags, ['cached', 'allowed']) + def _assert_followed_url(self, follow_obj, target_url, response=None): if response is None: response = self._links_response() @@ -562,6 +577,22 @@ class TextResponseTest(BaseResponseTest): ) self.assertEqual(req.encoding, 'cp1251') + def test_follow_flags(self): + res = self.response_class('http://example.com/') + fol = res.follow('http://example.com/', flags=['cached', 'allowed']) + self.assertEqual(fol.flags, ['cached', 'allowed']) + + def test_follow_all_flags(self): + re = self.response_class('http://www.example.com/') + urls = [ + 'http://www.example.com/', + 'http://www.example.com/2', + 'http://www.example.com/foo', + ] + fol = re.follow_all(urls, flags=['cached', 'allowed']) + for req in fol: + self.assertEqual(req.flags, ['cached', 'allowed']) + def test_follow_all_css(self): expected = [ 'http://example.com/sample3.html', From 037ae5b22e6d6600dc537ee5073652ce74e5f47b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 10 Feb 2020 19:54:47 +0100 Subject: [PATCH 067/281] =?UTF-8?q?Explicitly=20indicate=20None=20as=20ip?= =?UTF-8?q?=5Faddress=E2=80=99s=20default=20value?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/request-response.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 89e570028..8f2504a33 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -685,8 +685,10 @@ Response objects .. attribute:: Response.ip_address The IP address of the server from which the Response originated. + This attribute is currently only populated by the HTTP 1.1 download - handler, i.e. for ``http(s)`` responses. + handler, i.e. for ``http(s)`` responses. For other handlers, + :attr:`ip_address` is always ``None``. .. method:: Response.copy() From 36dcf901849014d7db00a0294ed86c6cc79b5cc6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 11 Feb 2020 00:57:58 +0500 Subject: [PATCH 068/281] Also test non-default async callbacks. --- tests/py36/_test_crawl.py | 13 ++++++++++--- tests/test_crawl.py | 7 +++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py index 74c7daf53..162a53760 100644 --- a/tests/py36/_test_crawl.py +++ b/tests/py36/_test_crawl.py @@ -32,12 +32,14 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): following_reqs = 3 depth = 2 - def _get_req(self, index): + def _get_req(self, index, cb=None): return Request(self.mockserver.url("/status?n=200&request=%d" % index), - meta={'index': index}) + meta={'index': index}, + dont_filter=True, + callback=cb) def start_requests(self): - for i in range(self.initial_reqs): + for i in range(1, self.initial_reqs + 1): yield self._get_req(i) async def parse(self, response): @@ -46,5 +48,10 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): if index < 10 ** self.depth: for new_index in range(10 * index, 10 * index + self.following_reqs): yield self._get_req(new_index) + yield self._get_req(index, cb=self.parse2) await asyncio.sleep(0.1) yield {'index': index + 5} + + async def parse2(self, response): + await asyncio.sleep(0.1) + yield {'index2': response.meta['index']} diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 626000147..64819acb6 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -392,9 +392,12 @@ with multiples lines crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) itemcount = crawler.stats.get_value('item_scraped_count') - self.assertEqual(itemcount, 80) - for i in [0, 3, 21, 22, 207, 311]: # some random items + self.assertEqual(itemcount, 156) + # some random items + for i in [1, 4, 21, 22, 207, 311]: self.assertIn({'index': i}, items) + for i in [10, 30, 122]: + self.assertIn({'index2': i}, items) @mark.only_asyncio() @defer.inlineCallbacks From 1f0f52cbf7bdc9f11f7b83c482ad52ad7ad32ba0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 11 Feb 2020 01:05:45 +0500 Subject: [PATCH 069/281] Improve async signal tests. --- tests/test_signals.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/test_signals.py b/tests/test_signals.py index 001e798e5..d6ae526be 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -1,8 +1,9 @@ +from pytest import mark from twisted.internet import defer from twisted.trial import unittest from scrapy import signals, Request, Spider -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.mockserver import MockServer @@ -11,12 +12,12 @@ class ItemSpider(Spider): name = 'itemspider' def start_requests(self): - for _ in range(10): - yield Request(self.mockserver.url('/status?n=200'), - dont_filter=True) + for index in range(10): + yield Request(self.mockserver.url('/status?n=200&id=%d' % index), + meta={'index': index}) def parse(self, response): - return {'field': 42} + return {'index': response.meta['index']} class AsyncSignalTestCase(unittest.TestCase): @@ -29,11 +30,15 @@ class AsyncSignalTestCase(unittest.TestCase): self.mockserver.__exit__(None, None, None) async def _on_item_scraped(self, item): + item = await get_from_asyncio_queue(item) self.items.append(item) + @mark.only_asyncio() @defer.inlineCallbacks def test_simple_pipeline(self): crawler = get_crawler(ItemSpider) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(self.items), 10) + for index in range(10): + self.assertIn({'index': index}, self.items) From 61e74bac765de0f786d2125e876e4d7934f1722b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 10 Feb 2020 21:57:21 +0100 Subject: [PATCH 070/281] Extract links with safe_url_string canonicalize_url changes links in undesirable ways. --- scrapy/linkextractors/lxmlhtml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index da525d52e..f5ef56ea4 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -5,7 +5,7 @@ from urllib.parse import urljoin import lxml.etree as etree from w3lib.html import strip_html5_whitespace -from w3lib.url import canonicalize_url +from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link from scrapy.utils.misc import arg_to_iter, rel_has_nofollow @@ -66,7 +66,7 @@ class LxmlParserLinkExtractor(object): url = self.process_attr(attr_val) if url is None: continue - url = canonicalize_url(url, encoding=response_encoding) + url = safe_url_string(url, encoding=response_encoding) # to fix relative links after process_value url = urljoin(response_url, url) link = Link(url, _collect_string_content(el) or u'', From 2d6d4fb2335ef24b1efca67dcedf5d264a642e0f Mon Sep 17 00:00:00 2001 From: Drew Seibert Date: Tue, 11 Feb 2020 03:35:23 -0600 Subject: [PATCH 071/281] Deprecate overriding settings with SCRAPY-prefixed environment variables (#4300) --- scrapy/utils/project.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index f28c2eaa1..d9a03ff63 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -68,7 +68,6 @@ def get_project_settings(): if settings_module_path: settings.setmodule(settings_module_path, priority='project') - # XXX: remove this hack pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE") if pickled_settings: warnings.warn("Use of environment variable " @@ -76,10 +75,9 @@ def get_project_settings(): "is deprecated.", ScrapyDeprecationWarning) settings.setdict(pickle.loads(pickled_settings), priority='project') - # XXX: deprecate and remove this functionality env_overrides = {k[7:]: v for k, v in os.environ.items() if k.startswith('SCRAPY_')} if env_overrides: + warnings.warn("Use of 'SCRAPY_'-prefixed environment variables to override settings is deprecated.", ScrapyDeprecationWarning) settings.setdict(env_overrides, priority='project') - return settings From b4958358e89b6611f7dd852684ba6d599831fe27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Feb 2020 19:00:04 +0100 Subject: [PATCH 072/281] Update tests to account for link extractors escaping spaces --- tests/test_linkextractors.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index e9d6c0abe..53968e60e 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -14,7 +14,6 @@ from tests import get_testdata class Base: class LinkExtractorTestCase(unittest.TestCase): extractor_cls = None - escapes_whitespace = False def setUp(self): body = get_testdata('link_extractor', 'linkextractor.html') @@ -28,10 +27,7 @@ class Base: def test_extract_all_links(self): lx = self.extractor_cls() - if self.escapes_whitespace: - page4_url = 'http://example.com/page%204.html' - else: - page4_url = 'http://example.com/page 4.html' + page4_url = 'http://example.com/page%204.html' self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), @@ -308,10 +304,7 @@ class Base: def test_attrs(self): lx = self.extractor_cls(attrs="href") - if self.escapes_whitespace: - page4_url = 'http://example.com/page%204.html' - else: - page4_url = 'http://example.com/page 4.html' + page4_url = 'http://example.com/page%204.html' self.assertEqual(lx.extract_links(self.response), [ Link(url='http://example.com/sample1.html', text=u''), From df937d8280fe0781f6cf1715a3a0cd28c6e94eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Feb 2020 22:33:36 +0100 Subject: [PATCH 073/281] Implement Response.cb_kwargs --- docs/topics/request-response.rst | 12 ++++++++++++ scrapy/http/response/__init__.py | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 4cf367d96..05b7bb5c7 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -672,6 +672,18 @@ Response objects .. seealso:: :attr:`Request.meta` attribute + .. attribute:: Response.cb_kwargs + + A shortcut to the :attr:`Request.cb_kwargs` attribute of the + :attr:`Response.request` object (ie. ``self.request.cb_kwargs``). + + Unlike the :attr:`Response.request` attribute, the + :attr:`Response.cb_kwargs` attribute is propagated along redirects and + retries, so you will get the original :attr:`Request.cb_kwargs` sent + from your spider. + + .. seealso:: :attr:`Request.cb_kwargs` attribute + .. attribute:: Response.flags A list that contains flags for this response. Flags are labels used for diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 64e9c6c20..ee9720d52 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -24,6 +24,16 @@ class Response(object_ref): self.request = request self.flags = [] if flags is None else list(flags) + @property + def cb_kwargs(self): + try: + return self.request.cb_kwargs + except AttributeError: + raise AttributeError( + "Response.cb_kwargs not available, this response " + "is not tied to any request" + ) + @property def meta(self): try: From 5ff9eb90ea9d533d3f960db75071f0fe638503ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Feb 2020 22:36:18 +0100 Subject: [PATCH 074/281] Add a test for the copy of cb_kwargs from Request to Response --- tests/test_http_response.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 960ecea3e..39f5fe750 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -72,6 +72,12 @@ class BaseResponseTest(unittest.TestCase): r1 = self.response_class("http://www.example.com", body=b"Some body", request=req) assert r1.meta is req.meta + def test_copy_cb_kwargs(self): + req = Request("http://www.example.com") + req.cb_kwargs['foo'] = 'bar' + r1 = self.response_class("http://www.example.com", body=b"Some body", request=req) + assert r1.cb_kwargs is req.cb_kwargs + def test_copy_inherited_classes(self): """Test Response children copies preserve their class""" From 43b43654a1dacae7b63fc067dc69929c262d9a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Feb 2020 22:39:58 +0100 Subject: [PATCH 075/281] Add tests for meta and cb_kwargs not being available --- tests/test_http_response.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 39f5fe750..5a19f9d54 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -78,6 +78,16 @@ class BaseResponseTest(unittest.TestCase): r1 = self.response_class("http://www.example.com", body=b"Some body", request=req) assert r1.cb_kwargs is req.cb_kwargs + def test_unavailable_meta(self): + r1 = self.response_class("http://www.example.com", body=b"Some body") + with self.assertRaisesRegex(AttributeError, r'Response\.meta not available'): + r1.meta + + def test_unavailable_cb_kwargs(self): + r1 = self.response_class("http://www.example.com", body=b"Some body") + with self.assertRaisesRegex(AttributeError, r'Response\.cb_kwargs not available'): + r1.cb_kwargs + def test_copy_inherited_classes(self): """Test Response children copies preserve their class""" From 5ae3e1678fa99b3c44cb8981079df51ec34b860f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 14 Feb 2020 22:30:36 +0100 Subject: [PATCH 076/281] =?UTF-8?q?ie.=20=E2=86=92=20i.e.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: elacuesta --- docs/topics/request-response.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 05b7bb5c7..260fe3caf 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -675,7 +675,7 @@ Response objects .. attribute:: Response.cb_kwargs A shortcut to the :attr:`Request.cb_kwargs` attribute of the - :attr:`Response.request` object (ie. ``self.request.cb_kwargs``). + :attr:`Response.request` object (i.e. ``self.request.cb_kwargs``). Unlike the :attr:`Response.request` attribute, the :attr:`Response.cb_kwargs` attribute is propagated along redirects and From a04dd13cd08f1ff392a8bbe284fa0c8fe8924b57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 14 Feb 2020 22:31:30 +0100 Subject: [PATCH 077/281] =?UTF-8?q?ie.=20=E2=86=92=20i.e.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/request-response.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 260fe3caf..d6c7cbec9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -664,7 +664,7 @@ Response objects .. attribute:: Response.meta A shortcut to the :attr:`Request.meta` attribute of the - :attr:`Response.request` object (ie. ``self.request.meta``). + :attr:`Response.request` object (i.e. ``self.request.meta``). Unlike the :attr:`Response.request` attribute, the :attr:`Response.meta` attribute is propagated along redirects and retries, so you will get @@ -770,7 +770,7 @@ TextResponse objects 1. the encoding passed in the ``__init__`` method ``encoding`` argument 2. the encoding declared in the Content-Type HTTP header. If this - encoding is not valid (ie. unknown), it is ignored and the next + encoding is not valid (i.e. unknown), it is ignored and the next resolution mechanism is tried. 3. the encoding declared in the response body. The TextResponse class From 182445f9d96130b1041ece8c4b2a9e9891107c73 Mon Sep 17 00:00:00 2001 From: Akshay Sharma <42249933+AKSHAYSHARMAJS@users.noreply.github.com> Date: Tue, 18 Feb 2020 22:28:31 +0530 Subject: [PATCH 078/281] =?UTF-8?q?Fix=20a=20spelling=20error:=20ie.=20?= =?UTF-8?q?=E2=86=92=20i.e.=20(#4338)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/intro/tutorial.rst | 2 +- docs/topics/downloader-middleware.rst | 4 ++-- docs/topics/extensions.rst | 6 +++--- docs/topics/feed-exports.rst | 2 +- docs/topics/jobs.rst | 2 +- docs/topics/link-extractors.rst | 2 +- docs/topics/request-response.rst | 4 ++-- docs/topics/selectors.rst | 4 ++-- docs/topics/settings.rst | 6 +++--- docs/topics/signals.rst | 4 ++-- scrapy/shell.py | 2 +- scrapy/utils/misc.py | 4 ++-- scrapy/utils/request.py | 2 +- scrapy/utils/spider.py | 2 +- sep/sep-003.rst | 4 ++-- sep/sep-013.rst | 2 +- sep/sep-021.rst | 2 +- 17 files changed, 27 insertions(+), 27 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index ee10048b5..798fe4a7a 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -212,7 +212,7 @@ using the :ref:`Scrapy shell `. Run:: .. note:: Remember to always enclose urls in quotes when running Scrapy shell from - command-line, otherwise urls containing arguments (ie. ``&`` character) + command-line, otherwise urls containing arguments (i.e. ``&`` character) will not work. On Windows, use double quotes instead:: diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 3ec6e0c17..a83cedcfd 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -259,8 +259,8 @@ COOKIES_DEBUG Default: ``False`` -If enabled, Scrapy will log all cookies sent in requests (ie. ``Cookie`` -header) and all cookies received in responses (ie. ``Set-Cookie`` header). +If enabled, Scrapy will log all cookies sent in requests (i.e. ``Cookie`` +header) and all cookies received in responses (i.e. ``Set-Cookie`` header). Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 0a7455ec9..dc057f6b6 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -63,7 +63,7 @@ but disabled unless the :setting:`HTTPCACHE_ENABLED` setting is set. Disabling an extension ====================== -In order to disable an extension that comes enabled by default (ie. those +In order to disable an extension that comes enabled by default (i.e. those included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to ``None``. For example:: @@ -345,7 +345,7 @@ signal is received. The information dumped is the following: After the stack trace and engine status is dumped, the Scrapy process continues running normally. -This extension only works on POSIX-compliant platforms (ie. not Windows), +This extension only works on POSIX-compliant platforms (i.e. not Windows), because the `SIGQUIT`_ and `SIGUSR2`_ signals are not available on Windows. There are at least two ways to send Scrapy the `SIGQUIT`_ signal: @@ -370,7 +370,7 @@ running normally. For more info see `Debugging in Python`_. -This extension only works on POSIX-compliant platforms (ie. not Windows). +This extension only works on POSIX-compliant platforms (i.e. not Windows). .. _Python debugger: https://docs.python.org/2/library/pdb.html .. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/ diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7481b1a99..1d94807a4 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -301,7 +301,7 @@ FEED_STORE_EMPTY Default: ``False`` -Whether to export empty feeds (ie. feeds with no items). +Whether to export empty feeds (i.e. feeds with no items). .. setting:: FEED_STORAGES diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 8816a028c..c34ba336b 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -22,7 +22,7 @@ Job directory To enable persistence support you just need to define a *job directory* through the ``JOBDIR`` setting. This directory will be for storing all required data to -keep the state of a single job (ie. a spider run). It's important to note that +keep the state of a single job (i.e. a spider run). It's important to note that this directory must not be shared by different spiders, or even different jobs/runs of the same spider, as it's meant to be used for storing the state of a *single* job. diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 2119cb8f8..8c8019438 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -49,7 +49,7 @@ LxmlLinkExtractor :type allow: a regular expression (or list of) :param deny: a single regular expression (or list of regular expressions) - that the (absolute) urls must match in order to be excluded (ie. not + that the (absolute) urls must match in order to be excluded (i.e. not extracted). It has precedence over the ``allow`` parameter. If not given (or empty) it won't exclude any links. :type deny: a regular expression (or list of) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 8997a7f19..34cc41a02 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -664,7 +664,7 @@ Response objects .. attribute:: Response.meta A shortcut to the :attr:`Request.meta` attribute of the - :attr:`Response.request` object (ie. ``self.request.meta``). + :attr:`Response.request` object (i.e. ``self.request.meta``). Unlike the :attr:`Response.request` attribute, the :attr:`Response.meta` attribute is propagated along redirects and retries, so you will get @@ -760,7 +760,7 @@ TextResponse objects 1. the encoding passed in the ``__init__`` method ``encoding`` argument 2. the encoding declared in the Content-Type HTTP header. If this - encoding is not valid (ie. unknown), it is ignored and the next + encoding is not valid (i.e. unknown), it is ignored and the next resolution mechanism is tried. 3. the encoding declared in the response body. The TextResponse class diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 8ec758b0e..c3d431e2a 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -986,7 +986,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this:: sel = Selector(html_response) 1. Select all ``

`` elements from an HTML response body, returning a list of - :class:`Selector` objects (ie. a :class:`SelectorList` object):: + :class:`Selector` objects (i.e. a :class:`SelectorList` object):: sel.xpath("//h1") @@ -1013,7 +1013,7 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object:: sel = Selector(xml_response) 1. Select all ```` elements from an XML response body, returning a list - of :class:`Selector` objects (ie. a :class:`SelectorList` object):: + of :class:`Selector` objects (i.e. a :class:`SelectorList` object):: sel.xpath("//product") diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index fa63a5807..5394147da 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -248,7 +248,7 @@ CONCURRENT_REQUESTS Default: ``16`` -The maximum number of concurrent (ie. simultaneous) requests that will be +The maximum number of concurrent (i.e. simultaneous) requests that will be performed by the Scrapy downloader. .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN @@ -258,7 +258,7 @@ CONCURRENT_REQUESTS_PER_DOMAIN Default: ``8`` -The maximum number of concurrent (ie. simultaneous) requests that will be +The maximum number of concurrent (i.e. simultaneous) requests that will be performed to any single domain. See also: :ref:`topics-autothrottle` and its @@ -272,7 +272,7 @@ CONCURRENT_REQUESTS_PER_IP Default: ``0`` -The maximum number of concurrent (ie. simultaneous) requests that will be +The maximum number of concurrent (i.e. simultaneous) requests that will be performed to any single IP. If non-zero, the :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is ignored, and this one is used instead. In other words, concurrency limits will be applied per IP, not diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 886d1b866..d3cfb0307 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -141,7 +141,7 @@ item_error .. signal:: item_error .. function:: item_error(item, response, spider, failure) - Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises + Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises an exception), except :exc:`~scrapy.exceptions.DropItem` exception. This signal supports returning deferreds from their handlers. @@ -232,7 +232,7 @@ spider_error .. signal:: spider_error .. function:: spider_error(failure, response, spider) - Sent when a spider callback generates an error (ie. raises an exception). + Sent when a spider callback generates an error (i.e. raises an exception). This signal does not support returning deferreds from their handlers. diff --git a/scrapy/shell.py b/scrapy/shell.py index a23b04df9..1d5341973 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -173,7 +173,7 @@ def _request_deferred(request): This returns a Deferred whose first pair of callbacks are the request callback and errback. The Deferred also triggers when the request - callback/errback is executed (ie. when the request is downloaded) + callback/errback is executed (i.e. when the request is downloaded) WARNING: Do not call request.replace() until after the deferred is called. """ diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index cb0ee5af3..a3e55d6ea 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -37,8 +37,8 @@ def arg_to_iter(arg): def load_object(path): """Load an object given its absolute object path, and return it. - object can be a class, function, variable or an instance. - path ie: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware' + object can be the import path of a class, function, variable or an + instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware' """ try: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 356753ab5..b8c140a7e 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -28,7 +28,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False): http://www.example.com/query?cat=222&id=111 Even though those are two different URLs both point to the same resource - and are equivalent (ie. they should return the same response). + and are equivalent (i.e. they should return the same response). Another example are cookies used to store session ids. Suppose the following page is only accessible to authenticated users: diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 72775df5c..e4a2d1ac2 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -15,7 +15,7 @@ def iterate_spider_output(result): def iter_spider_classes(module): """Return an iterator over all spider classes defined in the given module - that can be instantiated (ie. which have name) + that can be instantiated (i.e. which have name) """ # this needs to be imported here until get rid of the spider manager # singleton in scrapy.spider.spiders diff --git a/sep/sep-003.rst b/sep/sep-003.rst index 184839525..e6357313d 100644 --- a/sep/sep-003.rst +++ b/sep/sep-003.rst @@ -18,7 +18,7 @@ Prerequisites This API proposal relies on the following API: -1. instantiating a item with an item instance as its first argument (ie. +1. instantiating a item with an item instance as its first argument (i.e. ``item2 = MyItem(item1)``) must return a **copy** of the first item instance) 2. items can be instantiated using this syntax: ``item = Item(attr1=value1, @@ -78,7 +78,7 @@ Defining an item containing ItemField's variants2 = ListField(ItemField(Variant), default=[]) It's important to note here that the (perhaps most intuitive) way of defining a -Product-Variant relationship (ie. defining a recursive !ItemField) doesn't +Product-Variant relationship (i.e. defining a recursive !ItemField) doesn't work. For example, this fails to compile: :: diff --git a/sep/sep-013.rst b/sep/sep-013.rst index 5b18b7501..4bc9abd30 100644 --- a/sep/sep-013.rst +++ b/sep/sep-013.rst @@ -59,7 +59,7 @@ Global changes to all middlewares To be discussed: -1. should we support returning deferreds (ie. ``maybeDeferred``) in middleware +1. should we support returning deferreds (i.e. ``maybeDeferred``) in middleware methods? 2. should we pass Twisted Failures instead of exceptions to error methods? diff --git a/sep/sep-021.rst b/sep/sep-021.rst index 628a95dd2..372429791 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -38,7 +38,7 @@ Goals: * simple to manage: adding or removing extensions should be just a matter of adding or removing lines in a ``scrapy.cfg`` file -* backward compatibility with enabling extension the "old way" (ie. modifying +* backward compatibility with enabling extension the "old way" (i.e. modifying settings directly) Non-goals: From eb21dae5240d2b66feb72940cdd141dba31ecd7a Mon Sep 17 00:00:00 2001 From: Marc Hernandez Cabot Date: Wed, 19 Feb 2020 17:49:42 +0100 Subject: [PATCH 079/281] deprecare sel shortcut in scrapy shell --- scrapy/shell.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/scrapy/shell.py b/scrapy/shell.py index a23b04df9..e1b4a024e 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -126,7 +126,6 @@ class Shell(object): self.vars['spider'] = spider self.vars['request'] = request self.vars['response'] = response - self.vars['sel'] = _SelectorProxy(response) if self.inthread: self.vars['fetch'] = self.fetch self.vars['view'] = open_in_browser @@ -192,15 +191,3 @@ def _request_deferred(request): request.callback, request.errback = d.callback, d.errback return d - - -class _SelectorProxy(object): - - def __init__(self, response): - self._proxiedresponse = response - - def __getattr__(self, name): - warnings.warn('"sel" shortcut is deprecated. Use "response.xpath()", ' - '"response.css()" or "response.selector" instead', - category=ScrapyDeprecationWarning, stacklevel=2) - return getattr(self._proxiedresponse.selector, name) From 6972a197073af11bcb582cc03f6286fceda5ca6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 19 Feb 2020 18:59:09 +0100 Subject: [PATCH 080/281] Remove unused imports --- scrapy/shell.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/shell.py b/scrapy/shell.py index e1b4a024e..e22c48dc5 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -5,14 +5,13 @@ See documentation in docs/topics/shell.rst """ import os import signal -import warnings from twisted.internet import threads, defer from twisted.python import threadable from w3lib.url import any_to_uri from scrapy.crawler import Crawler -from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning +from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response from scrapy.item import BaseItem from scrapy.settings import Settings From 0f78a591f8796686dc65854c250d6ef5324024aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 19 Feb 2020 19:09:39 +0100 Subject: [PATCH 081/281] =?UTF-8?q?Fix=20Flake8-reported=20=E2=80=9CToo=20?= =?UTF-8?q?many=20blank=20lines=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/core/scraper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 7b62068f5..41f015017 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -18,7 +18,6 @@ from scrapy.item import BaseItem from scrapy.core.spidermw import SpiderMiddlewareManager - logger = logging.getLogger(__name__) From 91bbc70bc10cf326940eaf53294149444f43fb9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= Date: Fri, 21 Feb 2020 06:05:31 +0100 Subject: [PATCH 082/281] fix E30X flake8 (#4355) --- pytest.ini | 67 ++++++++++---------- scrapy/core/downloader/tls.py | 1 + scrapy/core/engine.py | 1 + scrapy/responsetypes.py | 1 + scrapy/utils/console.py | 1 + scrapy/utils/gz.py | 1 + tests/test_command_parse.py | 1 - tests/test_crawler.py | 1 + tests/test_dependencies.py | 1 + tests/test_downloadermiddleware_cookies.py | 1 - tests/test_downloadermiddleware_httpcache.py | 1 + tests/test_downloadermiddleware_redirect.py | 4 +- tests/test_exporters.py | 1 + tests/test_http_response.py | 1 + tests/test_item.py | 1 + tests/test_mail.py | 1 + tests/test_pipeline_files.py | 1 - tests/test_pipeline_images.py | 1 - tests/test_pipeline_media.py | 1 + tests/test_responsetypes.py | 1 + tests/test_utils_conf.py | 1 - tests/test_utils_defer.py | 2 + tests/test_utils_deprecate.py | 3 + tests/test_utils_iterators.py | 1 - tests/test_utils_python.py | 3 +- tests/test_utils_request.py | 1 + tests/test_utils_template.py | 1 + tests/test_utils_url.py | 1 + tests/test_webclient.py | 1 + 29 files changed, 58 insertions(+), 45 deletions(-) diff --git a/pytest.ini b/pytest.ini index 552829d4e..0758d2f8b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -47,17 +47,17 @@ flake8-ignore = scrapy/contracts/__init__.py E501 W504 scrapy/contracts/default.py E128 # scrapy/core - scrapy/core/engine.py E501 E128 E127 E306 E502 + scrapy/core/engine.py E501 E128 E127 E502 scrapy/core/scheduler.py E501 - scrapy/core/scraper.py E501 E306 E128 W504 + scrapy/core/scraper.py E501 E128 W504 scrapy/core/spidermw.py E501 E731 E126 E226 scrapy/core/downloader/__init__.py E501 scrapy/core/downloader/contextfactory.py E501 E128 E126 scrapy/core/downloader/middleware.py E501 E502 - scrapy/core/downloader/tls.py E501 E305 E241 + scrapy/core/downloader/tls.py E501 E241 scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226 scrapy/core/downloader/handlers/__init__.py E501 - scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127 + scrapy/core/downloader/handlers/ftp.py E501 E128 E127 scrapy/core/downloader/handlers/http10.py E501 scrapy/core/downloader/handlers/http11.py E501 scrapy/core/downloader/handlers/s3.py E501 E128 E126 @@ -76,7 +76,7 @@ flake8-ignore = scrapy/extensions/closespider.py E501 E128 E123 scrapy/extensions/corestats.py E501 scrapy/extensions/feedexport.py E128 E501 - scrapy/extensions/httpcache.py E128 E501 E303 + scrapy/extensions/httpcache.py E128 E501 scrapy/extensions/memdebug.py E501 scrapy/extensions/spiderstate.py E501 scrapy/extensions/telnet.py E501 W504 @@ -121,12 +121,11 @@ flake8-ignore = scrapy/utils/asyncio.py E501 scrapy/utils/benchserver.py E501 scrapy/utils/conf.py E402 E501 - scrapy/utils/console.py E306 E305 scrapy/utils/datatypes.py E501 E226 scrapy/utils/decorators.py E501 scrapy/utils/defer.py E501 E128 scrapy/utils/deprecate.py E128 E501 E127 E502 - scrapy/utils/gz.py E305 E501 W504 + scrapy/utils/gz.py E501 W504 scrapy/utils/http.py F403 E226 scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 E701 @@ -161,7 +160,7 @@ flake8-ignore = scrapy/middleware.py E128 E501 scrapy/pqueues.py E501 scrapy/resolver.py E501 - scrapy/responsetypes.py E128 E501 E305 + scrapy/responsetypes.py E128 E501 scrapy/robotstxt.py E501 scrapy/shell.py E501 scrapy/signalmanager.py E501 @@ -175,50 +174,50 @@ flake8-ignore = tests/spiders.py E501 E127 tests/test_closespider.py E501 E127 tests/test_command_fetch.py E501 - tests/test_command_parse.py E501 E128 E303 E226 + tests/test_command_parse.py E501 E128 E226 tests/test_command_shell.py E501 E128 tests/test_commands.py E128 E501 tests/test_contracts.py E501 E128 tests/test_crawl.py E501 E741 E265 - tests/test_crawler.py F841 E306 E501 - tests/test_dependencies.py F841 E501 E305 + tests/test_crawler.py F841 E501 + tests/test_dependencies.py F841 E501 tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 - tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126 + tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126 tests/test_downloadermiddleware_decompression.py E127 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 - tests/test_downloadermiddleware_httpcache.py E501 E305 + tests/test_downloadermiddleware_httpcache.py E501 tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123 tests/test_downloadermiddleware_httpproxy.py E501 E128 - tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305 - tests/test_downloadermiddleware_retry.py E501 E128 E251 E303 E126 + tests/test_downloadermiddleware_redirect.py E501 E128 E127 + tests/test_downloadermiddleware_retry.py E501 E128 E251 E126 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 tests/test_dupefilters.py E221 E501 E741 E128 E124 tests/test_engine.py E401 E501 E128 - tests/test_exporters.py E501 E731 E306 E128 E124 + tests/test_exporters.py E501 E731 E128 E124 tests/test_extension_telnet.py F841 tests/test_feedexport.py E501 F841 E241 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123 - tests/test_http_response.py E501 E301 E128 E265 - tests/test_item.py E701 E128 F841 E306 + tests/test_http_response.py E501 E128 E265 + tests/test_item.py E701 E128 F841 tests/test_link.py E501 tests/test_linkextractors.py E501 E128 E124 - tests/test_loader.py E501 E731 E303 E741 E128 E117 E241 + tests/test_loader.py E501 E731 E741 E128 E117 E241 tests/test_logformatter.py E128 E501 E122 - tests/test_mail.py E128 E501 E305 + tests/test_mail.py E128 E501 tests/test_middleware.py E501 E128 tests/test_pipeline_crawl.py E131 E501 E128 E126 - tests/test_pipeline_files.py E501 E303 E272 E226 - tests/test_pipeline_images.py F841 E501 E303 - tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502 + tests/test_pipeline_files.py E501 E272 E226 + tests/test_pipeline_images.py F841 E501 + tests/test_pipeline_media.py E501 E741 E731 E128 E502 tests/test_proxy_connect.py E501 E741 tests/test_request_cb_kwargs.py E501 - tests/test_responsetypes.py E501 E305 + tests/test_responsetypes.py E501 tests/test_robotstxt_interface.py E501 E501 tests/test_scheduler.py E501 E126 E123 tests/test_selector.py E501 E127 @@ -230,24 +229,22 @@ flake8-ignore = tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121 tests/test_squeues.py E501 E701 E741 tests/test_utils_asyncio.py E501 - tests/test_utils_conf.py E501 E303 E128 + tests/test_utils_conf.py E501 E128 tests/test_utils_curl.py E501 - tests/test_utils_datatypes.py E402 E501 E305 - tests/test_utils_defer.py E306 E501 F841 E226 - tests/test_utils_deprecate.py F841 E306 E501 + tests/test_utils_datatypes.py E402 E501 + tests/test_utils_defer.py E501 F841 E226 + tests/test_utils_deprecate.py F841 E501 tests/test_utils_http.py E501 E128 W504 - tests/test_utils_iterators.py E501 E128 E129 E303 E241 + tests/test_utils_iterators.py E501 E128 E129 E241 tests/test_utils_log.py E741 E226 - tests/test_utils_python.py E501 E303 E731 E701 E305 + tests/test_utils_python.py E501 E731 E701 tests/test_utils_reqser.py E501 E128 - tests/test_utils_request.py E501 E128 E305 + tests/test_utils_request.py E501 E128 tests/test_utils_response.py E501 tests/test_utils_signal.py E741 F841 E731 E226 tests/test_utils_sitemap.py E128 E501 E124 - tests/test_utils_spider.py E305 - tests/test_utils_template.py E305 - tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123 - tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126 + tests/test_utils_url.py E501 E127 E211 E125 E501 E226 E241 E126 E123 + tests/test_webclient.py E501 E128 E122 E402 E226 E241 E123 E126 tests/test_cmdline/__init__.py E501 tests/test_settings/__init__.py E501 E128 tests/test_spiderloader/__init__.py E128 E501 diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 4ed482058..a1c881d5e 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -89,4 +89,5 @@ class ScrapyClientTLSOptions(ClientTLSOptions): 'from host "{}" (exception: {})'.format( self._hostnameASCII, repr(e))) + DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT') diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 829e69993..6ab8cde6b 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -230,6 +230,7 @@ class ExecutionEngine(object): def _download(self, request, spider): slot = self.slot slot.add_request(request) + def _on_success(response): assert isinstance(response, (Response, Request)) if isinstance(response, Response): diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 91d309147..64bf93e86 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -116,4 +116,5 @@ class ResponseTypes(object): cls = self.from_body(body) return cls + responsetypes = ResponseTypes() diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 7eb40f0ce..c7a2ace88 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -54,6 +54,7 @@ def _embed_standard_shell(namespace={}, banner=''): else: import rlcompleter # noqa: F401 readline.parse_and_bind("tab:complete") + @wraps(_embed_standard_shell) def wrapper(namespace=namespace, banner=''): code.interact(banner=banner, local=namespace) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 9672e28da..c291ae237 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -42,6 +42,7 @@ def gunzip(data): raise return b''.join(output_list) + _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index b7035fdff..8a54d2c74 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -147,7 +147,6 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} self.url('/html')]) self.assertIn("DEBUG: It Works!", _textmode(stderr)) - @defer.inlineCallbacks def test_pipelines(self): _, _, stderr = yield self.execute(['--spider', self.spider_name, diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f8fa26def..7bd76601d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -107,6 +107,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): def test_spider_custom_settings_log_level(self): log_file = self.mktemp() + class MySpider(scrapy.Spider): name = 'spider' custom_settings = { diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index e31ccd9b5..a169acbe6 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -13,5 +13,6 @@ class ScrapyUtilsTest(unittest.TestCase): installed_version = [int(x) for x in module.__version__.split('.')[:2]] assert installed_version >= [0, 6], "OpenSSL >= 0.6 required" + if __name__ == "__main__": unittest.main() diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 04884fb78..051f66680 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -145,7 +145,6 @@ class CookiesMiddlewareTest(TestCase): {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'}, {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}] - req = Request('http://scrapytest.org/', cookies=cookies) self.mw.process_request(req, self.spider) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 9401dd66d..9b77c97a8 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -501,5 +501,6 @@ class RFC2616PolicyTest(DefaultStorageTest): self.assertEqualResponse(res1, res2) assert 'cached' in res2.flags + if __name__ == '__main__': unittest.main() diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index e0f145d0e..053e26fc3 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -68,7 +68,6 @@ class RedirectMiddlewareTest(unittest.TestCase): assert isinstance(r, Response) assert r is rsp - def test_redirect_302(self): url = 'http://www.example.com/302' url2 = 'http://www.example.com/redirected2' @@ -122,7 +121,6 @@ class RedirectMiddlewareTest(unittest.TestCase): del rsp.headers['Location'] assert self.mw.process_response(req, rsp, self.spider) is rsp - def test_max_redirect_times(self): self.mw.max_redirect_times = 1 req = Request('http://scrapytest.org/302') @@ -178,6 +176,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_request_meta_handling(self): url = 'http://www.example.com/301' url2 = 'http://www.example.com/redirected' + def _test_passthrough(req): rsp = Response(url, headers={'Location': url2}, status=301, request=req) r = self.mw.process_response(req, rsp, self.spider) @@ -316,5 +315,6 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): response = mw.process_response(req, rsp, self.spider) assert isinstance(response, Response) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 5d1f5c182..6e2507508 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -312,6 +312,7 @@ class XmlItemExporterTest(BaseItemExporterTest): for child in children] else: return [(elem.tag, [(elem.text, ())])] + def xmlsplit(xmlcontent): doc = lxml.etree.fromstring(xmlcontent) return xmltuple(doc) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 0dc603923..be17dfd6b 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -182,6 +182,7 @@ class BaseResponseTest(unittest.TestCase): def test_follow_whitespace_link(self): self._assert_followed_url(Link('http://example.com/foo '), 'http://example.com/foo%20') + def test_follow_flags(self): res = self.response_class('http://example.com/') fol = res.follow('http://example.com/', flags=['cached', 'allowed']) diff --git a/tests/test_item.py b/tests/test_item.py index 30463a0f5..823bf1ced 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -259,6 +259,7 @@ class ItemTest(unittest.TestCase): with catch_warnings(record=True) as warnings: item = Item() self.assertEqual(len(warnings), 0) + class SubclassedItem(Item): pass subclassed_item = SubclassedItem() diff --git a/tests/test_mail.py b/tests/test_mail.py index ddb0f1e70..f5cb81a8b 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -121,5 +121,6 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(text.get_charset(), Charset('utf-8')) self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e5bad2ed0..88ce1cf18 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -286,7 +286,6 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(pipeline.files_result_field, "this") self.assertEqual(pipeline.files_urls_field, "that") - def test_user_defined_subclass_default_key_names(self): """Test situation when user defines subclass of FilesPipeline, but uses attribute names for default pipeline (without prefixing diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 7f1cb4a11..5018d6802 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -177,7 +177,6 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): IMAGES_RESULT_FIELD='images' ) - def setUp(self): self.tempdir = mkdtemp() diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 1fcc5799e..d369e147d 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -304,6 +304,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): return response rsp1 = Response('http://url') + def rsp1_func(): dfd = Deferred().addCallback(_check_downloading) reactor.callLater(.1, dfd.callback, rsp1) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index d5a3371ab..8cdf7a176 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -90,5 +90,6 @@ class ResponseTypesTest(unittest.TestCase): # check that mime.types files shipped with scrapy are loaded self.assertEqual(responsetypes.mimetypes.guess_type('x.scrapytest')[0], 'x-scrapy/test') + if __name__ == "__main__": unittest.main() diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 02d8ba51e..61e110845 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -83,7 +83,6 @@ class BuildComponentListTest(unittest.TestCase): self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - class UtilsConfTestCase(unittest.TestCase): def test_arglist_to_dict(self): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index dfbe71ae2..89b5fb4fb 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -9,6 +9,7 @@ from scrapy.utils.defer import mustbe_deferred, process_chain, \ class MustbeDeferredTest(unittest.TestCase): def test_success_function(self): steps = [] + def _append(v): steps.append(v) return steps @@ -20,6 +21,7 @@ class MustbeDeferredTest(unittest.TestCase): def test_unfired_deferred(self): steps = [] + def _append(v): steps.append(v) dfd = defer.Deferred() diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 159ef8f25..b3a90d314 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -110,6 +110,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): # ignore subclassing warnings with warnings.catch_warnings(): warnings.simplefilter('ignore', ScrapyDeprecationWarning) + class UserClass(Deprecated): pass @@ -233,6 +234,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): with warnings.catch_warnings(record=True) as w: AlsoDeprecated() + class UserClass(AlsoDeprecated): pass @@ -247,6 +249,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): with mock.patch('inspect.stack', side_effect=IndexError): with warnings.catch_warnings(record=True) as w: DeprecatedName = create_deprecated_class('DeprecatedName', NewName) + class SubClass(DeprecatedName): pass diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 9776dfb2a..33fc4d570 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -387,7 +387,6 @@ class TestHelper(unittest.TestCase): self.assertTrue(type(r1) is type(r2)) self.assertTrue(type(r1) is not type(r3)) - def _assert_type_and_value(self, a, b, obj): self.assertTrue(type(a) is type(b), 'Got {}, expected {} for {!r}'.format(type(a), type(b), obj)) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index b79e0ac1c..4202e8c89 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -104,7 +104,6 @@ class BinaryIsTextTest(unittest.TestCase): assert not binary_is_text(b"\x02\xa3") - class UtilsPythonTestCase(unittest.TestCase): def test_equal_attributes(self): @@ -215,7 +214,6 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual( get_func_args(operator.itemgetter(2), stripself=True), ['obj']) - def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4)) @@ -223,5 +221,6 @@ class UtilsPythonTestCase(unittest.TestCase): without_none_values({'one': 1, 'none': None, 'three': 3, 'four': 4}), {'one': 1, 'three': 3, 'four': 4}) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 3e664fc74..45f0f59e4 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -83,5 +83,6 @@ class UtilsRequestTest(unittest.TestCase): request_httprepr(Request("file:///tmp/foo.txt")) request_httprepr(Request("ftp://localhost/tmp/foo.txt")) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 40b733233..5a52dd695 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -38,5 +38,6 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): os.remove(render_path) assert not os.path.exists(render_path) # Failure of test iself + if '__main__' == __name__: unittest.main() diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 21e9a056a..9f1acbc75 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -201,6 +201,7 @@ def create_skipped_scheme_t(args): assert url.startswith(args[1]) return do_expected + for k, args in enumerate ([ ('/index', 'file://'), ('/index.html', 'file://'), diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 746367b41..b602a3ea0 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -294,6 +294,7 @@ class WebClientTestCase(unittest.TestCase): finished = self.assertFailure( getPage(self.getURL("wait"), timeout=0.000001), defer.TimeoutError) + def cleanup(passthrough): # Clean up the server which is hanging around not doing # anything. From 6fb85951ce3843156a801e71441a4a3e387588e2 Mon Sep 17 00:00:00 2001 From: Marc Hernandez Cabot Date: Thu, 20 Feb 2020 16:32:58 +0100 Subject: [PATCH 083/281] fix E22X flake8 --- pytest.ini | 48 ++++++++++----------- scrapy/commands/parse.py | 6 +-- scrapy/core/downloader/webclient.py | 2 +- scrapy/core/spidermw.py | 6 +-- scrapy/downloadermiddlewares/ajaxcrawl.py | 2 +- scrapy/exporters.py | 4 +- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/settings/default_settings.py | 4 +- scrapy/spiderloader.py | 2 +- scrapy/utils/datatypes.py | 6 +-- scrapy/utils/http.py | 2 +- scrapy/utils/misc.py | 2 +- scrapy/utils/reactor.py | 2 +- tests/pipelines.py | 4 +- tests/test_command_parse.py | 8 ++-- tests/test_downloader_handlers.py | 4 +- tests/test_dupefilters.py | 10 ++--- tests/test_pipeline_files.py | 2 +- tests/test_spidermiddleware.py | 2 +- tests/test_spidermiddleware_output_chain.py | 2 +- tests/test_utils_defer.py | 2 +- tests/test_utils_log.py | 2 +- tests/test_utils_signal.py | 2 +- tests/test_utils_url.py | 2 +- tests/test_webclient.py | 10 ++--- 25 files changed, 69 insertions(+), 69 deletions(-) diff --git a/pytest.ini b/pytest.ini index 0758d2f8b..7806620d5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -37,7 +37,7 @@ flake8-ignore = scrapy/commands/edit.py E501 scrapy/commands/fetch.py E401 E501 E128 E731 scrapy/commands/genspider.py E128 E501 E502 - scrapy/commands/parse.py E128 E501 E731 E226 + scrapy/commands/parse.py E128 E501 E731 scrapy/commands/runspider.py E501 scrapy/commands/settings.py E128 scrapy/commands/shell.py E128 E501 E502 @@ -50,19 +50,19 @@ flake8-ignore = scrapy/core/engine.py E501 E128 E127 E502 scrapy/core/scheduler.py E501 scrapy/core/scraper.py E501 E128 W504 - scrapy/core/spidermw.py E501 E731 E126 E226 + scrapy/core/spidermw.py E501 E731 E126 scrapy/core/downloader/__init__.py E501 scrapy/core/downloader/contextfactory.py E501 E128 E126 scrapy/core/downloader/middleware.py E501 E502 scrapy/core/downloader/tls.py E501 E241 - scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226 + scrapy/core/downloader/webclient.py E731 E501 E128 E126 scrapy/core/downloader/handlers/__init__.py E501 scrapy/core/downloader/handlers/ftp.py E501 E128 E127 scrapy/core/downloader/handlers/http10.py E501 scrapy/core/downloader/handlers/http11.py E501 scrapy/core/downloader/handlers/s3.py E501 E128 E126 # scrapy/downloadermiddlewares - scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226 + scrapy/downloadermiddlewares/ajaxcrawl.py E501 scrapy/downloadermiddlewares/decompression.py E501 scrapy/downloadermiddlewares/defaultheaders.py E501 scrapy/downloadermiddlewares/httpcache.py E501 E126 @@ -91,7 +91,7 @@ flake8-ignore = scrapy/http/response/text.py E501 E128 E124 # scrapy/linkextractors scrapy/linkextractors/__init__.py E731 E501 E402 W504 - scrapy/linkextractors/lxmlhtml.py E501 E731 E226 + scrapy/linkextractors/lxmlhtml.py E501 E731 # scrapy/loader scrapy/loader/__init__.py E501 E128 scrapy/loader/processors.py E501 @@ -105,7 +105,7 @@ flake8-ignore = scrapy/selector/unified.py E501 E111 # scrapy/settings scrapy/settings/__init__.py E501 - scrapy/settings/default_settings.py E501 E114 E116 E226 + scrapy/settings/default_settings.py E501 E114 E116 scrapy/settings/deprecated.py E501 # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 @@ -121,21 +121,21 @@ flake8-ignore = scrapy/utils/asyncio.py E501 scrapy/utils/benchserver.py E501 scrapy/utils/conf.py E402 E501 - scrapy/utils/datatypes.py E501 E226 + scrapy/utils/datatypes.py E501 scrapy/utils/decorators.py E501 scrapy/utils/defer.py E501 E128 scrapy/utils/deprecate.py E128 E501 E127 E502 scrapy/utils/gz.py E501 W504 - scrapy/utils/http.py F403 E226 + scrapy/utils/http.py F403 scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 E701 scrapy/utils/log.py E128 E501 scrapy/utils/markup.py F403 - scrapy/utils/misc.py E501 E226 + scrapy/utils/misc.py E501 scrapy/utils/multipart.py F403 scrapy/utils/project.py E501 scrapy/utils/python.py E501 - scrapy/utils/reactor.py E226 E501 + scrapy/utils/reactor.py E501 scrapy/utils/reqser.py E501 scrapy/utils/request.py E127 E501 scrapy/utils/response.py E501 E128 @@ -151,7 +151,7 @@ flake8-ignore = scrapy/crawler.py E501 scrapy/dupefilters.py E501 E202 scrapy/exceptions.py E501 - scrapy/exporters.py E501 E226 + scrapy/exporters.py E501 scrapy/interfaces.py E501 scrapy/item.py E501 E128 scrapy/link.py E501 @@ -164,24 +164,24 @@ flake8-ignore = scrapy/robotstxt.py E501 scrapy/shell.py E501 scrapy/signalmanager.py E501 - scrapy/spiderloader.py E225 F841 E501 E126 + scrapy/spiderloader.py F841 E501 E126 scrapy/squeues.py E128 scrapy/statscollectors.py E501 # tests tests/__init__.py E402 E501 tests/mockserver.py E401 E501 E126 E123 - tests/pipelines.py F841 E226 + tests/pipelines.py F841 tests/spiders.py E501 E127 tests/test_closespider.py E501 E127 tests/test_command_fetch.py E501 - tests/test_command_parse.py E501 E128 E226 + tests/test_command_parse.py E501 E128 tests/test_command_shell.py E501 E128 tests/test_commands.py E128 E501 tests/test_contracts.py E501 E128 tests/test_crawl.py E501 E741 E265 tests/test_crawler.py F841 E501 tests/test_dependencies.py F841 E501 - tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123 + tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E701 E126 E123 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126 @@ -195,7 +195,7 @@ flake8-ignore = tests/test_downloadermiddleware_retry.py E501 E128 E251 E126 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E221 E501 E741 E128 E124 + tests/test_dupefilters.py E501 E741 E128 E124 tests/test_engine.py E401 E501 E128 tests/test_exporters.py E501 E731 E128 E124 tests/test_extension_telnet.py F841 @@ -212,7 +212,7 @@ flake8-ignore = tests/test_mail.py E128 E501 tests/test_middleware.py E501 E128 tests/test_pipeline_crawl.py E131 E501 E128 E126 - tests/test_pipeline_files.py E501 E272 E226 + tests/test_pipeline_files.py E501 E272 tests/test_pipeline_images.py F841 E501 tests/test_pipeline_media.py E501 E741 E731 E128 E502 tests/test_proxy_connect.py E501 E741 @@ -222,29 +222,29 @@ flake8-ignore = tests/test_scheduler.py E501 E126 E123 tests/test_selector.py E501 E127 tests/test_spider.py E501 - tests/test_spidermiddleware.py E501 E226 + tests/test_spidermiddleware.py E501 tests/test_spidermiddleware_httperror.py E128 E501 E127 E121 tests/test_spidermiddleware_offsite.py E501 E128 E111 - tests/test_spidermiddleware_output_chain.py E501 E226 + tests/test_spidermiddleware_output_chain.py E501 tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121 tests/test_squeues.py E501 E701 E741 tests/test_utils_asyncio.py E501 tests/test_utils_conf.py E501 E128 tests/test_utils_curl.py E501 tests/test_utils_datatypes.py E402 E501 - tests/test_utils_defer.py E501 F841 E226 + tests/test_utils_defer.py E501 F841 tests/test_utils_deprecate.py F841 E501 tests/test_utils_http.py E501 E128 W504 tests/test_utils_iterators.py E501 E128 E129 E241 - tests/test_utils_log.py E741 E226 + tests/test_utils_log.py E741 tests/test_utils_python.py E501 E731 E701 tests/test_utils_reqser.py E501 E128 tests/test_utils_request.py E501 E128 tests/test_utils_response.py E501 - tests/test_utils_signal.py E741 F841 E731 E226 + tests/test_utils_signal.py E741 F841 E731 tests/test_utils_sitemap.py E128 E501 E124 - tests/test_utils_url.py E501 E127 E211 E125 E501 E226 E241 E126 E123 - tests/test_webclient.py E501 E128 E122 E402 E226 E241 E123 E126 + tests/test_utils_url.py E501 E127 E211 E125 E501 E241 E126 E123 + tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126 tests/test_cmdline/__init__.py E501 tests/test_settings/__init__.py E501 E128 tests/test_spiderloader/__init__.py E128 E501 diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ff6f1d8cd..3ef8ddcb3 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -80,7 +80,7 @@ class Command(ScrapyCommand): else: items = self.items.get(lvl, []) - print("# Scraped Items ", "-"*60) + print("# Scraped Items ", "-" * 60) display.pprint([dict(x) for x in items], colorize=colour) def print_requests(self, lvl=None, colour=True): @@ -92,14 +92,14 @@ class Command(ScrapyCommand): else: requests = self.requests.get(lvl, []) - print("# Requests ", "-"*65) + print("# Requests ", "-" * 65) display.pprint(requests, colorize=colour) def print_results(self, opts): colour = not opts.nocolour if opts.verbose: - for level in range(1, self.max_level+1): + for level in range(1, self.max_level + 1): print('\n>>> DEPTH LEVEL: %s <<<' % level) if not opts.noitems: self.print_items(level, colour) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index fc796e8bb..a71dc5fb3 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -140,7 +140,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): self.headers['Content-Length'] = 0 def _build_response(self, body, request): - request.meta['download_latency'] = self.headers_time-self.start_time + request.meta['download_latency'] = self.headers_time - self.start_time status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index dd9b3c376..87d08cab7 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -82,7 +82,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - return process_spider_output(result, method_index+1) + return process_spider_output(result, method_index + 1) elif result is None: continue else: @@ -103,12 +103,12 @@ class SpiderMiddlewareManager(MiddlewareManager): # might fail directly if the output value is not a generator result = method(response=response, result=result, spider=spider) except Exception as ex: - exception_result = process_spider_exception(Failure(ex), method_index+1) + exception_result = process_spider_exception(Failure(ex), method_index + 1) if isinstance(exception_result, Failure): raise return exception_result if _isiterable(result): - result = _evaluate_iterable(result, method_index+1, recovered) + result = _evaluate_iterable(result, method_index + 1, recovered) else: msg = "Middleware {} must return an iterable, got {}" raise _InvalidOutput(msg.format(_fname(method), type(result))) diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 7a140fcad..16b046e99 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -47,7 +47,7 @@ class AjaxCrawlMiddleware(object): return response # scrapy already handles #! links properly - ajax_crawl_request = request.replace(url=request.url+'#!') + ajax_crawl_request = request.replace(url=request.url + '#!') logger.debug("Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s", {'ajax_crawl_request': ajax_crawl_request, 'request': request}, extra={'spider': spider}) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 1a3c9345f..96416f075 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -173,12 +173,12 @@ class XmlItemExporter(BaseItemExporter): if hasattr(serialized_value, 'items'): self._beautify_newline() for subname, value in serialized_value.items(): - self._export_xml_field(subname, value, depth=depth+1) + self._export_xml_field(subname, value, depth=depth + 1) self._beautify_indent(depth=depth) elif is_listlike(serialized_value): self._beautify_newline() for value in serialized_value: - self._export_xml_field('value', value, depth=depth+1) + self._export_xml_field('value', value, depth=depth + 1) self._beautify_indent(depth=depth) elif isinstance(serialized_value, str): self.xg.characters(serialized_value) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index f5ef56ea4..ab82e1915 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -22,7 +22,7 @@ _collect_string_content = etree.XPath("string()") def _nons(tag): if isinstance(tag, str): - if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE: + if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE: return tag.split('}')[-1] return tag diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index fc7b62e78..f8a0457ce 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -75,8 +75,8 @@ DOWNLOAD_HANDLERS_BASE = { DOWNLOAD_TIMEOUT = 180 # 3mins -DOWNLOAD_MAXSIZE = 1024*1024*1024 # 1024m -DOWNLOAD_WARNSIZE = 32*1024*1024 # 32m +DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m +DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m DOWNLOAD_FAIL_ON_DATALOSS = True diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 3beca4060..048e84e4f 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -28,7 +28,7 @@ class SpiderLoader(object): module=mod, cls=cls, name=name) for (mod, cls) in locations) for name, locations in self._found.items() - if len(locations)>1] + if len(locations) > 1] if dupes: msg = ("There are several spiders with the same name:\n\n" "{}\n\n This can cause unexpected behavior.".format( diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index a52bbc70e..b07f995cf 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -175,12 +175,12 @@ class SiteNode(object): node.parent = self def to_string(self, level=0): - s = "%s%s\n" % (' '*level, self.url) + s = "%s%s\n" % (' ' * level, self.url) if self.itemnames: for n in self.itemnames: - s += "%sScraped: %s\n" % (' '*(level+1), n) + s += "%sScraped: %s\n" % (' ' * (level + 1), n) for node in self.children: - s += node.to_string(level+1) + s += node.to_string(level + 1) return s diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py index bab262393..ceb3f0509 100644 --- a/scrapy/utils/http.py +++ b/scrapy/utils/http.py @@ -32,5 +32,5 @@ def decode_chunked_transfer(chunked_body): break size = int(h, 16) body += t[:size] - t = t[size+2:] + t = t[size + 2:] return body diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index a3e55d6ea..52cfba208 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -46,7 +46,7 @@ def load_object(path): except ValueError: raise ValueError("Error loading object '%s': not a full path" % path) - module, name = path[:dot], path[dot+1:] + module, name = path[:dot], path[dot + 1:] mod = import_module(module) try: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 80f52a4ef..6513e06c9 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -16,7 +16,7 @@ def listen_tcp(portrange, host, factory): return reactor.listenTCP(portrange, factory, interface=host) if len(portrange) == 1: return reactor.listenTCP(portrange[0], factory, interface=host) - for x in range(portrange[0], portrange[1]+1): + for x in range(portrange[0], portrange[1] + 1): try: return reactor.listenTCP(x, factory, interface=host) except error.CannotListenError: diff --git a/tests/pipelines.py b/tests/pipelines.py index d7d3b5259..de4894c32 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -6,7 +6,7 @@ Some pipelines used for testing class ZeroDivisionErrorPipeline(object): def open_spider(self, spider): - a = 1/0 + a = 1 / 0 def process_item(self, item, spider): return item @@ -15,4 +15,4 @@ class ZeroDivisionErrorPipeline(object): class ProcessWithZeroDivisionErrorPipiline(object): def process_item(self, item, spider): - 1/0 + 1 / 0 diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 8a54d2c74..5bf92b71a 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -182,7 +182,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} def test_crawlspider_matching_rule_callback_set(self): """If a rule matches the URL, use it's defined callback.""" status, out, stderr = yield self.execute( - ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')] + ['--spider', 'goodcrawl' + self.spider_name, '-r', self.url('/html')] ) self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @@ -190,7 +190,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} def test_crawlspider_matching_rule_default_callback(self): """If a rule match but it has no callback set, use the 'parse' callback.""" status, out, stderr = yield self.execute( - ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')] + ['--spider', 'goodcrawl' + self.spider_name, '-r', self.url('/text')] ) self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out)) @@ -206,7 +206,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} @defer.inlineCallbacks def test_crawlspider_missing_callback(self): status, out, stderr = yield self.execute( - ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')] + ['--spider', 'badcrawl' + self.spider_name, '-r', self.url('/html')] ) self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") @@ -214,7 +214,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} def test_crawlspider_no_matching_rule(self): """The requested URL has no matching rule, so no items should be scraped""" status, out, stderr = yield self.execute( - ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')] + ['--spider', 'badcrawl' + self.spider_name, '-r', self.url('/enc-gb18030')] ) self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 8d95d7cac..29d06bab4 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -348,7 +348,7 @@ class HttpTestCase(unittest.TestCase): return self.download_request(request, Spider('foo')).addCallback(_test) def test_payload(self): - body = b'1'*100 # PayloadResource requires body length to be 100 + body = b'1' * 100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) @@ -812,7 +812,7 @@ class S3TestCase(unittest.TestCase): def test_request_signing1(self): # gets an object from the johnsmith bucket. - date ='Tue, 27 Mar 2007 19:36:42 +0000' + date = 'Tue, 27 Mar 2007 19:36:42 +0000' req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date}) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 88ce9627f..9e24d86dd 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -43,7 +43,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_crawler_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -51,14 +51,14 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_settings_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'} + 'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, 'from_settings') def test_df_direct_scheduler(self): - settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'} + settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertEqual(scheduler.df.method, 'n/a') @@ -162,7 +162,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log(self): with LogCapture() as l: settings = {'DUPEFILTER_DEBUG': False, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(SimpleSpider, settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) spider = SimpleSpider.from_crawler(crawler) @@ -187,7 +187,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log_debug(self): with LogCapture() as l: settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(SimpleSpider, settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) spider = SimpleSpider.from_crawler(crawler) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 88ce1cf18..799782647 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -359,7 +359,7 @@ class TestGCSFilesStore(unittest.TestCase): self.assertIn('checksum', s) self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==') u = urlparse(uri) - content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) + content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:] + path) self.assertEqual(content, data) self.assertEqual(blob.metadata, {'foo': 'bar'}) self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 55d665e79..78e926adc 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -94,7 +94,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): class RaiseExceptionProcessSpiderOutputMiddleware: def process_spider_output(self, response, result, spider): - 1/0 + 1 / 0 self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index b19a74609..b26353d6c 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -125,7 +125,7 @@ class NotGeneratorCallbackSpider(Spider): yield Request(self.mockserver.url('/status?n=200')) def parse(self, response): - return [{'test': 1}, {'test': 1/0}] + return [{'test': 1}, {'test': 1 / 0}] # ================================================================================ diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 89b5fb4fb..a3b6e64f1 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -104,7 +104,7 @@ class IterErrbackTest(unittest.TestCase): def iterbad(): for x in range(10): if x == 5: - a = 1/0 + a = 1 / 0 yield x errors = [] diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 2c23f3616..21100aeb8 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -16,7 +16,7 @@ class FailureToExcInfoTest(unittest.TestCase): def test_failure(self): try: - 0/0 + 0 / 0 except ZeroDivisionError: exc_info = sys.exc_info() failure = Failure() diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index e5f6f0ed4..9f6da09ed 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -44,7 +44,7 @@ class SendCatchLogTest(unittest.TestCase): def error_handler(self, arg, handlers_called): handlers_called.add(self.error_handler) - a = 1/0 + a = 1 / 0 def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 9f1acbc75..1e18494c3 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -30,7 +30,7 @@ class UrlUtilsTest(unittest.TestCase): url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29' self.assertFalse(url_is_from_any_domain(url, ['testdomain.com'])) - self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com'])) + self.assertFalse(url_is_from_any_domain(url + '.testdomain.com', ['testdomain.com'])) def test_url_is_from_spider(self): spider = Spider(name='example.com') diff --git a/tests/test_webclient.py b/tests/test_webclient.py index b602a3ea0..99a998a46 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -51,22 +51,22 @@ class ParseUrlTestCase(unittest.TestCase): ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')), - ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip+':100', lip, 100, '/?c=v&c2=v2')), - ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip+':100', lip, 100, '/?c=v&c2=v2')), - ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip+':100', lip, 100, '/foo?c=v&c2=v2')), + ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), + ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), + ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')), ("http://127.0.0.1", ('http', lip, lip, 80, '/')), ("http://127.0.0.1/", ('http', lip, lip, 80, '/')), ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')), ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')), ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')), - ("http://127.0.0.1:12345/foo", ('http', lip+':12345', lip, 12345, '/foo')), + ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')), ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')), ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')), ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')), ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')), - ("https://127.0.0.1:12345/", ('https', lip+':12345', lip, 12345, '/')), + ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')), ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')), ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')), From 03ed9e17867b8c7533d08ef28108a67305050e9a Mon Sep 17 00:00:00 2001 From: Marc Hernandez Cabot Date: Fri, 21 Feb 2020 09:29:29 +0100 Subject: [PATCH 084/281] delete old deprecated functions from scrapy.utils.python --- scrapy/utils/python.py | 44 ------------------------------------------ 1 file changed, 44 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index e5582cc18..e95a4648e 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -4,7 +4,6 @@ This module contains essential stuff that should've come with Python itself ;) import errno import gc import inspect -import os import re import sys import weakref @@ -165,14 +164,6 @@ _BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\ _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS} -@deprecated("scrapy.utils.python.binary_is_text") -def isbinarytext(text): - """ This function is deprecated. - Please use scrapy.utils.python.binary_is_text, which was created to be more - clear about the functions behavior: it is behaving inverted to this one. """ - return not binary_is_text(text) - - def binary_is_text(data): """ Returns ``True`` if the given ``data`` argument (a ``bytes`` object) does not contain unprintable control characters. @@ -293,41 +284,6 @@ class WeakKeyCache(object): return self._weakdict[key] -@deprecated -def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): - """Return a (new) dict with unicode keys (and values when "keys_only" is - False) of the given dict converted to strings. ``dct_or_tuples`` can be a - dict or a list of tuples, like any dict ``__init__`` method supports. - """ - d = {} - for k, v in dict(dct_or_tuples).items(): - k = k.encode(encoding) if isinstance(k, str) else k - if not keys_only: - v = v.encode(encoding) if isinstance(v, str) else v - d[k] = v - return d - - -@deprecated -def is_writable(path): - """Return True if the given path can be written (if it exists) or created - (if it doesn't exist) - """ - if os.path.exists(path): - return os.access(path, os.W_OK) - else: - return os.access(os.path.dirname(path), os.W_OK) - - -@deprecated -def setattr_default(obj, name, value): - """Set attribute value, but only if it's not already set. Similar to - setdefault() for dicts. - """ - if not hasattr(obj, name): - setattr(obj, name, value) - - def retry_on_eintr(function, *args, **kw): """Run a function and retry it while getting EINTR errors""" while True: From b49ece0b8781c1d53cdec77b96445826a089afc1 Mon Sep 17 00:00:00 2001 From: Marc Hernandez Cabot Date: Fri, 21 Feb 2020 08:58:32 +0100 Subject: [PATCH 085/281] fix E701 and E271 flake8 --- pytest.ini | 14 +++++++------- scrapy/utils/iterators.py | 6 ++++-- tests/test_item.py | 21 ++++++++++++++------- tests/test_pipeline_files.py | 2 +- tests/test_squeues.py | 4 +++- tests/test_utils_python.py | 4 +++- 6 files changed, 32 insertions(+), 19 deletions(-) diff --git a/pytest.ini b/pytest.ini index 7806620d5..acdb5a27a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -128,7 +128,7 @@ flake8-ignore = scrapy/utils/gz.py E501 W504 scrapy/utils/http.py F403 scrapy/utils/httpobj.py E501 - scrapy/utils/iterators.py E501 E701 + scrapy/utils/iterators.py E501 scrapy/utils/log.py E128 E501 scrapy/utils/markup.py F403 scrapy/utils/misc.py E501 @@ -141,7 +141,7 @@ flake8-ignore = scrapy/utils/response.py E501 E128 scrapy/utils/signal.py E501 E128 scrapy/utils/sitemap.py E501 - scrapy/utils/spider.py E271 E501 + scrapy/utils/spider.py E501 scrapy/utils/ssl.py E501 scrapy/utils/test.py E501 scrapy/utils/url.py E501 F403 E128 F405 @@ -181,7 +181,7 @@ flake8-ignore = tests/test_crawl.py E501 E741 E265 tests/test_crawler.py F841 E501 tests/test_dependencies.py F841 E501 - tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E701 E126 E123 + tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126 @@ -204,7 +204,7 @@ flake8-ignore = tests/test_http_headers.py E501 tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123 tests/test_http_response.py E501 E128 E265 - tests/test_item.py E701 E128 F841 + tests/test_item.py E128 F841 tests/test_link.py E501 tests/test_linkextractors.py E501 E128 E124 tests/test_loader.py E501 E731 E741 E128 E117 E241 @@ -212,7 +212,7 @@ flake8-ignore = tests/test_mail.py E128 E501 tests/test_middleware.py E501 E128 tests/test_pipeline_crawl.py E131 E501 E128 E126 - tests/test_pipeline_files.py E501 E272 + tests/test_pipeline_files.py E501 tests/test_pipeline_images.py F841 E501 tests/test_pipeline_media.py E501 E741 E731 E128 E502 tests/test_proxy_connect.py E501 E741 @@ -227,7 +227,7 @@ flake8-ignore = tests/test_spidermiddleware_offsite.py E501 E128 E111 tests/test_spidermiddleware_output_chain.py E501 tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121 - tests/test_squeues.py E501 E701 E741 + tests/test_squeues.py E501 E741 tests/test_utils_asyncio.py E501 tests/test_utils_conf.py E501 E128 tests/test_utils_curl.py E501 @@ -237,7 +237,7 @@ flake8-ignore = tests/test_utils_http.py E501 E128 W504 tests/test_utils_iterators.py E501 E128 E129 E241 tests/test_utils_log.py E741 - tests/test_utils_python.py E501 E731 E701 + tests/test_utils_python.py E501 E731 tests/test_utils_reqser.py E501 E128 tests/test_utils_request.py E501 E128 tests/test_utils_response.py E501 diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 3c0cb68c3..7849174fb 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -101,8 +101,10 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): lines = StringIO(_body_or_str(obj, unicode=True)) kwargs = {} - if delimiter: kwargs["delimiter"] = delimiter - if quotechar: kwargs["quotechar"] = quotechar + if delimiter: + kwargs["delimiter"] = delimiter + if quotechar: + kwargs["quotechar"] = quotechar csv_r = csv.reader(lines, **kwargs) if not headers: diff --git a/tests/test_item.py b/tests/test_item.py index 823bf1ced..f70632d57 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -149,13 +149,15 @@ class ItemTest(unittest.TestCase): fields = {'load': Field(default='A')} save = Field(default='A') - class B(A): pass + class B(A): + pass class C(Item): fields = {'load': Field(default='C')} save = Field(default='C') - class D(B, C): pass + class D(B, C): + pass item = D(save='X', load='Y') self.assertEqual(item['save'], 'X') @@ -164,7 +166,8 @@ class ItemTest(unittest.TestCase): 'save': {'default': 'A'}}) # D class inverted - class E(C, B): pass + class E(C, B): + pass self.assertEqual(E(save='X')['save'], 'X') self.assertEqual(E(load='X')['load'], 'X') @@ -177,7 +180,8 @@ class ItemTest(unittest.TestCase): save = Field(default='A') load = Field(default='A') - class B(A): pass + class B(A): + pass class C(A): fields = {'update': Field(default='C')} @@ -206,14 +210,16 @@ class ItemTest(unittest.TestCase): fields = {'load': Field(default='A')} save = Field(default='A') - class B(A): pass + class B(A): + pass class C(object): fields = {'load': Field(default='C')} not_allowed = Field(default='not_allowed') save = Field(default='C') - class D(B, C): pass + class D(B, C): + pass self.assertRaises(KeyError, D, not_allowed='value') self.assertEqual(D(save='X')['save'], 'X') @@ -221,7 +227,8 @@ class ItemTest(unittest.TestCase): 'load': {'default': 'A'}}) # D class inverted - class E(C, B): pass + class E(C, B): + pass self.assertRaises(KeyError, E, not_allowed='value') self.assertEqual(E(save='X')['save'], 'X') diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 799782647..f155db4ce 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -272,7 +272,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): prefix = pipeline_cls.__name__.upper() settings = self._generate_fake_settings(prefix=prefix) user_pipeline = pipeline_cls.from_settings(Settings(settings)) - for pipe_cls_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: + for pipe_cls_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: custom_value = settings.get(prefix + "_" + settings_attr) self.assertNotEqual(custom_value, self.default_cls_settings[pipe_cls_attr]) self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index d5fcf2f7f..f6970162e 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -31,7 +31,9 @@ def nonserializable_object_test(self): self.assertRaises(ValueError, q.push, lambda x: x) else: # Use a different unpickleable object - class A(object): pass + class A(object): + pass + a = A() a.__reduce__ = a.__reduce_ex__ = None self.assertRaises(ValueError, q.push, a) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 4202e8c89..ec5b4c596 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -153,7 +153,9 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) def test_weakkeycache(self): - class _Weakme(object): pass + class _Weakme(object): + pass + _values = count() wk = WeakKeyCache(lambda k: next(_values)) k = _Weakme() From 9ad10bb6f727a3f1c5c59d490f444ebb32de97c6 Mon Sep 17 00:00:00 2001 From: Marc Hernandez Cabot Date: Fri, 21 Feb 2020 09:05:42 +0100 Subject: [PATCH 086/281] fix E131 --- pytest.ini | 2 +- tests/test_pipeline_crawl.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pytest.ini b/pytest.ini index acdb5a27a..2120264e0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -211,7 +211,7 @@ flake8-ignore = tests/test_logformatter.py E128 E501 E122 tests/test_mail.py E128 E501 tests/test_middleware.py E501 E128 - tests/test_pipeline_crawl.py E131 E501 E128 E126 + tests/test_pipeline_crawl.py E501 E128 E126 tests/test_pipeline_files.py E501 tests/test_pipeline_images.py F841 E501 tests/test_pipeline_media.py E501 E741 E731 E128 E502 diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index fb72c9d6d..962c33144 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -26,10 +26,9 @@ class MediaDownloadSpider(SimpleSpider): self.media_key: [], self.media_urls_key: [ self._process_url(response.urljoin(href)) - for href in response.xpath(''' - //table[thead/tr/th="Filename"] - /tbody//a/@href - ''').getall()], + for href in response.xpath( + '//table[thead/tr/th="Filename"]/tbody//a/@href' + ).getall()], } yield item @@ -99,8 +98,9 @@ class FileDownloadCrawlTestCase(TestCase): if self.expected_checksums is not None: checksums = set( i['checksum'] - for item in items - for i in item[self.media_key]) + for item in items + for i in item[self.media_key] + ) self.assertEqual(checksums, self.expected_checksums) # check that the image files where actually written to the media store From 69a8648bef6df38a5b7e79f9fbecb98869416654 Mon Sep 17 00:00:00 2001 From: Marc Hernandez Cabot Date: Fri, 21 Feb 2020 09:13:28 +0100 Subject: [PATCH 087/281] fix E251 --- pytest.ini | 4 ++-- tests/test_downloadermiddleware_httpcompression.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pytest.ini b/pytest.ini index 2120264e0..58f1cfeb3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -189,10 +189,10 @@ flake8-ignore = tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 tests/test_downloadermiddleware_httpcache.py E501 - tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123 + tests/test_downloadermiddleware_httpcompression.py E501 E126 E123 tests/test_downloadermiddleware_httpproxy.py E501 E128 tests/test_downloadermiddleware_redirect.py E501 E128 E127 - tests/test_downloadermiddleware_retry.py E501 E128 E251 E126 + tests/test_downloadermiddleware_retry.py E501 E128 E126 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 tests/test_dupefilters.py E501 E741 E128 E124 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 64488841a..106ca3360 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -245,7 +245,7 @@ class HttpCompressionTest(TestCase): response.headers['Content-Type'] = 'application/gzip' request = response.request request.method = 'HEAD' - response = response.replace(body = None) + response = response.replace(body=None) newresponse = self.mw.process_response(request, response, self.spider) self.assertIs(newresponse, response) self.assertEqual(response.body, b'') From 6e8e117aee4ddc5d6f6970019be212198d0b9e7a Mon Sep 17 00:00:00 2001 From: Marc Hernandez Cabot Date: Fri, 21 Feb 2020 09:14:55 +0100 Subject: [PATCH 088/281] fix flake E211 --- pytest.ini | 2 +- tests/test_utils_url.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pytest.ini b/pytest.ini index 58f1cfeb3..141a13a4f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -243,7 +243,7 @@ flake8-ignore = tests/test_utils_response.py E501 tests/test_utils_signal.py E741 F841 E731 tests/test_utils_sitemap.py E128 E501 E124 - tests/test_utils_url.py E501 E127 E211 E125 E501 E241 E126 E123 + tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123 tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126 tests/test_cmdline/__init__.py E501 tests/test_settings/__init__.py E501 E128 diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 1e18494c3..7abff8281 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -202,7 +202,7 @@ def create_skipped_scheme_t(args): return do_expected -for k, args in enumerate ([ +for k, args in enumerate([ ('/index', 'file://'), ('/index.html', 'file://'), ('./index.html', 'file://'), @@ -230,7 +230,7 @@ for k, args in enumerate ([ ], start=1): t_method = create_guess_scheme_t(args) t_method.__name__ = 'test_uri_%03d' % k - setattr (GuessSchemeTest, t_method.__name__, t_method) + setattr(GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation for k, args in enumerate([ @@ -239,7 +239,7 @@ for k, args in enumerate([ ], start=1): t_method = create_skipped_scheme_t(args) t_method.__name__ = 'test_uri_skipped_%03d' % k - setattr (GuessSchemeTest, t_method.__name__, t_method) + setattr(GuessSchemeTest, t_method.__name__, t_method) class StripUrl(unittest.TestCase): From 67ee0b097fe15aefa787bce64f6fa085d38e69d8 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Sat, 22 Feb 2020 17:02:57 +0500 Subject: [PATCH 089/281] Remove specifics of downstream request queues from scheduler (#3884) * move serialization/deserialization logic to downstream queues * make memory queues conform to common interface * make ScrapyPriorityQueue conform common interface * ScrapyPriorityQueue works with disk * make key as string * return list instead of dict as earlier * downloader aware pq works with new interface * we don`t need these methods anymore * create directories for files * remove dummy priority * remove priority as parameter, let every queue decide for itself * rename obj to request * DownloaderAwarePriorityQueue is too thin wrapper around _SlotPriorityQueues, just remove second one * remove priority as parameter, let every queue decide for itself * rename argument * more granular class separation * python2 compatible * one more argument for common interface * more simple downstream queue interface * single place for easier customization * rename function * shorter * shorter * use named arguments * fix typo * add docstring * Update scrapy/pqueues.py Co-Authored-By: Mikhail Korobov * Update scrapy/pqueues.py Co-Authored-By: Mikhail Korobov * 4 spaces indentation * we ok with existing directories * remove unused import * rename method * remove unused imports * it has no sense now * relining * note about queues * add value * Revert "it has no sense now" This reverts commit b61604275ba090ebd8e30a6d3a6fbe281c74c189. * pep8 E261 * pep8 E303 * pep8 E501 * pep8 E123 * pep8 E123 * use create instance * remove excessive import Co-authored-by: Mikhail Korobov --- scrapy/core/scheduler.py | 30 +++--- scrapy/pqueues.py | 201 +++++++++++++++++++++------------------ scrapy/squeues.py | 100 +++++++++++++++++-- tests/test_squeues.py | 7 +- 4 files changed, 214 insertions(+), 124 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 975aede0c..e184ed50e 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -119,7 +119,7 @@ class Scheduler(object): if self.dqs is None: return try: - self.dqs.push(request, -request.priority) + self.dqs.push(request) except ValueError as e: # non serializable request if self.logunser: msg = ("Unable to serialize request: %(request)s - reason:" @@ -135,35 +135,29 @@ class Scheduler(object): return True def _mqpush(self, request): - self.mqs.push(request, -request.priority) + self.mqs.push(request) def _dqpop(self): if self.dqs: return self.dqs.pop() - def _newmq(self, priority): - """ Factory for creating memory queues. """ - return self.mqclass() - - def _newdq(self, priority): - """ Factory for creating disk queues. """ - path = join(self.dqdir, 'p%s' % (priority, )) - return self.dqclass(path) - def _mq(self): """ Create a new priority queue instance, with in-memory storage """ - return create_instance(self.pqclass, None, self.crawler, self._newmq, - serialize=False) + return create_instance(self.pqclass, + settings=None, + crawler=self.crawler, + downstream_queue_cls=self.mqclass, + key='') def _dq(self): """ Create a new priority queue instance, with disk storage """ state = self._read_dqs_state(self.dqdir) q = create_instance(self.pqclass, - None, - self.crawler, - self._newdq, - state, - serialize=True) + settings=None, + crawler=self.crawler, + downstream_queue_cls=self.dqclass, + key=self.dqdir, + startprios=state) if q: logger.info("Resuming crawl (%(queuesize)d requests scheduled)", {'queuesize': len(q)}, extra={'spider': self.spider}) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 717ed4d27..1afe58dab 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,11 +1,7 @@ import hashlib import logging -from collections import namedtuple - -from queuelib import PriorityQueue - -from scrapy.utils.reqser import request_to_dict, request_from_dict +from scrapy.utils.misc import create_instance logger = logging.getLogger(__name__) @@ -29,88 +25,89 @@ def _path_safe(text): return '-'.join([pathable_slot, unique_slot]) -class _Priority(namedtuple("_Priority", ["priority", "slot"])): - """ Slot-specific priority. It is a hack - ``(priority, slot)`` tuple - which can be used instead of int priorities in queues: +class ScrapyPriorityQueue: + """A priority queue implemented using multiple internal queues (typically, + FIFO queues). It uses one internal queue for each priority value. The internal + queue must implement the following methods: + + * push(obj) + * pop() + * close() + * __len__() + + ``__init__`` method of ScrapyPriorityQueue receives a downstream_queue_cls + argument, which is a class used to instantiate a new (internal) queue when + a new priority is allocated. + + Only integer priorities should be used. Lower numbers are higher + priorities. + + startprios is a sequence of priorities to start with. If the queue was + previously closed leaving some priority buckets non-empty, those priorities + should be passed in startprios. - * they are ordered in the same way - order is still by priority value, - min(prios) works; - * str(p) representation is guaranteed to be different when slots - are different - this is important because str(p) is used to create - queue files on disk; - * they have readable str(p) representation which is safe - to use as a file name. """ - __slots__ = () - def __str__(self): - return '%s_%s' % (self.priority, _path_safe(str(self.slot))) + @classmethod + def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()): + return cls(crawler, downstream_queue_cls, key, startprios) + def __init__(self, crawler, downstream_queue_cls, key, startprios=()): + self.crawler = crawler + self.downstream_queue_cls = downstream_queue_cls + self.key = key + self.queues = {} + self.curprio = None + self.init_prios(startprios) -class _SlotPriorityQueues(object): - """ Container for multiple priority queues. """ - def __init__(self, pqfactory, slot_startprios=None): - """ - ``pqfactory`` is a factory for creating new PriorityQueues. - It must be a function which accepts a single optional ``startprios`` - argument, with a list of priorities to create queues for. + def init_prios(self, startprios): + if not startprios: + return - ``slot_startprios`` is a ``{slot: startprios}`` dict. - """ - self.pqfactory = pqfactory - self.pqueues = {} # slot -> priority queue - for slot, startprios in (slot_startprios or {}).items(): - self.pqueues[slot] = self.pqfactory(startprios) + for priority in startprios: + self.queues[priority] = self.qfactory(priority) - def pop_slot(self, slot): - """ Pop an object from a priority queue for this slot """ - queue = self.pqueues[slot] - request = queue.pop() - if len(queue) == 0: - del self.pqueues[slot] - return request + self.curprio = min(startprios) - def push_slot(self, slot, obj, priority): - """ Push an object to a priority queue for this slot """ - if slot not in self.pqueues: - self.pqueues[slot] = self.pqfactory() - queue = self.pqueues[slot] - queue.push(obj, priority) + def qfactory(self, key): + return create_instance(self.downstream_queue_cls, + None, + self.crawler, + self.key + '/' + str(key)) + + def priority(self, request): + return -request.priority + + def push(self, request): + priority = self.priority(request) + if priority not in self.queues: + self.queues[priority] = self.qfactory(priority) + q = self.queues[priority] + q.push(request) # this may fail (eg. serialization error) + if self.curprio is None or priority < self.curprio: + self.curprio = priority + + def pop(self): + if self.curprio is None: + return + q = self.queues[self.curprio] + m = q.pop() + if not q: + del self.queues[self.curprio] + q.close() + prios = [p for p, q in self.queues.items() if q] + self.curprio = min(prios) if prios else None + return m def close(self): - active = {slot: queue.close() - for slot, queue in self.pqueues.items()} - self.pqueues.clear() + active = [] + for p, q in self.queues.items(): + active.append(p) + q.close() return active def __len__(self): - return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 - - -class ScrapyPriorityQueue(PriorityQueue): - """ - PriorityQueue which works with scrapy.Request instances and - can optionally convert them to/from dicts before/after putting to a queue. - """ - def __init__(self, crawler, qfactory, startprios=(), serialize=False): - super(ScrapyPriorityQueue, self).__init__(qfactory, startprios) - self.serialize = serialize - self.spider = crawler.spider - - @classmethod - def from_crawler(cls, crawler, qfactory, startprios=(), serialize=False): - return cls(crawler, qfactory, startprios, serialize) - - def push(self, request, priority=0): - if self.serialize: - request = request_to_dict(request, self.spider) - super(ScrapyPriorityQueue, self).push(request, priority) - - def pop(self): - request = super(ScrapyPriorityQueue, self).pop() - if request and self.serialize: - request = request_from_dict(request, self.spider) - return request + return sum(len(x) for x in self.queues.values()) if self.queues else 0 class DownloaderInterface(object): @@ -133,16 +130,16 @@ class DownloaderInterface(object): class DownloaderAwarePriorityQueue(object): - """ PriorityQueue which takes Downlaoder activity in account: + """ PriorityQueue which takes Downloader activity in account: domains (slots) with the least amount of active downloads are dequeued first. """ @classmethod - def from_crawler(cls, crawler, qfactory, slot_startprios=None, serialize=False): - return cls(crawler, qfactory, slot_startprios, serialize) + def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()): + return cls(crawler, downstream_queue_cls, key, startprios) - def __init__(self, crawler, qfactory, slot_startprios=None, serialize=False): + def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()): if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0: raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP' % (self.__class__,)) @@ -156,35 +153,49 @@ class DownloaderAwarePriorityQueue(object): "queue class can be resumed." % slot_startprios.__class__) - slot_startprios = { - slot: [_Priority(p, slot) for p in startprios] - for slot, startprios in (slot_startprios or {}).items()} - - def pqfactory(startprios=()): - return ScrapyPriorityQueue(crawler, qfactory, startprios, serialize) - self._slot_pqueues = _SlotPriorityQueues(pqfactory, slot_startprios) - self.serialize = serialize self._downloader_interface = DownloaderInterface(crawler) + self.downstream_queue_cls = downstream_queue_cls + self.key = key + self.crawler = crawler + + self.pqueues = {} # slot -> priority queue + for slot, startprios in (slot_startprios or {}).items(): + self.pqueues[slot] = self.pqfactory(slot, startprios) + + def pqfactory(self, slot, startprios=()): + return ScrapyPriorityQueue(self.crawler, + self.downstream_queue_cls, + self.key + '/' + _path_safe(slot), + startprios) def pop(self): - stats = self._downloader_interface.stats(self._slot_pqueues.pqueues) + stats = self._downloader_interface.stats(self.pqueues) if not stats: return slot = min(stats)[1] - request = self._slot_pqueues.pop_slot(slot) + queue = self.pqueues[slot] + request = queue.pop() + if len(queue) == 0: + del self.pqueues[slot] return request - def push(self, request, priority): + def push(self, request): slot = self._downloader_interface.get_slot_key(request) - priority_slot = _Priority(priority=priority, slot=slot) - self._slot_pqueues.push_slot(slot, request, priority_slot) + if slot not in self.pqueues: + self.pqueues[slot] = self.pqfactory(slot) + queue = self.pqueues[slot] + queue.push(request) def close(self): - active = self._slot_pqueues.close() - return {slot: [p.priority for p in startprios] - for slot, startprios in active.items()} + active = {slot: queue.close() + for slot, queue in self.pqueues.items()} + self.pqueues.clear() + return active def __len__(self): - return len(self._slot_pqueues) + return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 + + def __contains__(self, slot): + return slot in self.pqueues diff --git a/scrapy/squeues.py b/scrapy/squeues.py index d5d3be67e..d0686dac3 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -3,10 +3,27 @@ Scheduler queues """ import marshal +import os import pickle from queuelib import queue +from scrapy.utils.reqser import request_to_dict, request_from_dict + + +def _with_mkdir(queue_class): + + class DirectoriesCreated(queue_class): + + def __init__(self, path, *args, **kwargs): + dirname = os.path.dirname(path) + if not os.path.exists(dirname): + os.makedirs(dirname, exist_ok=True) + + super(DirectoriesCreated, self).__init__(path, *args, **kwargs) + + return DirectoriesCreated + def _serializable_queue(queue_class, serialize, deserialize): @@ -24,6 +41,44 @@ def _serializable_queue(queue_class, serialize, deserialize): return SerializableQueue +def _scrapy_serialization_queue(queue_class): + + class ScrapyRequestQueue(queue_class): + + def __init__(self, crawler, key): + self.spider = crawler.spider + super(ScrapyRequestQueue, self).__init__(key) + + @classmethod + def from_crawler(cls, crawler, key, *args, **kwargs): + return cls(crawler, key) + + def push(self, request): + request = request_to_dict(request, self.spider) + return super(ScrapyRequestQueue, self).push(request) + + def pop(self): + request = super(ScrapyRequestQueue, self).pop() + + if not request: + return None + + request = request_from_dict(request, self.spider) + return request + + return ScrapyRequestQueue + + +def _scrapy_non_serialization_queue(queue_class): + + class ScrapyRequestQueue(queue_class): + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + return cls() + + return ScrapyRequestQueue + + def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) @@ -34,13 +89,38 @@ def _pickle_serialize(obj): raise ValueError(str(e)) -PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, - _pickle_serialize, pickle.loads) -PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, - _pickle_serialize, pickle.loads) -MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, - marshal.dumps, marshal.loads) -MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, - marshal.dumps, marshal.loads) -FifoMemoryQueue = queue.FifoMemoryQueue -LifoMemoryQueue = queue.LifoMemoryQueue +PickleFifoDiskQueueNonRequest = _serializable_queue( + _with_mkdir(queue.FifoDiskQueue), + _pickle_serialize, + pickle.loads +) +PickleLifoDiskQueueNonRequest = _serializable_queue( + _with_mkdir(queue.LifoDiskQueue), + _pickle_serialize, + pickle.loads +) +MarshalFifoDiskQueueNonRequest = _serializable_queue( + _with_mkdir(queue.FifoDiskQueue), + marshal.dumps, + marshal.loads +) +MarshalLifoDiskQueueNonRequest = _serializable_queue( + _with_mkdir(queue.LifoDiskQueue), + marshal.dumps, + marshal.loads +) + +PickleFifoDiskQueue = _scrapy_serialization_queue( + PickleFifoDiskQueueNonRequest +) +PickleLifoDiskQueue = _scrapy_serialization_queue( + PickleLifoDiskQueueNonRequest +) +MarshalFifoDiskQueue = _scrapy_serialization_queue( + MarshalFifoDiskQueueNonRequest +) +MarshalLifoDiskQueue = _scrapy_serialization_queue( + MarshalLifoDiskQueueNonRequest +) +FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) +LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index d5fcf2f7f..5c626fbcb 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,7 +1,12 @@ import pickle from queuelib.tests import test_queue as t -from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue +from scrapy.squeues import ( + MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue, + MarshalLifoDiskQueueNonRequest as MarshalLifoDiskQueue, + PickleFifoDiskQueueNonRequest as PickleFifoDiskQueue, + PickleLifoDiskQueueNonRequest as PickleLifoDiskQueue +) from scrapy.item import Item, Field from scrapy.http import Request from scrapy.loader import ItemLoader From 9d983c1b9962a018686111e20e42d25bbffb579e Mon Sep 17 00:00:00 2001 From: elacuesta Date: Sat, 22 Feb 2020 09:20:31 -0300 Subject: [PATCH 090/281] Expose certificate for HTTPS responses (#4054) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Expose certificate for HTTPS responses * Fix test (missing inlineCallbacks decorator) * Note about Response.certificate * Explicitly cover None as the default value of Response.certificate Co-authored-by: Adrián Chaves --- docs/topics/request-response.rst | 12 +++++++++- scrapy/core/downloader/handlers/http11.py | 22 +++++++++++------ scrapy/http/response/__init__.py | 5 ++-- tests/test_crawl.py | 29 +++++++++++++++++++++++ 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 672c0b3d6..f009facd6 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -609,7 +609,10 @@ Response objects :param request: the initial value of the :attr:`Response.request` attribute. This represents the :class:`Request` that generated this response. - :type request: :class:`Request` object + :type request: scrapy.http.Request + + :param certificate: an object representing the server's SSL certificate. + :type certificate: twisted.internet.ssl.Certificate .. attribute:: Response.url @@ -691,6 +694,13 @@ Response objects they're shown on the string representation of the Response (`__str__` method) which is used by the engine for logging. + .. attribute:: Response.certificate + + A :class:`twisted.internet.ssl.Certificate` object representing + the server's SSL certificate. + + Only populated for ``https`` responses, ``None`` otherwise. + .. method:: Response.copy() Returns a new Response which is a copy of this Response. diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 5a5f6cf0a..93951d3b5 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -3,11 +3,12 @@ import logging import re import warnings +from contextlib import suppress from io import BytesIO from time import time from urllib.parse import urldefrag -from twisted.internet import defer, protocol, reactor +from twisted.internet import defer, protocol, reactor, ssl from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI @@ -382,7 +383,7 @@ class ScrapyAgent(object): def _cb_bodyready(self, txresponse, request): # deliverBody hangs for responses without body if txresponse.length == 0: - return txresponse, b'', None + return txresponse, b'', None, None maxsize = request.meta.get('download_maxsize', self._maxsize) warnsize = request.meta.get('download_warnsize', self._warnsize) @@ -418,11 +419,12 @@ class ScrapyAgent(object): return d def _cb_bodydone(self, result, request, url): - txresponse, body, flags = result + txresponse, body, flags, certificate = result status = int(txresponse.code) headers = Headers(txresponse.headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url, body=body) - return respcls(url=url, status=status, headers=headers, body=body, flags=flags) + return respcls(url=url, status=status, headers=headers, body=body, + flags=flags, certificate=certificate) @implementer(IBodyProducer) @@ -456,6 +458,12 @@ class _ResponseReader(protocol.Protocol): self._fail_on_dataloss_warned = False self._reached_warnsize = False self._bytes_received = 0 + self._certificate = None + + def connectionMade(self): + if self._certificate is None: + with suppress(AttributeError): + self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate()) def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. @@ -488,16 +496,16 @@ class _ResponseReader(protocol.Protocol): body = self._bodybuf.getvalue() if reason.check(ResponseDone): - self._finished.callback((self._txresponse, body, None)) + self._finished.callback((self._txresponse, body, None, self._certificate)) return if reason.check(PotentialDataLoss): - self._finished.callback((self._txresponse, body, ['partial'])) + self._finished.callback((self._txresponse, body, ['partial'], self._certificate)) return if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons): if not self._fail_on_dataloss: - self._finished.callback((self._txresponse, body, ['dataloss'])) + self._finished.callback((self._txresponse, body, ['dataloss'], self._certificate)) return elif not self._fail_on_dataloss_warned: diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f60d09608..119dd2f63 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -17,13 +17,14 @@ from scrapy.utils.trackref import object_ref class Response(object_ref): - def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None): + def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, certificate=None): self.headers = Headers(headers or {}) self.status = int(status) self._set_body(body) self._set_url(url) self.request = request self.flags = [] if flags is None else list(flags) + self.certificate = certificate @property def cb_kwargs(self): @@ -86,7 +87,7 @@ class Response(object_ref): """Create a new Response with the same attributes except for those given new values. """ - for x in ['url', 'status', 'headers', 'body', 'request', 'flags']: + for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'certificate']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 64819acb6..bbe97d034 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -5,6 +5,7 @@ import sys from pytest import mark from testfixtures import LogCapture from twisted.internet import defer +from twisted.internet.ssl import Certificate from twisted.trial.unittest import TestCase from scrapy import signals @@ -407,3 +408,31 @@ with multiples lines yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) for req_id in range(3): self.assertIn("Got response 200, req_id %d" % req_id, str(log)) + + @defer.inlineCallbacks + def test_response_ssl_certificate_none(self): + crawler = self.runner.create_crawler(SingleRequestSpider) + url = self.mockserver.url("/echo?body=test", is_secure=False) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + self.assertIsNone(crawler.spider.meta['responses'][0].certificate) + + @defer.inlineCallbacks + def test_response_ssl_certificate(self): + crawler = self.runner.create_crawler(SingleRequestSpider) + url = self.mockserver.url("/echo?body=test", is_secure=True) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + cert = crawler.spider.meta['responses'][0].certificate + self.assertIsInstance(cert, Certificate) + self.assertEqual(cert.getSubject().commonName, b"localhost") + self.assertEqual(cert.getIssuer().commonName, b"localhost") + + @mark.xfail(reason="Responses with no body return early and contain no certificate") + @defer.inlineCallbacks + def test_response_ssl_certificate_empty_response(self): + crawler = self.runner.create_crawler(SingleRequestSpider) + url = self.mockserver.url("/status?n=200", is_secure=True) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + cert = crawler.spider.meta['responses'][0].certificate + self.assertIsInstance(cert, Certificate) + self.assertEqual(cert.getSubject().commonName, b"localhost") + self.assertEqual(cert.getIssuer().commonName, b"localhost") From f85bf77da3c8943f0791dcae893e8294c4d118d7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 23 Feb 2020 18:31:13 -0300 Subject: [PATCH 091/281] Restore unrelated change --- scrapy/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f69894b1e..554a3a14d 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -29,7 +29,7 @@ class CachingThreadedResolver(ThreadedResolver): cache_size = 0 return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT')) - def install_on_reactor(self): + def install_on_reactor(self,): self.reactor.installResolver(self) def getHostByName(self, name, timeout=None): From 889b4718520220d1a81e702ff754ec210a7d3c79 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 23 Feb 2020 18:40:43 -0300 Subject: [PATCH 092/281] Import changes --- scrapy/core/downloader/handlers/http11.py | 4 ++-- tests/test_crawl.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index e72275021..190ae1d3b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,11 +1,11 @@ """Download handlers for http and https schemes""" +import ipaddress import logging import re import warnings from contextlib import suppress from io import BytesIO -from ipaddress import ip_address from time import time from urllib.parse import urldefrag @@ -468,7 +468,7 @@ class _ResponseReader(protocol.Protocol): self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate()) if self._ip_address is None: - self._ip_address = ip_address(self.transport._producer.getPeer().host) + self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host) def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3a9b00ab3..3c110e7a6 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -441,13 +441,15 @@ with multiples lines self.assertEqual(cert.getIssuer().commonName, b"localhost") @defer.inlineCallbacks - def test_dns_server_ip_address(self): + def test_dns_server_ip_address_none(self): crawler = self.runner.create_crawler(SingleRequestSpider) url = self.mockserver.url('/status?n=200') yield crawler.crawl(seed=url, mockserver=self.mockserver) ip_address = crawler.spider.meta['responses'][0].ip_address self.assertIsNone(ip_address) + @defer.inlineCallbacks + def test_dns_server_ip_address(self): crawler = self.runner.create_crawler(SingleRequestSpider) url = self.mockserver.url('/echo?body=test') expected_netloc, _ = urlparse(url).netloc.split(':') From 31f35c9c002178de20a3e124be4aab98c0a0f892 Mon Sep 17 00:00:00 2001 From: elacuesta Date: Mon, 24 Feb 2020 08:02:00 -0300 Subject: [PATCH 093/281] Remove unnecessary comma (#4369) --- scrapy/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 554a3a14d..f69894b1e 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -29,7 +29,7 @@ class CachingThreadedResolver(ThreadedResolver): cache_size = 0 return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT')) - def install_on_reactor(self,): + def install_on_reactor(self): self.reactor.installResolver(self) def getHostByName(self, name, timeout=None): From 7417a9871c489686b7d3ac1b85b964eca253c979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 24 Feb 2020 13:28:15 +0100 Subject: [PATCH 094/281] =?UTF-8?q?Make=20BaseItemExporter=E2=80=99s=20don?= =?UTF-8?q?t=5Ffail=20parameter=20keyword-only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/exporters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 96416f075..2e20a7180 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -23,7 +23,7 @@ __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', class BaseItemExporter(object): - def __init__(self, dont_fail=False, **kwargs): + def __init__(self, *, dont_fail=False, **kwargs): self._kwargs = kwargs self._configure(kwargs, dont_fail=dont_fail) From a34c366fa4f226cb107a19561ca64fac0d1dbdd5 Mon Sep 17 00:00:00 2001 From: nyov Date: Fri, 21 Feb 2020 08:15:51 +0000 Subject: [PATCH 095/281] DOC linkcheck run; https and 301 link updates. Closes #4359 --- docs/conf.py | 1 + docs/contributing.rst | 6 +++--- docs/faq.rst | 6 +++--- docs/intro/install.rst | 12 ++++++------ docs/intro/tutorial.rst | 4 ++-- docs/news.rst | 19 +++++++++---------- docs/topics/broad-crawls.rst | 2 +- docs/topics/downloader-middleware.rst | 12 ++++++------ docs/topics/dynamic-content.rst | 6 +++--- docs/topics/item-pipeline.rst | 4 ++-- docs/topics/items.rst | 4 ++-- docs/topics/leaks.rst | 10 +++++----- docs/topics/request-response.rst | 2 +- docs/topics/selectors.rst | 17 ++++++++--------- docs/topics/shell.rst | 6 +++--- docs/topics/spiders.rst | 6 +++--- 16 files changed, 58 insertions(+), 59 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index c3418cfb3..6e2399f66 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -281,6 +281,7 @@ coverage_ignore_pyobjects = [ intersphinx_mapping = { 'coverage': ('https://coverage.readthedocs.io/en/stable', None), + 'cssselect': ('https://cssselect.readthedocs.io/en/latest', None), 'pytest': ('https://docs.pytest.org/en/latest', None), 'python': ('https://docs.python.org/3', None), 'sphinx': ('https://www.sphinx-doc.org/en/master', None), diff --git a/docs/contributing.rst b/docs/contributing.rst index f40a6bba2..aed5ab92e 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -143,7 +143,7 @@ by running ``git fetch upstream pull/$PR_NUMBER/head:$BRANCH_NAME_TO_CREATE`` (replace 'upstream' with a remote name for scrapy repository, ``$PR_NUMBER`` with an ID of the pull request, and ``$BRANCH_NAME_TO_CREATE`` with a name of the branch you want to create locally). -See also: https://help.github.com/articles/checking-out-pull-requests-locally/#modifying-an-inactive-pull-request-locally. +See also: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally. When writing GitHub pull requests, try to keep titles short but descriptive. E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests" @@ -168,7 +168,7 @@ Scrapy: * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. - See https://help.github.com/articles/setting-your-username-in-git/ for + See https://help.github.com/en/github/using-git/setting-your-username-in-git for setup instructions. .. _documentation-policies: @@ -266,5 +266,5 @@ And their unit-tests are in:: .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues .. _PEP 257: https://www.python.org/dev/peps/pep-0257/ -.. _pull request: https://help.github.com/en/articles/creating-a-pull-request +.. _pull request: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist diff --git a/docs/faq.rst b/docs/faq.rst index f72e4cf01..75a0f4864 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -22,8 +22,8 @@ In other words, comparing `BeautifulSoup`_ (or `lxml`_) to Scrapy is like comparing `jinja2`_ to `Django`_. .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ -.. _lxml: http://lxml.de/ -.. _jinja2: http://jinja.pocoo.org/ +.. _lxml: https://lxml.de/ +.. _jinja2: https://palletsprojects.com/p/jinja/ .. _Django: https://www.djangoproject.com/ Can I use Scrapy with BeautifulSoup? @@ -269,7 +269,7 @@ The ``__VIEWSTATE`` parameter is used in sites built with ASP.NET/VB.NET. For more info on how it works see `this page`_. Also, here's an `example spider`_ which scrapes one of these sites. -.. _this page: http://search.cpan.org/~ecarroll/HTML-TreeBuilderX-ASP_NET-0.09/lib/HTML/TreeBuilderX/ASP_NET.pm +.. _this page: https://metacpan.org/pod/release/ECARROLL/HTML-TreeBuilderX-ASP_NET-0.09/lib/HTML/TreeBuilderX/ASP_NET.pm .. _example spider: https://github.com/AmbientLighter/rpn-fas/blob/master/fas/spiders/rnp.py What's the best way to parse big XML/CSV data feeds? diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 49968437c..871281460 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -65,7 +65,7 @@ please refer to their respective installation instructions: * `lxml installation`_ * `cryptography installation`_ -.. _lxml installation: http://lxml.de/installation.html +.. _lxml installation: https://lxml.de/installation.html .. _cryptography installation: https://cryptography.io/en/latest/installation/ @@ -253,11 +253,11 @@ For details, see `Issue #2473 `_. .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing/ .. _lxml: https://lxml.de/index.html -.. _parsel: https://pypi.python.org/pypi/parsel -.. _w3lib: https://pypi.python.org/pypi/w3lib -.. _twisted: https://twistedmatrix.com/ -.. _cryptography: https://cryptography.io/ -.. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL +.. _parsel: https://pypi.org/project/parsel/ +.. _w3lib: https://pypi.org/project/w3lib/ +.. _twisted: https://twistedmatrix.com/trac/ +.. _cryptography: https://cryptography.io/en/latest/ +.. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/ .. _setuptools: https://pypi.python.org/pypi/setuptools .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/ .. _homebrew: https://brew.sh/ diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 798fe4a7a..1768badbb 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -306,7 +306,7 @@ with a selector (see :ref:`topics-developer-tools`). visually selected elements, which works in many browsers. .. _regular expressions: https://docs.python.org/3/library/re.html -.. _Selector Gadget: http://selectorgadget.com/ +.. _Selector Gadget: https://selectorgadget.com/ XPath: a brief intro @@ -337,7 +337,7 @@ recommend `this tutorial to learn XPath through examples `_, and `this tutorial to learn "how to think in XPath" `_. -.. _XPath: https://www.w3.org/TR/xpath +.. _XPath: https://www.w3.org/TR/xpath/all/ .. _CSS: https://www.w3.org/TR/selectors Extracting quotes and authors diff --git a/docs/news.rst b/docs/news.rst index e4b985c77..338b53dc4 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -26,7 +26,7 @@ Backward-incompatible changes * Python 3.4 is no longer supported, and some of the minimum requirements of Scrapy have also changed: - * cssselect_ 0.9.1 + * :doc:`cssselect ` 0.9.1 * cryptography_ 2.0 * lxml_ 3.5.0 * pyOpenSSL_ 16.2.0 @@ -1616,7 +1616,7 @@ Deprecations and Removals + ``scrapy.utils.datatypes.SiteNode`` - The previously bundled ``scrapy.xlib.pydispatch`` library was deprecated and - replaced by `pydispatcher `_. + replaced by `pydispatcher `_. Relocations @@ -2450,7 +2450,7 @@ Other ~~~~~ - Dropped Python 2.6 support (:issue:`448`) -- Add `cssselect`_ python package as install dependency +- Add :doc:`cssselect ` python package as install dependency - Drop libxml2 and multi selector's backend support, `lxml`_ is required from now on. - Minimum Twisted version increased to 10.0.0, dropped Twisted 8.0 support. - Running test suite now requires ``mock`` python library (:issue:`390`) @@ -3047,17 +3047,16 @@ Scrapy 0.7 First release of Scrapy. -.. _AJAX crawleable urls: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started?csw=1 +.. _AJAX crawleable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1 .. _botocore: https://github.com/boto/botocore .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/ .. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request .. _cryptography: https://cryptography.io/en/latest/ -.. _cssselect: https://github.com/scrapy/cssselect/ -.. _docstrings: https://docs.python.org/glossary.html#term-docstring -.. _KeyboardInterrupt: https://docs.python.org/library/exceptions.html#KeyboardInterrupt +.. _docstrings: https://docs.python.org/3/glossary.html#term-docstring +.. _KeyboardInterrupt: https://docs.python.org/3/library/exceptions.html#KeyboardInterrupt .. _LevelDB: https://github.com/google/leveldb -.. _lxml: http://lxml.de/ +.. _lxml: https://lxml.de/ .. _marshal: https://docs.python.org/2/library/marshal.html .. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator @@ -3068,11 +3067,11 @@ First release of Scrapy. .. _queuelib: https://github.com/scrapy/queuelib .. _registered with IANA: https://www.iana.org/assignments/media-types/media-types.xhtml .. _resource: https://docs.python.org/2/library/resource.html -.. _robots.txt: http://www.robotstxt.org/ +.. _robots.txt: https://www.robotstxt.org/ .. _scrapely: https://github.com/scrapy/scrapely .. _service_identity: https://service-identity.readthedocs.io/en/stable/ .. _six: https://six.readthedocs.io/ -.. _tox: https://pypi.python.org/pypi/tox +.. _tox: https://pypi.org/project/tox/ .. _Twisted: https://twistedmatrix.com/trac/ .. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ .. _w3lib: https://github.com/scrapy/w3lib diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 4922694ee..63b60312e 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -188,7 +188,7 @@ AjaxCrawlMiddleware helps to crawl them correctly. It is turned OFF by default because it has some performance overhead, and enabling it for focused crawls doesn't make much sense. -.. _ajax crawlable: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started +.. _ajax crawlable: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started .. _broad-crawls-bfo: diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index a83cedcfd..0297ef3a0 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -709,7 +709,7 @@ HttpCompressionMiddleware provided `brotlipy`_ is installed. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt -.. _brotlipy: https://pypi.python.org/pypi/brotlipy +.. _brotlipy: https://pypi.org/project/brotlipy/ HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1038,7 +1038,7 @@ Based on `RobotFileParser * is Python's built-in robots.txt_ parser * is compliant with `Martijn Koster's 1996 draft specification - `_ + `_ * lacks support for wildcard matching @@ -1061,7 +1061,7 @@ Based on `Reppy `_: `_ * is compliant with `Martijn Koster's 1996 draft specification - `_ + `_ * supports wildcard matching @@ -1086,7 +1086,7 @@ Based on `Robotexclusionrulesparser `_: * implemented in Python * is compliant with `Martijn Koster's 1996 draft specification - `_ + `_ * supports wildcard matching @@ -1115,7 +1115,7 @@ implementing the methods described below. .. autoclass:: RobotParser :members: -.. _robots.txt: http://www.robotstxt.org/ +.. _robots.txt: https://www.robotstxt.org/ DownloaderStats --------------- @@ -1155,7 +1155,7 @@ AjaxCrawlMiddleware Middleware that finds 'AJAX crawlable' page variants based on meta-fragment html tag. See - https://developers.google.com/webmasters/ajax-crawling/docs/getting-started + https://developers.google.com/search/docs/ajax-crawling/docs/getting-started for more info. .. note:: diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 1c3607860..b98133676 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -241,12 +241,12 @@ along with `scrapy-selenium`_ for seamless integration. .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser .. _JavaScript: https://en.wikipedia.org/wiki/JavaScript .. _js2xml: https://github.com/scrapinghub/js2xml -.. _json.loads: https://docs.python.org/library/json.html#json.loads +.. _json.loads: https://docs.python.org/3/library/json.html#json.loads .. _pytesseract: https://github.com/madmaze/pytesseract -.. _regular expression: https://docs.python.org/library/re.html +.. _regular expression: https://docs.python.org/3/library/re.html .. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium .. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash -.. _Selenium: https://www.seleniumhq.org/ +.. _Selenium: https://www.selenium.dev/ .. _Splash: https://github.com/scrapinghub/splash .. _tabula-py: https://github.com/chezou/tabula-py .. _wget: https://www.gnu.org/software/wget/ diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index cdc4953c2..801d48fd5 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -158,8 +158,8 @@ method and how to clean up the resources properly.:: self.db[self.collection_name].insert_one(dict(item)) return item -.. _MongoDB: https://www.mongodb.org/ -.. _pymongo: https://api.mongodb.org/python/current/ +.. _MongoDB: https://www.mongodb.com/ +.. _pymongo: https://api.mongodb.com/python/current/ Take screenshot of item diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 15313775b..44643cb67 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -166,7 +166,7 @@ If your item contains mutable_ values like lists or dictionaries, a shallow copy will keep references to the same mutable values across all different copies. -.. _mutable: https://docs.python.org/glossary.html#term-mutable +.. _mutable: https://docs.python.org/3/glossary.html#term-mutable For example, if you have an item with a list of tags, and you create a shallow copy of that item, both the original item and the copy have the same list of @@ -177,7 +177,7 @@ If that is not the desired behavior, use a deep copy instead. See the `documentation of the copy module`_ for more information. -.. _documentation of the copy module: https://docs.python.org/library/copy.html +.. _documentation of the copy module: https://docs.python.org/3/library/copy.html To create a shallow copy of an item, you can either call :meth:`~scrapy.item.Item.copy` on an existing item diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 9fee333ac..c0c83fc84 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -206,7 +206,7 @@ objects. If this is your case, and you can't find your leaks using ``trackref``, you still have another resource: the `Guppy library`_. If you're using Python3, see :ref:`topics-leaks-muppy`. -.. _Guppy library: https://pypi.python.org/pypi/guppy +.. _Guppy library: https://pypi.org/project/guppy/ If you use ``pip``, you can install Guppy with the following command:: @@ -311,9 +311,9 @@ though neither Scrapy nor your project are leaking memory. This is due to a (not so well) known problem of Python, which may not return released memory to the operating system in some cases. For more information on this issue see: -* `Python Memory Management `_ -* `Python Memory Management Part 2 `_ -* `Python Memory Management Part 3 `_ +* `Python Memory Management `_ +* `Python Memory Management Part 2 `_ +* `Python Memory Management Part 3 `_ The improvements proposed by Evan Jones, which are detailed in `this paper`_, got merged in Python 2.5, but this only reduces the problem, it doesn't fix it @@ -327,7 +327,7 @@ completely. To quote the paper: to move to a compacting garbage collector, which is able to move objects in memory. This would require significant changes to the Python interpreter.* -.. _this paper: http://www.evanjones.ca/memoryallocator/ +.. _this paper: https://www.evanjones.ca/memoryallocator/ To keep memory consumption reasonable you can split the job into several smaller jobs or enable :ref:`persistent job queue ` diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f009facd6..c4c2845c9 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -396,7 +396,7 @@ The FormRequest class extends the base :class:`Request` with functionality for dealing with HTML forms. It uses `lxml.html forms`_ to pre-populate form fields with form data from :class:`Response` objects. -.. _lxml.html forms: http://lxml.de/lxmlhtml.html#forms +.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms .. class:: FormRequest(url, [formdata, ...]) diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index c3d431e2a..1f7802c98 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -35,12 +35,11 @@ defines selectors to associate those styles with specific HTML elements. in speed and parsing accuracy to lxml. .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ -.. _lxml: http://lxml.de/ +.. _lxml: https://lxml.de/ .. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html -.. _cssselect: https://pypi.python.org/pypi/cssselect/ -.. _XPath: https://www.w3.org/TR/xpath +.. _XPath: https://www.w3.org/TR/xpath/all/ .. _CSS: https://www.w3.org/TR/selectors -.. _parsel: https://parsel.readthedocs.io/ +.. _parsel: https://parsel.readthedocs.io/en/latest/ Using selectors =============== @@ -255,7 +254,7 @@ that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**: They will most probably not work with other libraries like `lxml`_ or `PyQuery`_. -.. _PyQuery: https://pypi.python.org/pypi/pyquery +.. _PyQuery: https://pypi.org/project/pyquery/ Examples: @@ -309,7 +308,7 @@ Examples: make much sense: text nodes do not have attributes, and attribute values are string values already and do not have children nodes. -.. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors +.. _CSS Selectors: https://www.w3.org/TR/selectors-3/#selectors .. _topics-selectors-nesting-selectors: @@ -504,7 +503,7 @@ Another common case would be to extract all direct ``

`` children: For more details about relative XPaths see the `Location Paths`_ section in the XPath specification. -.. _Location Paths: https://www.w3.org/TR/xpath#location-paths +.. _Location Paths: https://www.w3.org/TR/xpath/all/#location-paths When querying by class, consider using CSS ------------------------------------------ @@ -612,7 +611,7 @@ But using the ``.`` to mean the node, works: >>> sel.xpath("//a[contains(., 'Next Page')]").getall() ['Click here to go to the Next Page'] -.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions +.. _`XPath string function`: https://www.w3.org/TR/xpath/all/#section-String-Functions .. _topics-selectors-xpath-variables: @@ -764,7 +763,7 @@ Set operations These can be handy for excluding parts of a document tree before extracting text elements for example. -Example extracting microdata (sample content taken from http://schema.org/Product) +Example extracting microdata (sample content taken from https://schema.org/Product) with groups of itemscopes and corresponding itemprops:: >>> doc = u""" diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 3cf8311a6..8f7518b19 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -41,7 +41,7 @@ variable; or by defining it in your :ref:`scrapy.cfg `:: .. _IPython: https://ipython.org/ .. _IPython installation guide: https://ipython.org/install.html -.. _bpython: https://www.bpython-interpreter.org/ +.. _bpython: https://bpython-interpreter.org/ Launch the shell ================ @@ -142,7 +142,7 @@ Example of shell session ======================== Here's an example of a typical shell session where we start by scraping the -https://scrapy.org page, and then proceed to scrape the https://reddit.com +https://scrapy.org page, and then proceed to scrape the https://old.reddit.com/ page. Finally, we modify the (Reddit) request method to POST and re-fetch it getting an error. We end the session by typing Ctrl-D (in Unix systems) or Ctrl-Z in Windows. @@ -182,7 +182,7 @@ After that, we can start playing with the objects: >>> response.xpath('//title/text()').get() 'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' ->>> fetch("https://reddit.com") +>>> fetch("https://old.reddit.com/") >>> response.xpath('//title/text()').get() 'reddit: the front page of the internet' diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index b0fb14e24..e0f33de66 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -299,8 +299,8 @@ The spider will not do any parsing on its own. If you were to set the ``start_urls`` attribute from the command line, you would have to parse it on your own into a list using something like -`ast.literal_eval `_ -or `json.loads `_ +`ast.literal_eval `_ +or `json.loads `_ and then set it as an attribute. Otherwise, you would cause iteration over a ``start_urls`` string (a very common python pitfall) @@ -811,6 +811,6 @@ Combine SitemapSpider with other sources of urls:: .. _Sitemaps: https://www.sitemaps.org/index.html .. _Sitemap index files: https://www.sitemaps.org/protocol.html#index -.. _robots.txt: http://www.robotstxt.org/ +.. _robots.txt: https://www.robotstxt.org/ .. _TLD: https://en.wikipedia.org/wiki/Top-level_domain .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/ From 034e2c31c7d55333c3de208f80dcee1bf45ef9b9 Mon Sep 17 00:00:00 2001 From: gunblues Date: Wed, 26 Feb 2020 03:46:05 +0800 Subject: [PATCH 096/281] Use a non-zero exit code when a pipeline's open_spider method throws an exception (#4207) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix issue 4175 - Scrapy does not use a non-zero exit code when pipeline's open_spider throws the exception * remove extra blank lines * remove redundant code * remove blank line at end of file * more suitable naming for response and make if-condition shorter * avoid error - AttributeError: 'Deferred' object has no attribute 'result' * use getattr to make code concisely * add test * remove useless file * modify test class name * remove unneccessary files * Fix Flake8-reported issue * fix these items which are suggested by Gallaecio ・Sort those imports at tests/test_cmdline_crawl_with_pipeline/__init__.py ・Remove the unused setUp method. ・Remove comments generated by Scrapy’s project generation tool. ・Remove the [deploy] section from the scrapy.cfg file (I don’t think it’s needed here) ・Remove BOT_NAME and NEWSPIDER_MODULE from settings.py (I think there are not needed either, although I’m less sure about NEWSPIDER_MODULE) * have to reserve BOT_NAME, SPIDER_MODULES in settings.py * Remove unneeded empty lines * Empty __init__.py file with unneeded comments * Remove an unneeded empty line at the end * Remove unneeed empty line from __init__.py file * Update __init__.py * Update __init__.py * Update exception.py * Update normal.py * Update __init__.py * Update __init__.py * fix W391 blank line at end of file Co-authored-by: Adrián Chaves --- scrapy/commands/crawl.py | 11 +++++++--- .../__init__.py | 20 +++++++++++++++++++ .../scrapy.cfg | 2 ++ .../test_spider/__init__.py | 0 .../test_spider/pipelines.py | 16 +++++++++++++++ .../test_spider/settings.py | 2 ++ .../test_spider/spiders/__init__.py | 0 .../test_spider/spiders/exception.py | 14 +++++++++++++ .../test_spider/spiders/normal.py | 14 +++++++++++++ 9 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 tests/test_cmdline_crawl_with_pipeline/__init__.py create mode 100644 tests/test_cmdline_crawl_with_pipeline/scrapy.cfg create mode 100644 tests/test_cmdline_crawl_with_pipeline/test_spider/__init__.py create mode 100644 tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py create mode 100644 tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py create mode 100644 tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/__init__.py create mode 100644 tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py create mode 100644 tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 8093fd402..7b417e2eb 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -54,8 +54,13 @@ class Command(ScrapyCommand): raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported") spname = args[0] - self.crawler_process.crawl(spname, **opts.spargs) - self.crawler_process.start() + crawl_defer = self.crawler_process.crawl(spname, **opts.spargs) - if self.crawler_process.bootstrap_failed: + if getattr(crawl_defer, 'result', None) is not None and issubclass(crawl_defer.result.type, Exception): self.exitcode = 1 + else: + self.crawler_process.start() + + if self.crawler_process.bootstrap_failed or \ + (hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception): + self.exitcode = 1 diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py new file mode 100644 index 000000000..d341888d3 --- /dev/null +++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py @@ -0,0 +1,20 @@ +import os +import sys +import unittest +from subprocess import Popen, PIPE + + +class CmdlineCrawlPipelineTest(unittest.TestCase): + + def _execute(self, spname): + args = (sys.executable, '-m', 'scrapy.cmdline', 'crawl', spname) + cwd = os.path.dirname(os.path.abspath(__file__)) + proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd) + proc.communicate() + return proc.returncode + + def test_open_spider_normally_in_pipeline(self): + self.assertEqual(self._execute('normal'), 0) + + def test_exception_at_open_spider_in_pipeline(self): + self.assertEqual(self._execute('exception'), 1) diff --git a/tests/test_cmdline_crawl_with_pipeline/scrapy.cfg b/tests/test_cmdline_crawl_with_pipeline/scrapy.cfg new file mode 100644 index 000000000..2f238dba3 --- /dev/null +++ b/tests/test_cmdline_crawl_with_pipeline/scrapy.cfg @@ -0,0 +1,2 @@ +[settings] +default = test_spider.settings diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/__init__.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py new file mode 100644 index 000000000..ce916f699 --- /dev/null +++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py @@ -0,0 +1,16 @@ +class TestSpiderPipeline(object): + + def open_spider(self, spider): + pass + + def process_item(self, item, spider): + return item + + +class TestSpiderExceptionPipeline(object): + + def open_spider(self, spider): + raise Exception('exception') + + def process_item(self, item, spider): + return item diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py new file mode 100644 index 000000000..ae782c0d8 --- /dev/null +++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/settings.py @@ -0,0 +1,2 @@ +BOT_NAME = 'test_spider' +SPIDER_MODULES = ['test_spider.spiders'] diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/__init__.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py new file mode 100644 index 000000000..300f45ebf --- /dev/null +++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/exception.py @@ -0,0 +1,14 @@ +import scrapy + + +class ExceptionSpider(scrapy.Spider): + name = 'exception' + + custom_settings = { + 'ITEM_PIPELINES': { + 'test_spider.pipelines.TestSpiderExceptionPipeline': 300 + } + } + + def parse(self, response): + pass diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py new file mode 100644 index 000000000..87a40fdcb --- /dev/null +++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/spiders/normal.py @@ -0,0 +1,14 @@ +import scrapy + + +class NormalSpider(scrapy.Spider): + name = 'normal' + + custom_settings = { + 'ITEM_PIPELINES': { + 'test_spider.pipelines.TestSpiderPipeline': 300 + } + } + + def parse(self, response): + pass From 7291173f6b6a8e1768ab9d5f52474cd8ada8381e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 25 Feb 2020 21:35:21 +0100 Subject: [PATCH 097/281] Have ReadTheDocs builds fail on warning --- .readthedocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index 563add75f..0b9e15018 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,6 +1,7 @@ version: 2 sphinx: configuration: docs/conf.py + fail_on_warning: true python: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image From a9d7d8f064fe3086227d4737d07e3ca4a296b4b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 25 Feb 2020 21:41:07 +0100 Subject: [PATCH 098/281] Add Scrapy dependencies back to docs/requirements.txt --- docs/requirements.txt | 15 +++++++++++++++ setup.py | 1 + 2 files changed, 16 insertions(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 773b92cea..215cdd64d 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,3 +2,18 @@ Sphinx>=2.1 sphinx-hoverxref sphinx-notfound-page sphinx_rtd_theme + +# Required for ReadTheDocs +# Keep in sync with setup.py +Twisted>=17.9.0 +cryptography>=2.0 +cssselect>=0.9.1 +lxml>=3.5.0 +parsel>=1.5.0 +PyDispatcher>=2.0.5 +pyOpenSSL>=16.2.0 +queuelib>=1.4.2 +service_identity>=16.0.0 +w3lib>=1.17.0 +zope.interface>=4.1.3 +protego>=0.1.15 diff --git a/setup.py b/setup.py index 85d797f88..6f15ca277 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ setup( 'Topic :: Software Development :: Libraries :: Python Modules', ], python_requires='>=3.5', + # Keep in sync with docs/requirements.txt install_requires=[ 'Twisted>=17.9.0', 'cryptography>=2.0', From 778813717df0d5fcd4359266f6785068feea0785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 25 Feb 2020 21:58:28 +0100 Subject: [PATCH 099/281] Use ReadTheDocs install.path --- .readthedocs.yml | 1 + docs/requirements.txt | 15 --------------- setup.py | 1 - 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 0b9e15018..17eba34f3 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,3 +8,4 @@ python: version: 3.7 # Keep in sync with .travis.yml install: - requirements: docs/requirements.txt + - path: . diff --git a/docs/requirements.txt b/docs/requirements.txt index 215cdd64d..773b92cea 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,18 +2,3 @@ Sphinx>=2.1 sphinx-hoverxref sphinx-notfound-page sphinx_rtd_theme - -# Required for ReadTheDocs -# Keep in sync with setup.py -Twisted>=17.9.0 -cryptography>=2.0 -cssselect>=0.9.1 -lxml>=3.5.0 -parsel>=1.5.0 -PyDispatcher>=2.0.5 -pyOpenSSL>=16.2.0 -queuelib>=1.4.2 -service_identity>=16.0.0 -w3lib>=1.17.0 -zope.interface>=4.1.3 -protego>=0.1.15 diff --git a/setup.py b/setup.py index 6f15ca277..85d797f88 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,6 @@ setup( 'Topic :: Software Development :: Libraries :: Python Modules', ], python_requires='>=3.5', - # Keep in sync with docs/requirements.txt install_requires=[ 'Twisted>=17.9.0', 'cryptography>=2.0', From 6109ad9aacd5897f67842685b2405627b4af4ad6 Mon Sep 17 00:00:00 2001 From: HEndo12345 <38522238+HEndo12345@users.noreply.github.com> Date: Thu, 27 Feb 2020 23:15:30 +0900 Subject: [PATCH 100/281] Clean up the deprecated settings list (#4378) --- scrapy/settings/deprecated.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scrapy/settings/deprecated.py b/scrapy/settings/deprecated.py index 91ed689e8..1211908df 100644 --- a/scrapy/settings/deprecated.py +++ b/scrapy/settings/deprecated.py @@ -9,10 +9,8 @@ DEPRECATED_SETTINGS = [ ('ENCODING_ALIASES', 'no longer needed (encoding discovery uses w3lib now)'), ('STATS_ENABLED', 'no longer supported (change STATS_CLASS instead)'), ('SQLITE_DB', 'no longer supported'), - ('SELECTORS_BACKEND', 'use SCRAPY_SELECTORS_BACKEND environment variable instead'), ('AUTOTHROTTLE_MIN_DOWNLOAD_DELAY', 'use DOWNLOAD_DELAY instead'), ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'), - ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'), ('REDIRECT_MAX_METAREFRESH_DELAY', 'use METAREFRESH_MAXDELAY instead'), ('LOG_UNSERIALIZABLE_REQUESTS', 'use SCHEDULER_DEBUG instead'), ] From 2acaa86231e8a743333928907c2933feadf40cd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 27 Feb 2020 15:39:49 +0100 Subject: [PATCH 101/281] Do not warn about valid environment variables --- scrapy/utils/project.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index d9a03ff63..d1dec2543 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -75,9 +75,24 @@ def get_project_settings(): "is deprecated.", ScrapyDeprecationWarning) settings.setdict(pickle.loads(pickled_settings), priority='project') - env_overrides = {k[7:]: v for k, v in os.environ.items() if - k.startswith('SCRAPY_')} - if env_overrides: - warnings.warn("Use of 'SCRAPY_'-prefixed environment variables to override settings is deprecated.", ScrapyDeprecationWarning) - settings.setdict(env_overrides, priority='project') + scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if + k.startswith('SCRAPY_')} + valid_envvars = { + 'SCRAPY_CHECK', + 'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE', + 'SCRAPY_PROJECT', + 'SCRAPY_PYTHON_SHELL', + 'SCRAPY_SETTINGS_MODULE', + } + setting_envvars = {k for k in scrapy_envvars if k not in valid_envvars} + if setting_envvars: + setting_envvar_list = ', '.join(sorted(setting_envvars)) + warnings.warn( + 'Use of environment variables prefixed with SCRAPY_ to override ' + 'settings is deprecated. The following environment variables are ' + 'currently defined: {}'.format(setting_envvar_list), + ScrapyDeprecationWarning + ) + settings.setdict(scrapy_envvars, priority='project') + return settings From 9aae4c0be7b42e27daa2750b6f01eb497edcd98a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 27 Feb 2020 16:31:43 +0100 Subject: [PATCH 102/281] Add tests for envvar setting warnings --- scrapy/utils/project.py | 10 +++---- tests/test_utils_project.py | 56 ++++++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index d1dec2543..b8d3ebf9d 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -78,11 +78,11 @@ def get_project_settings(): scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if k.startswith('SCRAPY_')} valid_envvars = { - 'SCRAPY_CHECK', - 'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE', - 'SCRAPY_PROJECT', - 'SCRAPY_PYTHON_SHELL', - 'SCRAPY_SETTINGS_MODULE', + 'CHECK', + 'PICKLED_SETTINGS_TO_OVERRIDE', + 'PROJECT', + 'PYTHON_SHELL', + 'SETTINGS_MODULE', } setting_envvars = {k for k in scrapy_envvars if k not in valid_envvars} if setting_envvars: diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index bd74b0c34..1ef4eeb14 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -3,7 +3,11 @@ import os import tempfile import shutil import contextlib -from scrapy.utils.project import data_path + +from pytest import warns + +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.project import data_path, get_project_settings @contextlib.contextmanager @@ -41,3 +45,53 @@ class ProjectUtilsTest(unittest.TestCase): ) abspath = os.path.join(os.path.sep, 'absolute', 'path') self.assertEqual(abspath, data_path(abspath)) + + +@contextlib.contextmanager +def set_env(**update): + modified = set(update.keys()) & set(os.environ.keys()) + update_after = {k: os.environ[k] for k in modified} + remove_after = frozenset(k for k in update if k not in os.environ) + try: + os.environ.update(update) + yield + finally: + os.environ.update(update_after) + for k in remove_after: + os.environ.pop(k) + + +class GetProjectSettingsTestCase(unittest.TestCase): + + def test_valid_envvar(self): + value = 'tests.test_cmdline.settings' + envvars = { + 'SCRAPY_SETTINGS_MODULE': value, + } + with set_env(**envvars), warns(None) as warnings: + settings = get_project_settings() + assert not warnings + assert settings.get('SETTINGS_MODULE') == value + + def test_invalid_envvar(self): + envvars = { + 'SCRAPY_FOO': 'bar', + } + with set_env(**envvars), warns(None) as warnings: + get_project_settings() + assert len(warnings) == 1 + assert warnings[0].category == ScrapyDeprecationWarning + assert str(warnings[0].message).endswith(': FOO') + + def test_valid_and_invalid_envvars(self): + value = 'tests.test_cmdline.settings' + envvars = { + 'SCRAPY_FOO': 'bar', + 'SCRAPY_SETTINGS_MODULE': value, + } + with set_env(**envvars), warns(None) as warnings: + settings = get_project_settings() + assert len(warnings) == 1 + assert warnings[0].category == ScrapyDeprecationWarning + assert str(warnings[0].message).endswith(': FOO') + assert settings.get('SETTINGS_MODULE') == value From c411a51f42a5e6d241d69349b228f1584fdbd31b Mon Sep 17 00:00:00 2001 From: sakshamb2113 <44064539+sakshamb2113@users.noreply.github.com> Date: Fri, 28 Feb 2020 17:47:02 +0530 Subject: [PATCH 103/281] Fix random failures from test_fixed_delay in some machines (#4372) --- tests/test_crawl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index bbe97d034..e93c668c5 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -37,7 +37,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_fixed_delay(self): - yield self._test_delay(total=3, delay=0.1) + yield self._test_delay(total=3, delay=0.2) @defer.inlineCallbacks def test_randomized_delay(self): From ef00f8eb8eb4f5727409fd40c5826661db2bb665 Mon Sep 17 00:00:00 2001 From: MaliCN <40772522+MaliYudina@users.noreply.github.com> Date: Fri, 28 Feb 2020 22:42:07 +0300 Subject: [PATCH 104/281] updated with new macOS name (#4308) (#4323) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * changed for new name as "macOS" (issue #4308) * updated macOS name * update macOS name * updated macOS name * update for new macOS name * docs/intro/install.rst: fix macOS header symbols Co-Authored-By: elacuesta Co-authored-by: Adrián Chaves Co-authored-by: elacuesta --- README.rst | 2 +- docs/intro/install.rst | 12 ++++++------ docs/news.rst | 12 ++++++------ scrapy/extensions/memusage.py | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index 7fefaeec9..ce5973bcd 100644 --- a/README.rst +++ b/README.rst @@ -41,7 +41,7 @@ Requirements ============ * Python 3.5+ -* Works on Linux, Windows, Mac OSX, BSD +* Works on Linux, Windows, macOS, BSD Install ======= diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 871281460..89ba0c154 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -12,7 +12,7 @@ under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). If you're using `Anaconda`_ or `Miniconda`_, you can install the package from the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows -and OS X. +and macOS. To install Scrapy using ``conda``, run:: @@ -148,11 +148,11 @@ you can install Scrapy with ``pip`` after that:: .. _intro-install-macos: -Mac OS X --------- +macOS +----- Building Scrapy's dependencies requires the presence of a C compiler and -development headers. On OS X this is typically provided by Apple’s Xcode +development headers. On macOS this is typically provided by Apple’s Xcode development tools. To install the Xcode command line tools open a terminal window and run:: @@ -191,7 +191,7 @@ solutions: * *(Optional)* :ref:`Install Scrapy inside a Python virtual environment `. - This method is a workaround for the above OS X issue, but it's an overall + This method is a workaround for the above macOS issue, but it's an overall good practice for managing dependencies and can complement the first method. After any of these workarounds you should be able to install Scrapy:: @@ -207,7 +207,7 @@ For PyPy3, only Linux installation was tested. Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy. This means that these dependecies will be built during installation. -On OS X, you are likely to face an issue with building Cryptography dependency, +On macOS, you are likely to face an issue with building Cryptography dependency, solution to this problem is described `here `_, that is to ``brew install openssl`` and then export the flags that this command diff --git a/docs/news.rst b/docs/news.rst index 338b53dc4..c1daedaf2 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1076,7 +1076,7 @@ Cleanups & Refactoring ~~~~~~~~~~~~~~~~~~~~~~ - Tests: remove temp files and folders (:issue:`2570`), - fixed ProjectUtilsTest on OS X (:issue:`2569`), + fixed ProjectUtilsTest on macOS (:issue:`2569`), use portable pypy for Linux on Travis CI (:issue:`2710`) - Separate building request from ``_requests_to_follow`` in CrawlSpider (:issue:`2562`) - Remove “Python 3 progress” badge (:issue:`2567`) @@ -1645,7 +1645,7 @@ Bugfixes - Makes ``_monkeypatches`` more robust (:issue:`1634`). - Fixed bug on ``XMLItemExporter`` with non-string fields in items (:issue:`1738`). -- Fixed startproject command in OS X (:issue:`1635`). +- Fixed startproject command in macOS (:issue:`1635`). - Fixed :class:`~scrapy.exporters.PythonItemExporter` and CSVExporter for non-string item types (:issue:`1737`). - Various logging related fixes (:issue:`1294`, :issue:`1419`, :issue:`1263`, @@ -1713,12 +1713,12 @@ Scrapy 1.0.4 (2015-12-30) - Typos corrections (:commit:`7067117`) - fix typos in downloader-middleware.rst and exceptions.rst, middlware -> middleware (:commit:`32f115c`) - Add note to Ubuntu install section about Debian compatibility (:commit:`23fda69`) -- Replace alternative OSX install workaround with virtualenv (:commit:`98b63ee`) +- Replace alternative macOS install workaround with virtualenv (:commit:`98b63ee`) - Reference Homebrew's homepage for installation instructions (:commit:`1925db1`) - Add oldest supported tox version to contributing docs (:commit:`5d10d6d`) - Note in install docs about pip being already included in python>=2.7.9 (:commit:`85c980e`) - Add non-python dependencies to Ubuntu install section in the docs (:commit:`fbd010d`) -- Add OS X installation section to docs (:commit:`d8f4cba`) +- Add macOS installation section to docs (:commit:`d8f4cba`) - DOC(ENH): specify path to rtd theme explicitly (:commit:`de73b1a`) - minor: scrapy.Spider docs grammar (:commit:`1ddcc7b`) - Make common practices sample code match the comments (:commit:`1b85bcf`) @@ -2571,7 +2571,7 @@ Scrapy 0.18.0 (released 2013-08-09) - MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62 - added from_crawler method to spiders - added system tests with mock server -- more improvements to Mac OS compatibility (thanks Alex Cepoi) +- more improvements to macOS compatibility (thanks Alex Cepoi) - several more cleanups to singletons and multi-spider support (thanks Nicolas Ramirez) - support custom download slots - added --spider option to "shell" command. @@ -2647,7 +2647,7 @@ Scrapy 0.16.3 (released 2012-12-07) - Remove concurrency limitation when using download delays and still ensure inter-request delays are enforced (:commit:`487b9b5`) - add error details when image pipeline fails (:commit:`8232569`) -- improve mac os compatibility (:commit:`8dcf8aa`) +- improve macOS compatibility (:commit:`8dcf8aa`) - setup.py: use README.rst to populate long_description (:commit:`7b5310d`) - doc: removed obsolete references to ClientForm (:commit:`80f9bb6`) - correct docs for default storage backend (:commit:`2aa491b`) diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index c0570567e..14e0fb32d 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -47,7 +47,7 @@ class MemoryUsage(object): def get_virtual_size(self): size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss if sys.platform != 'darwin': - # on Mac OS X ru_maxrss is in bytes, on Linux it is in KB + # on macOS ru_maxrss is in bytes, on Linux it is in KB size *= 1024 return size From 6aa0ba45532a4fd8e868bb4ea15bf002e430e67f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 3 Mar 2020 09:11:11 +0100 Subject: [PATCH 105/281] Write release notes for Scrapy 2.0.0 (#4329) --- docs/index.rst | 8 + docs/intro/install.rst | 4 +- docs/news.rst | 458 +++++++++++++++++++++++++- docs/topics/asyncio.rst | 28 ++ docs/topics/coroutines.rst | 110 +++++++ docs/topics/downloader-middleware.rst | 4 + docs/topics/exporters.rst | 5 +- docs/topics/feed-exports.rst | 3 + docs/topics/item-pipeline.rst | 13 +- docs/topics/jobs.rst | 3 + docs/topics/link-extractors.rst | 10 +- docs/topics/loaders.rst | 3 + docs/topics/media-pipeline.rst | 12 +- docs/topics/request-response.rst | 8 + docs/topics/settings.rst | 43 ++- docs/topics/signals.rst | 3 + docs/topics/spiders.rst | 3 + scrapy/http/response/__init__.py | 5 + scrapy/logformatter.py | 17 +- scrapy/utils/reactor.py | 5 + tests/test_crawl.py | 2 +- 21 files changed, 704 insertions(+), 43 deletions(-) create mode 100644 docs/topics/asyncio.rst create mode 100644 docs/topics/coroutines.rst diff --git a/docs/index.rst b/docs/index.rst index a4343b7e0..11aa5c9be 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -165,6 +165,8 @@ Solving specific problems topics/autothrottle topics/benchmarking topics/jobs + topics/coroutines + topics/asyncio :doc:`faq` Get answers to most frequently asked questions. @@ -205,6 +207,12 @@ Solving specific problems :doc:`topics/jobs` Learn how to pause and resume crawls for large spiders. +:doc:`topics/coroutines` + Use the :ref:`coroutine syntax `. + +:doc:`topics/asyncio` + Use :mod:`asyncio` and :mod:`asyncio`-powered libraries. + .. _extending-scrapy: Extending Scrapy diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 89ba0c154..6356e0eea 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,8 +7,8 @@ Installation guide Installing Scrapy ================= -Scrapy runs on Python 3.5 or above -under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). +Scrapy runs on Python 3.5 or above under CPython (default Python +implementation) and PyPy (starting with PyPy 5.9). If you're using `Anaconda`_ or `Miniconda`_, you can install the package from the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows diff --git a/docs/news.rst b/docs/news.rst index c1daedaf2..dd5e00223 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,8 +3,452 @@ Release notes ============= -.. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0, - planned for Q4 2019 or Q1 2020, will support **Python 3 only**. +.. _release-2.0.0: + +Scrapy 2.0.0 (2020-03-03) +------------------------- + +Highlights: + +* Python 2 support has been removed +* :doc:`Partial ` :ref:`coroutine syntax ` support + and :doc:`experimental ` :mod:`asyncio` support +* New :meth:`Response.follow_all ` method +* :ref:`FTP support ` for media pipelines +* New :attr:`Response.certificate ` + attribute +* IPv6 support through :setting:`DNS_RESOLVER` + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Python 2 support has been removed, following `Python 2 end-of-life on + January 1, 2020`_ (:issue:`4091`, :issue:`4114`, :issue:`4115`, + :issue:`4121`, :issue:`4138`, :issue:`4231`, :issue:`4242`, :issue:`4304`, + :issue:`4309`, :issue:`4373`) + +* Retry gaveups (see :setting:`RETRY_TIMES`) are now logged as errors instead + of as debug information (:issue:`3171`, :issue:`3566`) + +* File extensions that + :class:`LinkExtractor ` + ignores by default now also include ``7z``, ``7zip``, ``apk``, ``bz2``, + ``cdr``, ``dmg``, ``ico``, ``iso``, ``tar``, ``tar.gz``, ``webm``, and + ``xz`` (:issue:`1837`, :issue:`2067`, :issue:`4066`) + +* The :setting:`METAREFRESH_IGNORE_TAGS` setting is now an empty list by + default, following web browser behavior (:issue:`3844`, :issue:`4311`) + +* The + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + now includes spaces after commas in the value of the ``Accept-Encoding`` + header that it sets, following web browser behavior (:issue:`4293`) + +* The ``__init__`` method of custom download handlers (see + :setting:`DOWNLOAD_HANDLERS`) or subclasses of the following downloader + handlers no longer receives a ``settings`` parameter: + + * :class:`scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler` + + * :class:`scrapy.core.downloader.handlers.file.FileDownloadHandler` + + Use the ``from_settings`` or ``from_crawler`` class methods to expose such + a parameter to your custom download handlers. + + (:issue:`4126`) + +* We have refactored the :class:`scrapy.core.scheduler.Scheduler` class and + related queue classes (see :setting:`SCHEDULER_PRIORITY_QUEUE`, + :setting:`SCHEDULER_DISK_QUEUE` and :setting:`SCHEDULER_MEMORY_QUEUE`) to + make it easier to implement custom scheduler queue classes. See + :ref:`2-0-0-scheduler-queue-changes` below for details. + +* Overridden settings are now logged in a different format. This is more in + line with similar information logged at startup (:issue:`4199`) + +.. _Python 2 end-of-life on January 1, 2020: https://www.python.org/doc/sunset-python-2/ + + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +* The :ref:`Scrapy shell ` no longer provides a `sel` proxy + object, use :meth:`response.selector ` + instead (:issue:`4347`) + +* LevelDB support has been removed (:issue:`4112`) + +* The following functions have been removed from :mod:`scrapy.utils.python`: + ``isbinarytext``, ``is_writable``, ``setattr_default``, ``stringify_dict`` + (:issue:`4362`) + + +Deprecations +~~~~~~~~~~~~ + +* Using environment variables prefixed with ``SCRAPY_`` to override settings + is deprecated (:issue:`4300`, :issue:`4374`, :issue:`4375`) + +* :class:`scrapy.linkextractors.FilteringLinkExtractor` is deprecated, use + :class:`scrapy.linkextractors.LinkExtractor + ` instead (:issue:`4045`) + +* The ``noconnect`` query string argument of proxy URLs is deprecated and + should be removed from proxy URLs (:issue:`4198`) + +* The :meth:`next ` method of + :class:`scrapy.utils.python.MutableChain` is deprecated, use the global + :func:`next` function or :meth:`MutableChain.__next__ + ` instead (:issue:`4153`) + + +New features +~~~~~~~~~~~~ + +* Added :doc:`partial support ` for Python’s + :ref:`coroutine syntax ` and :doc:`experimental support + ` for :mod:`asyncio` and :mod:`asyncio`-powered libraries + (:issue:`4010`, :issue:`4259`, :issue:`4269`, :issue:`4270`, :issue:`4271`, + :issue:`4316`, :issue:`4318`) + +* The new :meth:`Response.follow_all ` + method offers the same functionality as + :meth:`Response.follow ` but supports an + iterable of URLs as input and returns an iterable of requests + (:issue:`2582`, :issue:`4057`, :issue:`4286`) + +* :ref:`Media pipelines ` now support :ref:`FTP + storage ` (:issue:`3928`, :issue:`3961`) + +* The new :attr:`Response.certificate ` + attribute exposes the SSL certificate of the server as a + :class:`twisted.internet.ssl.Certificate` object for HTTPS responses + (:issue:`2726`, :issue:`4054`) + +* A new :setting:`DNS_RESOLVER` setting allows enabling IPv6 support + (:issue:`1031`, :issue:`4227`) + +* A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring + the existing soft limit that pauses request downloads when the total + response data being processed is too high (:issue:`1410`, :issue:`3551`) + +* A new :setting:`TWISTED_REACTOR` setting allows customizing the + :mod:`~twisted.internet.reactor` that Scrapy uses, allowing to + :doc:`enable asyncio support ` or deal with a + :ref:`common macOS issue ` (:issue:`2905`, + :issue:`4294`) + +* Scheduler disk and memory queues may now use the class methods + ``from_crawler`` or ``from_settings`` (:issue:`3884`) + +* The new :attr:`Response.cb_kwargs ` + attribute serves as a shortcut for :attr:`Response.request.cb_kwargs + ` (:issue:`4331`) + +* :meth:`Response.follow ` now supports a + ``flags`` parameter, for consistency with :class:`~scrapy.http.Request` + (:issue:`4277`, :issue:`4279`) + +* :ref:`Item loader processors ` can now be + regular functions, they no longer need to be methods (:issue:`3899`) + +* :class:`~scrapy.spiders.Rule` now accepts an ``errback`` parameter + (:issue:`4000`) + +* :class:`~scrapy.http.Request` no longer requires a ``callback`` parameter + when an ``errback`` parameter is specified (:issue:`3586`, :issue:`4008`) + +* :class:`~scrapy.logformatter.LogFormatter` now supports some additional + methods: + + * :class:`~scrapy.logformatter.LogFormatter.download_error` for + download errors + + * :class:`~scrapy.logformatter.LogFormatter.item_error` for exceptions + raised during item processing by :ref:`item pipelines + ` + + * :class:`~scrapy.logformatter.LogFormatter.spider_error` for exceptions + raised from :ref:`spider callbacks ` + + (:issue:`374`, :issue:`3986`, :issue:`3989`, :issue:`4176`, :issue:`4188`) + +* The :setting:`FEED_URI` setting now supports :class:`pathlib.Path` values + (:issue:`3731`, :issue:`4074`) + +* A new :signal:`request_left_downloader` signal is sent when a request + leaves the downloader (:issue:`4303`) + +* Scrapy logs a warning when it detects a request callback or errback that + uses ``yield`` but also returns a value, since the returned value would be + lost (:issue:`3484`, :issue:`3869`) + +* :class:`~scrapy.spiders.Spider` objects now raise an :exc:`AttributeError` + exception if they do not have a :class:`~scrapy.spiders.Spider.start_urls` + attribute nor reimplement :class:`~scrapy.spiders.Spider.start_requests`, + but have a ``start_url`` attribute (:issue:`4133`, :issue:`4170`) + +* :class:`~scrapy.exporters.BaseItemExporter` subclasses may now use + ``super().__init__(**kwargs)`` instead of ``self._configure(kwargs)`` in + their ``__init__`` method, passing ``dont_fail=True`` to the parent + ``__init__`` method if needed, and accessing ``kwargs`` at ``self._kwargs`` + after calling their parent ``__init__`` method (:issue:`4193`, + :issue:`4370`) + +* A new ``keep_fragments`` parameter of + :func:`scrapy.utils.request.request_fingerprint` allows to generate + different fingerprints for requests with different fragments in their URL + (:issue:`4104`) + +* Download handlers (see :setting:`DOWNLOAD_HANDLERS`) may now use the + ``from_settings`` and ``from_crawler`` class methods that other Scrapy + components already supported (:issue:`4126`) + +* :class:`scrapy.utils.python.MutableChain.__iter__` now returns ``self``, + `allowing it to be used as a sequence `_ + (:issue:`4153`) + + +Bug fixes +~~~~~~~~~ + +* The :command:`crawl` command now also exits with exit code 1 when an + exception happens before the crawling starts (:issue:`4175`, :issue:`4207`) + +* :class:`LinkExtractor.extract_links + ` no longer + re-encodes the query string or URLs from non-UTF-8 responses in UTF-8 + (:issue:`998`, :issue:`1403`, :issue:`1949`, :issue:`4321`) + +* The first spider middleware (see :setting:`SPIDER_MIDDLEWARES`) now also + processes exceptions raised from callbacks that are generators + (:issue:`4260`, :issue:`4272`) + +* Redirects to URLs starting with 3 slashes (``///``) are now supported + (:issue:`4032`, :issue:`4042`) + +* :class:`~scrapy.http.Request` no longer accepts strings as ``url`` simply + because they have a colon (:issue:`2552`, :issue:`4094`) + +* The correct encoding is now used for attach names in + :class:`~scrapy.mail.MailSender` (:issue:`4229`, :issue:`4239`) + +* :class:`~scrapy.dupefilters.RFPDupeFilter`, the default + :setting:`DUPEFILTER_CLASS`, no longer writes an extra ``\r`` character on + each line in Windows, which made the size of the ``requests.seen`` file + unnecessarily large on that platform (:issue:`4283`) + +* Z shell auto-completion now looks for ``.html`` files, not ``.http`` files, + and covers the ``-h`` command-line switch (:issue:`4122`, :issue:`4291`) + +* Adding items to a :class:`scrapy.utils.datatypes.LocalCache` object + without a ``limit`` defined no longer raises a :exc:`TypeError` exception + (:issue:`4123`) + +* Fixed a typo in the message of the :exc:`ValueError` exception raised when + :func:`scrapy.utils.misc.create_instance` gets both ``settings`` and + ``crawler`` set to ``None`` (:issue:`4128`) + + +Documentation +~~~~~~~~~~~~~ + +* API documentation now links to an online, syntax-highlighted view of the + corresponding source code (:issue:`4148`) + +* Links to unexisting documentation pages now allow access to the sidebar + (:issue:`4152`, :issue:`4169`) + +* Cross-references within our documentation now display a tooltip when + hovered (:issue:`4173`, :issue:`4183`) + +* Improved the documentation about :meth:`LinkExtractor.extract_links + ` and + simplified :ref:`topics-link-extractors` (:issue:`4045`) + +* Clarified how :class:`ItemLoader.item ` + works (:issue:`3574`, :issue:`4099`) + +* Clarified that :func:`logging.basicConfig` should not be used when also + using :class:`~scrapy.crawler.CrawlerProcess` (:issue:`2149`, + :issue:`2352`, :issue:`3146`, :issue:`3960`) + +* Clarified the requirements for :class:`~scrapy.http.Request` objects + :ref:`when using persistence ` (:issue:`4124`, + :issue:`4139`) + +* Clarified how to install a :ref:`custom image pipeline + ` (:issue:`4034`, :issue:`4252`) + +* Fixed the signatures of the ``file_path`` method in :ref:`media pipeline + ` examples (:issue:`4290`) + +* Covered a backward-incompatible change in Scrapy 1.7.0 affecting custom + :class:`scrapy.core.scheduler.Scheduler` subclasses (:issue:`4274`) + +* Improved the ``README.rst`` and ``CODE_OF_CONDUCT.md`` files + (:issue:`4059`) + +* Documentation examples are now checked as part of our test suite and we + have fixed some of the issues detected (:issue:`4142`, :issue:`4146`, + :issue:`4171`, :issue:`4184`, :issue:`4190`) + +* Fixed logic issues, broken links and typos (:issue:`4247`, :issue:`4258`, + :issue:`4282`, :issue:`4288`, :issue:`4305`, :issue:`4308`, :issue:`4323`, + :issue:`4338`, :issue:`4359`, :issue:`4361`) + +* Improved consistency when referring to the ``__init__`` method of an object + (:issue:`4086`, :issue:`4088`) + +* Fixed an inconsistency between code and output in :ref:`intro-overview` + (:issue:`4213`) + +* Extended :mod:`~sphinx.ext.intersphinx` usage (:issue:`4147`, + :issue:`4172`, :issue:`4185`, :issue:`4194`, :issue:`4197`) + +* We now use a recent version of Python to build the documentation + (:issue:`4140`, :issue:`4249`) + +* Cleaned up documentation (:issue:`4143`, :issue:`4275`) + + +Quality assurance +~~~~~~~~~~~~~~~~~ + +* Re-enabled proxy ``CONNECT`` tests (:issue:`2545`, :issue:`4114`) + +* Added Bandit_ security checks to our test suite (:issue:`4162`, + :issue:`4181`) + +* Added Flake8_ style checks to our test suite and applied many of the + corresponding changes (:issue:`3944`, :issue:`3945`, :issue:`4137`, + :issue:`4157`, :issue:`4167`, :issue:`4174`, :issue:`4186`, :issue:`4195`, + :issue:`4238`, :issue:`4246`, :issue:`4355`, :issue:`4360`, :issue:`4365`) + +* Improved test coverage (:issue:`4097`, :issue:`4218`, :issue:`4236`) + +* Started reporting slowest tests, and improved the performance of some of + them (:issue:`4163`, :issue:`4164`) + +* Fixed broken tests and refactored some tests (:issue:`4014`, :issue:`4095`, + :issue:`4244`, :issue:`4268`, :issue:`4372`) + +* Modified the :doc:`tox ` configuration to allow running tests + with any Python version, run Bandit_ and Flake8_ tests by default, and + enforce a minimum tox version programmatically (:issue:`4179`) + +* Cleaned up code (:issue:`3937`, :issue:`4208`, :issue:`4209`, + :issue:`4210`, :issue:`4212`, :issue:`4369`, :issue:`4376`, :issue:`4378`) + +.. _Bandit: https://bandit.readthedocs.io/ +.. _Flake8: https://flake8.pycqa.org/en/latest/ + + +.. _2-0-0-scheduler-queue-changes: + +Changes to scheduler queue classes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following changes may impact any custom queue classes of all types: + +* The ``push`` method no longer receives a second positional parameter + containing ``request.priority * -1``. If you need that value, get it + from the first positional parameter, ``request``, instead, or use + the new :meth:`~scrapy.core.scheduler.ScrapyPriorityQueue.priority` + method in :class:`scrapy.core.scheduler.ScrapyPriorityQueue` + subclasses. + +The following changes may impact custom priority queue classes: + +* In the ``__init__`` method or the ``from_crawler`` or ``from_settings`` + class methods: + + * The parameter that used to contain a factory function, + ``qfactory``, is now passed as a keyword parameter named + ``downstream_queue_cls``. + + * A new keyword parameter has been added: ``key``. It is a string + that is always an empty string for memory queues and indicates the + :setting:`JOB_DIR` value for disk queues. + + * The parameter for disk queues that contains data from the previous + crawl, ``startprios`` or ``slot_startprios``, is now passed as a + keyword parameter named ``startprios``. + + * The ``serialize`` parameter is no longer passed. The disk queue + class must take care of request serialization on its own before + writing to disk, using the + :func:`~scrapy.utils.reqser.request_to_dict` and + :func:`~scrapy.utils.reqser.request_from_dict` functions from the + :mod:`scrapy.utils.reqser` module. + +The following changes may impact custom disk and memory queue classes: + +* The signature of the ``__init__`` method is now + ``__init__(self, crawler, key)``. + +The following changes affect specifically the +:class:`~scrapy.core.scheduler.ScrapyPriorityQueue` and +:class:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue` classes from +:mod:`scrapy.core.scheduler` and may affect subclasses: + +* In the ``__init__`` method, most of the changes described above apply. + + ``__init__`` may still receive all parameters as positional parameters, + however: + + * ``downstream_queue_cls``, which replaced ``qfactory``, must be + instantiated differently. + + ``qfactory`` was instantiated with a priority value (integer). + + Instances of ``downstream_queue_cls`` should be created using + the new + :meth:`ScrapyPriorityQueue.qfactory ` + or + :meth:`DownloaderAwarePriorityQueue.pqfactory ` + methods. + + * The new ``key`` parameter displaced the ``startprios`` + parameter 1 position to the right. + +* The following class attributes have been added: + + * :attr:`~scrapy.core.scheduler.ScrapyPriorityQueue.crawler` + + * :attr:`~scrapy.core.scheduler.ScrapyPriorityQueue.downstream_queue_cls` + (details above) + + * :attr:`~scrapy.core.scheduler.ScrapyPriorityQueue.key` (details above) + +* The ``serialize`` attribute has been removed (details above) + +The following changes affect specifically the +:class:`~scrapy.core.scheduler.ScrapyPriorityQueue` class and may affect +subclasses: + +* A new :meth:`~scrapy.core.scheduler.ScrapyPriorityQueue.priority` + method has been added which, given a request, returns + ``request.priority * -1``. + + It is used in :meth:`~scrapy.core.scheduler.ScrapyPriorityQueue.push` + to make up for the removal of its ``priority`` parameter. + +* The ``spider`` attribute has been removed. Use + :attr:`crawler.spider ` + instead. + +The following changes affect specifically the +:class:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue` class and may +affect subclasses: + +* A new :attr:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue.pqueues` + attribute offers a mapping of downloader slot names to the + corresponding instances of + :attr:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue.downstream_queue_cls`. + +(:issue:`3884`) + .. _release-1.8.0: @@ -288,12 +732,12 @@ Backward-incompatible changes :class:`~scrapy.http.Request` objects instead of arbitrary Python data structures. -* An additional ``crawler`` parameter has been added to the ``__init__`` method - of the :class:`scrapy.core.scheduler.Scheduler` class. - Custom scheduler subclasses which don't accept arbitrary parameters in - their ``__init__`` method might break because of this change. +* An additional ``crawler`` parameter has been added to the ``__init__`` + method of the :class:`~scrapy.core.scheduler.Scheduler` class. Custom + scheduler subclasses which don't accept arbitrary parameters in their + ``__init__`` method might break because of this change. - For more information, refer to the documentation for the :setting:`SCHEDULER` setting. + For more information, see :setting:`SCHEDULER`. See also :ref:`1.7-deprecation-removals` below. diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst new file mode 100644 index 000000000..038a459fd --- /dev/null +++ b/docs/topics/asyncio.rst @@ -0,0 +1,28 @@ +======= +asyncio +======= + +.. versionadded:: 2.0 + +Scrapy has partial support :mod:`asyncio`. After you :ref:`install the asyncio +reactor `, you may use :mod:`asyncio` and +:mod:`asyncio`-powered libraries in any :doc:`coroutine `. + +.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy + versions may introduce related changes without a deprecation + period or warning. + +.. _install-asyncio: + +Installing the asyncio reactor +============================== + +To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to +``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``. + +If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to +install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` +reactor manually. You can do that using +:func:`~scrapy.utils.reactor.install_reactor`:: + + install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst new file mode 100644 index 000000000..487cf4c6c --- /dev/null +++ b/docs/topics/coroutines.rst @@ -0,0 +1,110 @@ +========== +Coroutines +========== + +.. versionadded:: 2.0 + +Scrapy has :ref:`partial support ` for the +:ref:`coroutine syntax `. + +.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy + versions may introduce related API and behavior changes without a + deprecation period or warning. + +.. _coroutine-support: + +Supported callables +=================== + +The following callables may be defined as coroutines using ``async def``, and +hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): + +- :class:`~scrapy.http.Request` callbacks. + + The following are known caveats of the current implementation that we aim + to address in future versions of Scrapy: + + - The callback output is not processed until the whole callback finishes. + + As a side effect, if the callback raises an exception, none of its + output is processed. + + - Because `asynchronous generators were introduced in Python 3.6`_, you + can only use ``yield`` if you are using Python 3.6 or later. + + If you need to output multiple items or requests and you are using + Python 3.5, return an iterable (e.g. a list) instead. + +- The :meth:`process_item` method of + :ref:`item pipelines `. + +- The + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request`, + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`, + and + :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception` + methods of + :ref:`downloader middlewares `. + +- :ref:`Signal handlers that support deferreds `. + +.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/ + +Usage +===== + +There are several use cases for coroutines in Scrapy. Code that would +return Deferreds when written for previous Scrapy versions, such as downloader +middlewares and signal handlers, can be rewritten to be shorter and cleaner:: + + class DbPipeline: + def _update_item(self, data, item): + item['field'] = data + return item + + def process_item(self, item, spider): + dfd = db.get_some_data(item['id']) + dfd.addCallback(self._update_item, item) + return dfd + +becomes:: + + class DbPipeline: + async def process_item(self, item, spider): + item['field'] = await db.get_some_data(item['id']) + return item + +Coroutines may be used to call asynchronous code. This includes other +coroutines, functions that return Deferreds and functions that return +`awaitable objects`_ such as :class:`~asyncio.Future`. This means you can use +many useful Python libraries providing such code:: + + class MySpider(Spider): + # ... + async def parse_with_deferred(self, response): + additional_response = await treq.get('https://additional.url') + additional_data = await treq.content(additional_response) + # ... use response and additional_data to yield items and requests + + async def parse_with_asyncio(self, response): + async with aiohttp.ClientSession() as session: + async with session.get('https://additional.url') as additional_response: + additional_data = await r.text() + # ... use response and additional_data to yield items and requests + +.. note:: Many libraries that use coroutines, such as `aio-libs`_, require the + :mod:`asyncio` loop and to use them you need to + :doc:`enable asyncio support in Scrapy`. + +Common use cases for asynchronous code include: + +* requesting data from websites, databases and other services (in callbacks, + pipelines and middlewares); +* storing data in databases (in pipelines and middlewares); +* delaying the spider initialization until some external event (in the + :signal:`spider_opened` handler); +* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see + :ref:`the screenshot pipeline example`). + +.. _aio-libs: https://github.com/aio-libs +.. _awaitable objects: https://docs.python.org/3/glossary.html#term-awaitable diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0297ef3a0..73648994d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -872,6 +872,10 @@ Default: ``[]`` Meta tags within these tags are ignored. +.. versionchanged:: 2.0 + The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from + ``['script', 'noscript']`` to ``[]``. + .. setting:: METAREFRESH_MAXDELAY METAREFRESH_MAXDELAY diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index b8d898022..d411e2eed 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -137,7 +137,7 @@ output examples, which assume you're exporting these two items:: BaseItemExporter ---------------- -.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0) +.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0, dont_fail=False) This is the (abstract) base class for all Item Exporters. It provides support for common features used by all (concrete) Item Exporters, such as @@ -148,6 +148,9 @@ BaseItemExporter populate their respective instance attributes: :attr:`fields_to_export`, :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`. + .. versionadded:: 2.0 + The *dont_fail* parameter. + .. method:: export_item(item) Exports the given item. This method must be implemented in subclasses. diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 1d94807a4..42f1cad90 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -236,6 +236,9 @@ supported URI schemes. This setting is required for enabling the feed exports. +.. versionchanged:: 2.0 + Added :class:`pathlib.Path` support. + .. setting:: FEED_FORMAT FEED_FORMAT diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 801d48fd5..98e2506e5 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -162,14 +162,16 @@ method and how to clean up the resources properly.:: .. _pymongo: https://api.mongodb.com/python/current/ +.. _ScreenshotPipeline: + Take screenshot of item ----------------------- This example demonstrates how to return a :class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method. It uses Splash_ to render screenshot of item url. Pipeline -makes request to locally running instance of Splash_. After request is downloaded -and Deferred callback fires, it saves item to a file and adds filename to an item. +makes request to locally running instance of Splash_. After request is downloaded, +it saves the screenshot to a file and adds filename to the item. :: @@ -184,15 +186,12 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite SPLASH_URL = "http://localhost:8050/render.png?url={}" - def process_item(self, item, spider): + async def process_item(self, item, spider): encoded_item_url = quote(item["url"]) screenshot_url = self.SPLASH_URL.format(encoded_item_url) request = scrapy.Request(screenshot_url) - dfd = spider.crawler.engine.download(request, spider) - dfd.addBoth(self.return_item, item) - return dfd + response = await spider.crawler.engine.download(request, spider) - def return_item(self, response, item): if response.status != 200: # Error happened, return item. return item diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index c34ba336b..58601824a 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -68,6 +68,9 @@ Cookies may expire. So, if you don't resume your spider quickly the requests scheduled may no longer work. This won't be an issue if you spider doesn't rely on cookies. + +.. _request-serialization: + Request serialization --------------------- diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 8c8019438..0162a331a 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -64,9 +64,13 @@ LxmlLinkExtractor :param deny_extensions: a single value or list of strings containing extensions that should be ignored when extracting links. - If not given, it will default to the - ``IGNORED_EXTENSIONS`` list defined in the - `scrapy.linkextractors`_ package. + If not given, it will default to + :data:`scrapy.linkextractors.IGNORED_EXTENSIONS`. + + .. versionchanged:: 2.0 + :data:`~scrapy.linkextractors.IGNORED_EXTENSIONS` now includes + ``7z``, ``7zip``, ``apk``, ``bz2``, ``cdr``, ``dmg``, ``ico``, + ``iso``, ``tar``, ``tar.gz``, ``webm``, and ``xz``. :type deny_extensions: list :param restrict_xpaths: is an XPath (or list of XPath's) which defines diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 9d5fccbbc..5f75ccbff 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -136,6 +136,9 @@ with the data to be parsed, and return a parsed value. So you can use any function as input or output processor. The only requirement is that they must accept one (and only one) positional argument, which will be an iterable. +.. versionchanged:: 2.0 + Processors no longer need to be methods. + .. note:: Both input and output processors must receive an iterable as their first argument. The output of those functions can be anything. The result of input processors will be appended to an internal list (in the Loader) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 67a0bfdba..cd84905c5 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -116,12 +116,6 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting:: Supported Storage ================= -File system is currently the only officially supported storage, but there are -also support for storing files in `Amazon S3`_ and `Google Cloud Storage`_. - -.. _Amazon S3: https://aws.amazon.com/s3/ -.. _Google Cloud Storage: https://cloud.google.com/storage/ - File system storage ------------------- @@ -147,9 +141,13 @@ Where: * ``full`` is a sub-directory to separate full images from thumbnails (if used). For more info see :ref:`topics-images-thumbnails`. +.. _media-pipeline-ftp: + FTP server storage ------------------ +.. versionadded:: 2.0 + :setting:`FILES_STORE` and :setting:`IMAGES_STORE` can point to an FTP server. Scrapy will automatically upload the files to the server. @@ -573,6 +571,8 @@ See here the methods that you can override in your custom Images Pipeline: By default, the :meth:`item_completed` method returns the item. +.. _media-pipeline-example: + Custom Images pipeline example ============================== diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index c4c2845c9..b2a60ff39 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -31,6 +31,8 @@ Request objects a :class:`Response`. :param url: the URL of this request + + If the URL is invalid, a :exc:`ValueError` exception is raised. :type url: string :param callback: the function that will be called with the response of this @@ -125,6 +127,10 @@ Request objects :exc:`~twisted.python.failure.Failure` as first parameter. For more information, see :ref:`topics-request-response-ref-errbacks` below. + + .. versionchanged:: 2.0 + The *callback* parameter is no longer required when the *errback* + parameter is specified. :type errback: callable :param flags: Flags sent to the request, can be used for logging or similar purposes. @@ -677,6 +683,8 @@ Response objects .. attribute:: Response.cb_kwargs + .. versionadded:: 2.0 + A shortcut to the :attr:`Request.cb_kwargs` attribute of the :attr:`Response.request` object (i.e. ``self.request.cb_kwargs``). diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5394147da..a70023efa 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -381,6 +381,8 @@ DNS in-memory cache size. DNS_RESOLVER ------------ +.. versionadded:: 2.0 + Default: ``'scrapy.resolver.CachingThreadedResolver'`` The class to be used to resolve DNS names. The default ``scrapy.resolver.CachingThreadedResolver`` @@ -1258,6 +1260,9 @@ does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`. SCRAPER_SLOT_MAX_ACTIVE_SIZE ---------------------------- + +.. versionadded:: 2.0 + Default: ``5_000_000`` Soft limit (in bytes) for response data being processed. @@ -1447,24 +1452,36 @@ in the ``project`` subdirectory. TWISTED_REACTOR --------------- +.. versionadded:: 2.0 + Default: ``None`` -Import path of a given Twisted reactor, for instance: -:class:`twisted.internet.asyncioreactor.AsyncioSelectorReactor`. +Import path of a given :mod:`~twisted.internet.reactor`. -Scrapy will install this reactor if no other is installed yet, such as when -the ``scrapy`` CLI program is invoked or when using the -:class:`~scrapy.crawler.CrawlerProcess` class. If you are using the -:class:`~scrapy.crawler.CrawlerRunner` class, you need to install the correct -reactor manually. An exception will be raised if the installation fails. +Scrapy will install this reactor if no other reactor is installed yet, such as +when the ``scrapy`` CLI program is invoked or when using the +:class:`~scrapy.crawler.CrawlerProcess` class. -The default value for this option is currently ``None``, which means that Scrapy -will not attempt to install any specific reactor, and the default one defined by -Twisted for the current platform will be used. This is to maintain backward -compatibility and avoid possible problems caused by using a non-default reactor. +If you are using the :class:`~scrapy.crawler.CrawlerRunner` class, you also +need to install the correct reactor manually. You can do that using +:func:`~scrapy.utils.reactor.install_reactor`: -For additional information, please see -:doc:`core/howto/choosing-reactor`. +.. autofunction:: scrapy.utils.reactor.install_reactor + +If a reactor is already installed, +:func:`~scrapy.utils.reactor.install_reactor` has no effect. + +:meth:`CrawlerRunner.__init__ ` raises +:exc:`Exception` if the installed reactor does not match the +:setting:`TWISTED_REACTOR` setting. + +The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which +means that Scrapy will not attempt to install any specific reactor, and the +default reactor defined by Twisted for the current platform will be used. This +is to maintain backward compatibility and avoid possible problems caused by +using a non-default reactor. + +For additional information, see :doc:`core/howto/choosing-reactor`. .. setting:: URLLENGTH_LIMIT diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index d3cfb0307..2def53848 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -46,6 +46,7 @@ Here is a simple example showing how you can catch signals and perform some acti def parse(self, response): pass +.. _signal-deferred: Deferred signal handlers ======================== @@ -301,6 +302,8 @@ request_left_downloader .. signal:: request_left_downloader .. function:: request_left_downloader(request, spider) + .. versionadded:: 2.0 + Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of failure. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index e0f33de66..89609db7d 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -420,6 +420,9 @@ Crawling rules It receives a :class:`Twisted Failure ` instance as first parameter. + .. versionadded:: 2.0 + The *errback* parameter. + CrawlSpider example ~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 119dd2f63..682cec161 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -129,6 +129,9 @@ class Response(object_ref): :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow` method which supports selectors in addition to absolute/relative URLs and Link objects. + + .. versionadded:: 2.0 + The *flags* parameter. """ if isinstance(url, Link): url = url.url @@ -157,6 +160,8 @@ class Response(object_ref): dont_filter=False, errback=None, cb_kwargs=None, flags=None): # type: (...) -> Generator[Request, None, None] """ + .. versionadded:: 2.0 + Return an iterable of :class:`~.Request` instances to follow all links in ``urls``. It accepts the same arguments as ``Request.__init__`` method, but elements of ``urls`` can be relative URLs or :class:`~scrapy.link.Link` objects, diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 194013642..14cec44a6 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -97,7 +97,11 @@ class LogFormatter(object): } def item_error(self, item, exception, response, spider): - """Logs a message when an item causes an error while it is passing through the item pipeline.""" + """Logs a message when an item causes an error while it is passing + through the item pipeline. + + .. versionadded:: 2.0 + """ return { 'level': logging.ERROR, 'msg': ITEMERRORMSG, @@ -107,7 +111,10 @@ class LogFormatter(object): } def spider_error(self, failure, request, response, spider): - """Logs an error message from a spider.""" + """Logs an error message from a spider. + + .. versionadded:: 2.0 + """ return { 'level': logging.ERROR, 'msg': SPIDERERRORMSG, @@ -118,7 +125,11 @@ class LogFormatter(object): } def download_error(self, failure, request, spider, errmsg=None): - """Logs a download error message from a spider (typically coming from the engine).""" + """Logs a download error message from a spider (typically coming from + the engine). + + .. versionadded:: 2.0 + """ args = {'request': request} if errmsg: msg = DOWNLOADERRORMSG_LONG diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 6513e06c9..17d6b2857 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -50,6 +50,8 @@ class CallLaterOnce(object): def install_reactor(reactor_path): + """Installs the :mod:`~twisted.internet.reactor` with the specified + import path.""" reactor_class = load_object(reactor_path) if reactor_class is asyncioreactor.AsyncioSelectorReactor: with suppress(error.ReactorAlreadyInstalledError): @@ -63,6 +65,9 @@ def install_reactor(reactor_path): def verify_installed_reactor(reactor_path): + """Raises :exc:`Exception` if the installed + :mod:`~twisted.internet.reactor` does not match the specified import + path.""" from twisted.internet import reactor reactor_class = load_object(reactor_path) if not isinstance(reactor, reactor_class): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index e93c668c5..3f8a7435c 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -325,7 +325,7 @@ with multiples lines @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncio_parse(self): - runner = CrawlerRunner({"ASYNCIO_REACTOR": True}) + runner = CrawlerRunner({"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"}) runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) with LogCapture() as log: yield runner.join() From a4dbb7754b999c8c6a5239bb3f58e951369e017e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 3 Mar 2020 09:13:00 +0100 Subject: [PATCH 106/281] =?UTF-8?q?Bump=20version:=201.8.0=20=E2=86=92=202?= =?UTF-8?q?.0.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 3 +-- scrapy/VERSION | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c9f1abea5..f347a0cd0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,8 +1,7 @@ [bumpversion] -current_version = 1.8.0 +current_version = 2.0.0 commit = True tag = True tag_name = {new_version} [bumpversion:file:scrapy/VERSION] - diff --git a/scrapy/VERSION b/scrapy/VERSION index 27f9cd322..227cea215 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.8.0 +2.0.0 From 1b591ff061f2b38bf328e1d2a4acd9643d45ad80 Mon Sep 17 00:00:00 2001 From: nyov Date: Fri, 28 Feb 2020 02:10:13 +0000 Subject: [PATCH 107/281] Obsolete deprecated settings Obsolete REDIRECT_MAX_METAREFRESH_DELAY which has been deprecated since Scrapy 0.18 Obsolete LOG_UNSERIALIZABLE_REQUESTS which has been deprecated since Scrapy 1.2.0 and is replaced by SCHEDULER_DEBUG --- scrapy/core/scheduler.py | 3 +-- scrapy/downloadermiddlewares/redirect.py | 3 +-- scrapy/settings/deprecated.py | 2 -- tests/test_scheduler.py | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index e184ed50e..c96b9b719 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -66,8 +66,7 @@ class Scheduler(object): dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) - logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', - settings.getbool('SCHEDULER_DEBUG')) + logunser = settings.getbool('SCHEDULER_DEBUG') return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser, stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass, crawler=crawler) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 77cb5aa94..08cff8a55 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -93,8 +93,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): def __init__(self, settings): super(MetaRefreshMiddleware, self).__init__(settings) self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS') - self._maxdelay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY', - settings.getint('METAREFRESH_MAXDELAY')) + self._maxdelay = settings.getint('METAREFRESH_MAXDELAY') def process_response(self, request, response, spider): if request.meta.get('dont_redirect', False) or request.method == 'HEAD' or \ diff --git a/scrapy/settings/deprecated.py b/scrapy/settings/deprecated.py index 1211908df..f6f878725 100644 --- a/scrapy/settings/deprecated.py +++ b/scrapy/settings/deprecated.py @@ -11,8 +11,6 @@ DEPRECATED_SETTINGS = [ ('SQLITE_DB', 'no longer supported'), ('AUTOTHROTTLE_MIN_DOWNLOAD_DELAY', 'use DOWNLOAD_DELAY instead'), ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'), - ('REDIRECT_MAX_METAREFRESH_DELAY', 'use METAREFRESH_MAXDELAY instead'), - ('LOG_UNSERIALIZABLE_REQUESTS', 'use SCHEDULER_DEBUG instead'), ] diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index e0e3600e5..13c297084 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -46,7 +46,7 @@ class MockCrawler(Crawler): def __init__(self, priority_queue_cls, jobdir): settings = dict( - LOG_UNSERIALIZABLE_REQUESTS=False, + SCHEDULER_DEBUG=False, SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue', SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue', SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, From 64002255554aa2aa79863b657e5d1674d72b228d Mon Sep 17 00:00:00 2001 From: nyov Date: Fri, 28 Feb 2020 00:06:11 +0000 Subject: [PATCH 108/281] Drop horribly outdated deb package build files --- Makefile.buildbot | 24 -------------------- debian/changelog | 5 ----- debian/compat | 1 - debian/control | 20 ----------------- debian/copyright | 40 --------------------------------- debian/pyversions | 1 - debian/rules | 5 ----- debian/scrapy.docs | 2 -- debian/scrapy.install | 2 -- debian/scrapy.lintian-overrides | 1 - debian/scrapy.manpages | 1 - 11 files changed, 102 deletions(-) delete mode 100644 Makefile.buildbot delete mode 100644 debian/changelog delete mode 100644 debian/compat delete mode 100644 debian/control delete mode 100644 debian/copyright delete mode 100644 debian/pyversions delete mode 100755 debian/rules delete mode 100644 debian/scrapy.docs delete mode 100644 debian/scrapy.install delete mode 100644 debian/scrapy.lintian-overrides delete mode 100644 debian/scrapy.manpages diff --git a/Makefile.buildbot b/Makefile.buildbot deleted file mode 100644 index 775538259..000000000 --- a/Makefile.buildbot +++ /dev/null @@ -1,24 +0,0 @@ -TRIAL := $(shell which trial) -BRANCH := $(shell git rev-parse --abbrev-ref HEAD) -export PYTHONPATH=$(PWD) - -test: - coverage run --branch $(TRIAL) --reporter=text tests - rm -rf htmlcov && coverage html - -s3cmd sync -P htmlcov/ s3://static.scrapy.org/coverage-scrapy-$(BRANCH)/ - -build: - git describe --tags --match '[0-9]*' |sed 's/-/.post/;s/-g/+g/' >scrapy/VERSION - debchange -m -D unstable --force-distribution -v \ - $$(python setup.py --version |sed -r 's/([0-9]+.[0-9]+.[0-9]+)(a|b|rc|dev)([0-9]*)/\1~\2\3/')-$$(date +%s) \ - "Automatic build" - debuild -us -uc -b - -clean: - git checkout debian scrapy/VERSION - git clean -dfq - -pypi: - umask 0022 && chmod -R a+rX . && python setup.py sdist upload - -.PHONY: clean test build diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index dde97f9e3..000000000 --- a/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -scrapy (0.11) unstable; urgency=low - - * Initial release. - - -- Scrapinghub Team Thu, 10 Jun 2010 17:24:02 -0300 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index 7f8f011eb..000000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -7 diff --git a/debian/control b/debian/control deleted file mode 100644 index 2cc8eedf4..000000000 --- a/debian/control +++ /dev/null @@ -1,20 +0,0 @@ -Source: scrapy -Section: python -Priority: optional -Maintainer: Scrapinghub Team -Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2) -Standards-Version: 3.8.4 -Homepage: https://scrapy.org/ - -Package: scrapy -Architecture: all -Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl, - python-w3lib (>= 1.8.0), python-queuelib, python-cssselect (>= 0.9), python-six (>=1.5.2) -Recommends: python-setuptools -Conflicts: python-scrapy, scrapy-0.25 -Provides: python-scrapy, scrapy-0.25 -Description: Python web crawling and web scraping framework - Scrapy is a fast high-level web crawling and web scraping framework, - used to crawl websites and extract structured data from their pages. - It can be used for a wide range of purposes, from data mining to - monitoring and automated testing. diff --git a/debian/copyright b/debian/copyright deleted file mode 100644 index c1bf47565..000000000 --- a/debian/copyright +++ /dev/null @@ -1,40 +0,0 @@ -This package was debianized by the Scrapinghub team . - -It was downloaded from https://scrapy.org - -Upstream Author: Scrapy Developers - -Copyright: 2007-2013 Scrapy Developers - -License: bsd - -Copyright (c) Scrapy developers. -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of Scrapy nor the names of its contributors may be used - to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The Debian packaging is (C) 2010-2013, Scrapinghub and -is licensed under the BSD, see `/usr/share/common-licenses/BSD'. diff --git a/debian/pyversions b/debian/pyversions deleted file mode 100644 index 1effb0034..000000000 --- a/debian/pyversions +++ /dev/null @@ -1 +0,0 @@ -2.7 diff --git a/debian/rules b/debian/rules deleted file mode 100755 index b8796e6e3..000000000 --- a/debian/rules +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/make -f -# -*- makefile -*- - -%: - dh $@ diff --git a/debian/scrapy.docs b/debian/scrapy.docs deleted file mode 100644 index c19ffba4d..000000000 --- a/debian/scrapy.docs +++ /dev/null @@ -1,2 +0,0 @@ -README.rst -AUTHORS diff --git a/debian/scrapy.install b/debian/scrapy.install deleted file mode 100644 index c288ebed3..000000000 --- a/debian/scrapy.install +++ /dev/null @@ -1,2 +0,0 @@ -extras/scrapy_bash_completion etc/bash_completion.d/ -extras/scrapy_zsh_completion /usr/share/zsh/vendor-completions/_scrapy diff --git a/debian/scrapy.lintian-overrides b/debian/scrapy.lintian-overrides deleted file mode 100644 index b5de7f67d..000000000 --- a/debian/scrapy.lintian-overrides +++ /dev/null @@ -1 +0,0 @@ -new-package-should-close-itp-bug diff --git a/debian/scrapy.manpages b/debian/scrapy.manpages deleted file mode 100644 index 4818e9c92..000000000 --- a/debian/scrapy.manpages +++ /dev/null @@ -1 +0,0 @@ -extras/scrapy.1 From 6c35baae25517ed942c72d14d93363a389e5a9d3 Mon Sep 17 00:00:00 2001 From: nyov Date: Wed, 4 Mar 2020 00:40:11 +0000 Subject: [PATCH 109/281] Remove deprecated SiteNode and MultiValueDict classes --- scrapy/utils/datatypes.py | 174 -------------------------------------- 1 file changed, 174 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index b07f995cf..175f92d77 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -6,183 +6,9 @@ This module must not depend on any module outside the Standard Library. """ import collections -import copy -import warnings import weakref from collections.abc import Mapping -from scrapy.exceptions import ScrapyDeprecationWarning - - -class MultiValueDictKeyError(KeyError): - def __init__(self, *args, **kwargs): - warnings.warn( - "scrapy.utils.datatypes.MultiValueDictKeyError is deprecated " - "and will be removed in future releases.", - category=ScrapyDeprecationWarning, - stacklevel=2 - ) - super(MultiValueDictKeyError, self).__init__(*args, **kwargs) - - -class MultiValueDict(dict): - """ - A subclass of dictionary customized to handle multiple values for the same key. - - >>> d = MultiValueDict({'name': ['Adrian', 'Simon'], 'position': ['Developer']}) - >>> d['name'] - 'Simon' - >>> d.getlist('name') - ['Adrian', 'Simon'] - >>> d.get('lastname', 'nonexistent') - 'nonexistent' - >>> d.setlist('lastname', ['Holovaty', 'Willison']) - - This class exists to solve the irritating problem raised by cgi.parse_qs, - which returns a list for every key, even though most Web forms submit - single name-value pairs. - """ - def __init__(self, key_to_list_mapping=()): - warnings.warn("scrapy.utils.datatypes.MultiValueDict is deprecated " - "and will be removed in future releases.", - category=ScrapyDeprecationWarning, - stacklevel=2) - dict.__init__(self, key_to_list_mapping) - - def __repr__(self): - return "<%s: %s>" % (self.__class__.__name__, dict.__repr__(self)) - - def __getitem__(self, key): - """ - Returns the last data value for this key, or [] if it's an empty list; - raises KeyError if not found. - """ - try: - list_ = dict.__getitem__(self, key) - except KeyError: - raise MultiValueDictKeyError("Key %r not found in %r" % (key, self)) - try: - return list_[-1] - except IndexError: - return [] - - def __setitem__(self, key, value): - dict.__setitem__(self, key, [value]) - - def __copy__(self): - return self.__class__(dict.items(self)) - - def __deepcopy__(self, memo=None): - if memo is None: - memo = {} - result = self.__class__() - memo[id(self)] = result - for key, value in dict.items(self): - dict.__setitem__(result, copy.deepcopy(key, memo), copy.deepcopy(value, memo)) - return result - - def get(self, key, default=None): - "Returns the default value if the requested data doesn't exist" - try: - val = self[key] - except KeyError: - return default - if val == []: - return default - return val - - def getlist(self, key): - "Returns an empty list if the requested data doesn't exist" - try: - return dict.__getitem__(self, key) - except KeyError: - return [] - - def setlist(self, key, list_): - dict.__setitem__(self, key, list_) - - def setdefault(self, key, default=None): - if key not in self: - self[key] = default - return self[key] - - def setlistdefault(self, key, default_list=()): - if key not in self: - self.setlist(key, default_list) - return self.getlist(key) - - def appendlist(self, key, value): - "Appends an item to the internal list associated with key" - self.setlistdefault(key, []) - dict.__setitem__(self, key, self.getlist(key) + [value]) - - def items(self): - """ - Returns a list of (key, value) pairs, where value is the last item in - the list associated with the key. - """ - return [(key, self[key]) for key in self.keys()] - - def lists(self): - "Returns a list of (key, list) pairs." - return dict.items(self) - - def values(self): - "Returns a list of the last value on every key list." - return [self[key] for key in self.keys()] - - def copy(self): - "Returns a copy of this object." - return self.__deepcopy__() - - def update(self, *args, **kwargs): - "update() extends rather than replaces existing key lists. Also accepts keyword args." - if len(args) > 1: - raise TypeError("update expected at most 1 arguments, got %d" % len(args)) - if args: - other_dict = args[0] - if isinstance(other_dict, MultiValueDict): - for key, value_list in other_dict.lists(): - self.setlistdefault(key, []).extend(value_list) - else: - try: - for key, value in other_dict.items(): - self.setlistdefault(key, []).append(value) - except TypeError: - raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary") - for key, value in kwargs.items(): - self.setlistdefault(key, []).append(value) - - -class SiteNode(object): - """Class to represent a site node (page, image or any other file)""" - - def __init__(self, url): - warnings.warn( - "scrapy.utils.datatypes.SiteNode is deprecated " - "and will be removed in future releases.", - category=ScrapyDeprecationWarning, - stacklevel=2 - ) - - self.url = url - self.itemnames = [] - self.children = [] - self.parent = None - - def add_child(self, node): - self.children.append(node) - node.parent = self - - def to_string(self, level=0): - s = "%s%s\n" % (' ' * level, self.url) - if self.itemnames: - for n in self.itemnames: - s += "%sScraped: %s\n" % (' ' * (level + 1), n) - for node in self.children: - s += node.to_string(level + 1) - return s - class CaselessDict(dict): From b1566a696217244d2d506ce8962418943d0b1edc Mon Sep 17 00:00:00 2001 From: nyov Date: Mon, 2 Mar 2020 23:01:26 +0000 Subject: [PATCH 110/281] Remove deprecated Crawler.spiders property Deprecated since 419026615 (2014, Scrapy 0.25) --- scrapy/crawler.py | 45 ++++++++++++++++--------------------------- tests/test_crawler.py | 15 --------------- 2 files changed, 17 insertions(+), 43 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 49b8e4511..77a13d0c1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -68,17 +68,6 @@ class Crawler: self.spider = None self.engine = None - @property - def spiders(self): - if not hasattr(self, '_spiders'): - warnings.warn("Crawler.spiders is deprecated, use " - "CrawlerRunner.spider_loader or instantiate " - "scrapy.spiderloader.SpiderLoader with your " - "settings.", - category=ScrapyDeprecationWarning, stacklevel=2) - self._spiders = _get_spider_loader(self.settings.frozencopy()) - return self._spiders - @defer.inlineCallbacks def crawl(self, *args, **kwargs): assert not self.crawling, "Crawling already taking place" @@ -130,11 +119,27 @@ class CrawlerRunner: ":meth:`crawl` and managed by this class." ) + @staticmethod + def _get_spider_loader(settings): + """ Get SpiderLoader instance from settings """ + cls_path = settings.get('SPIDER_LOADER_CLASS') + loader_cls = load_object(cls_path) + try: + verifyClass(ISpiderLoader, loader_cls) + except DoesNotImplement: + warnings.warn( + 'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does ' + 'not fully implement scrapy.interfaces.ISpiderLoader interface. ' + 'Please add all missing methods to avoid unexpected runtime errors.', + category=ScrapyDeprecationWarning, stacklevel=2 + ) + return loader_cls.from_settings(settings.frozencopy()) + def __init__(self, settings=None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings - self.spider_loader = _get_spider_loader(settings) + self.spider_loader = self._get_spider_loader(settings) self._crawlers = set() self._active = set() self.bootstrap_failed = False @@ -327,19 +332,3 @@ class CrawlerProcess(CrawlerRunner): if self.settings.get("TWISTED_REACTOR"): install_reactor(self.settings["TWISTED_REACTOR"]) super()._handle_twisted_reactor() - - -def _get_spider_loader(settings): - """ Get SpiderLoader instance from settings """ - cls_path = settings.get('SPIDER_LOADER_CLASS') - loader_cls = load_object(cls_path) - try: - verifyClass(ISpiderLoader, loader_cls) - except DoesNotImplement: - warnings.warn( - 'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does ' - 'not fully implement scrapy.interfaces.ISpiderLoader interface. ' - 'Please add all missing methods to avoid unexpected runtime errors.', - category=ScrapyDeprecationWarning, stacklevel=2 - ) - return loader_cls.from_settings(settings.frozencopy()) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 7bd76601d..37a069611 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -33,21 +33,6 @@ class CrawlerTestCase(BaseCrawlerTest): def setUp(self): self.crawler = Crawler(DefaultSpider, Settings()) - def test_deprecated_attribute_spiders(self): - with warnings.catch_warnings(record=True) as w: - spiders = self.crawler.spiders - self.assertEqual(len(w), 1) - self.assertIn("Crawler.spiders", str(w[0].message)) - sl_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS']) - self.assertIsInstance(spiders, sl_cls) - - self.crawler.spiders - is_one_warning = len(w) == 1 - if not is_one_warning: - for warning in w: - print(warning) - self.assertTrue(is_one_warning, "Warn deprecated access only once") - def test_populate_spidercls_settings(self): spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'} project_settings = {'TEST1': 'project', 'TEST3': 'project'} From ada37c5409047291ee5852fb2220e8e256424402 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 6 Jul 2019 22:37:31 -0300 Subject: [PATCH 111/281] Export to multiple formats in a single crawl --- docs/topics/feed-exports.rst | 85 ++++++--- scrapy/commands/crawl.py | 23 +-- scrapy/commands/runspider.py | 20 +- scrapy/extensions/feedexport.py | 191 +++++++++++-------- scrapy/settings/default_settings.py | 3 +- scrapy/utils/conf.py | 68 ++++++- tests/test_commands.py | 32 +++- tests/test_feedexport.py | 273 ++++++++++++++++++++-------- tests/test_utils_conf.py | 50 ++++- 9 files changed, 511 insertions(+), 234 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 42f1cad90..6d6ba33c9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -12,7 +12,7 @@ generating an "export file" with the scraped data (commonly called "export feed") to be consumed by other systems. Scrapy provides this functionality out of the box with the Feed Exports, which -allows you to generate a feed with the scraped items, using multiple +allows you to generate feeds with the scraped items, using multiple serialization formats and storage backends. .. _topics-feed-format: @@ -36,7 +36,7 @@ But you can also extend the supported format through the JSON ---- - * :setting:`FEED_FORMAT`: ``json`` + * Value for the ``format`` key in the :setting:`FEEDS` setting: ``json`` * Exporter used: :class:`~scrapy.exporters.JsonItemExporter` * See :ref:`this warning ` if you're using JSON with large feeds. @@ -46,7 +46,7 @@ JSON JSON lines ---------- - * :setting:`FEED_FORMAT`: ``jsonlines`` + * Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines`` * Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter` .. _topics-feed-format-csv: @@ -54,7 +54,7 @@ JSON lines CSV --- - * :setting:`FEED_FORMAT`: ``csv`` + * Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv`` * Exporter used: :class:`~scrapy.exporters.CsvItemExporter` * To specify columns to export and their order use :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this @@ -66,7 +66,7 @@ CSV XML --- - * :setting:`FEED_FORMAT`: ``xml`` + * Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml`` * Exporter used: :class:`~scrapy.exporters.XmlItemExporter` .. _topics-feed-format-pickle: @@ -74,7 +74,7 @@ XML Pickle ------ - * :setting:`FEED_FORMAT`: ``pickle`` + * Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle`` * Exporter used: :class:`~scrapy.exporters.PickleItemExporter` .. _topics-feed-format-marshal: @@ -82,7 +82,7 @@ Pickle Marshal ------- - * :setting:`FEED_FORMAT`: ``marshal`` + * Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal`` * Exporter used: :class:`~scrapy.exporters.MarshalItemExporter` @@ -91,8 +91,8 @@ Marshal Storages ======== -When using the feed exports you define where to store the feed using a URI_ -(through the :setting:`FEED_URI` setting). The feed exports supports multiple +When using the feed exports you define where to store the feed using one or multiple URIs_ +(through the :setting:`FEEDS` setting). The feed exports supports multiple storage backend types which are defined by the URI scheme. The storages backends supported out of the box are: @@ -211,41 +211,66 @@ Settings These are the settings used for configuring the feed exports: - * :setting:`FEED_URI` (mandatory) - * :setting:`FEED_FORMAT` + * :setting:`FEEDS` (mandatory) + * :setting:`FEED_EXPORT_ENCODING` + * :setting:`FEED_STORE_EMPTY` + * :setting:`FEED_EXPORT_FIELDS` + * :setting:`FEED_EXPORT_INDENT` * :setting:`FEED_STORAGES` * :setting:`FEED_STORAGE_FTP_ACTIVE` * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` - * :setting:`FEED_STORE_EMPTY` - * :setting:`FEED_EXPORT_ENCODING` - * :setting:`FEED_EXPORT_FIELDS` - * :setting:`FEED_EXPORT_INDENT` .. currentmodule:: scrapy.extensions.feedexport -.. setting:: FEED_URI +.. setting:: FEEDS -FEED_URI --------- +FEEDS +----- -Default: ``None`` +.. versionadded:: 2.1 -The URI of the export feed. See :ref:`topics-feed-storage-backends` for -supported URI schemes. +Default: ``{}`` -This setting is required for enabling the feed exports. +A dictionary in which every key is a feed URI (or a :class:`pathlib.Path` +object) and each value is a nested dictionary containing configuration +parameters for the specific feed. +This setting is required for enabling the feed export feature. -.. versionchanged:: 2.0 - Added :class:`pathlib.Path` support. +See :ref:`topics-feed-storage-backends` for supported URI schemes. -.. setting:: FEED_FORMAT +For instance:: -FEED_FORMAT ------------ + { + 'items.json': { + 'format': 'json', + 'encoding': 'utf8', + 'store_empty': False, + 'fields': None, + 'indent': 4, + }, + 'items.xml': { + 'format': 'xml', + 'fields': ['name', 'price'], + 'encoding': 'latin1', + 'indent': 8, + }, + pathlib.Path('items.csv'): { + 'format': 'csv', + 'fields': ['price', 'name'], + }, + } -The serialization format to be used for the feed. See -:ref:`topics-feed-format` for possible values. +The following is a list of the accepted keys and the setting that is used +as a fallback value if that key is not provided for a specific feed definition. + +* ``format``: the serialization format to be used for the feed. + See :ref:`topics-feed-format` for possible values. + Mandatory, no fallback setting +* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING` +* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS` +* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT` +* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY` .. setting:: FEED_EXPORT_ENCODING @@ -400,7 +425,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter 'csv': None, } -.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier +.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 7b417e2eb..4b2f9484b 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,7 +1,5 @@ -import os from scrapy.commands import ScrapyCommand -from scrapy.utils.conf import arglist_to_dict -from scrapy.utils.python import without_none_values +from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli from scrapy.exceptions import UsageError @@ -19,7 +17,7 @@ class Command(ScrapyCommand): ScrapyCommand.add_options(self, parser) parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", + parser.add_option("-o", "--output", metavar="FILE", action="append", help="dump scraped items into FILE (use - for stdout)") parser.add_option("-t", "--output-format", metavar="FORMAT", help="format to use for dumping items with -o") @@ -31,21 +29,8 @@ class Command(ScrapyCommand): except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output: - if opts.output == '-': - self.settings.set('FEED_URI', 'stdout:', priority='cmdline') - else: - self.settings.set('FEED_URI', opts.output, priority='cmdline') - feed_exporters = without_none_values( - self.settings.getwithbase('FEED_EXPORTERS')) - valid_output_formats = feed_exporters.keys() - if not opts.output_format: - opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") - if opts.output_format not in valid_output_formats: - raise UsageError("Unrecognized output format '%s', set one" - " using the '-t' switch or as a file extension" - " from the supported list %s" % (opts.output_format, - tuple(valid_output_formats))) - self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline') + feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) + self.settings.set('FEEDS', feeds, priority='cmdline') def run(self, args, opts): if len(args) < 1: diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 57d8471ca..62510609a 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -5,8 +5,7 @@ from importlib import import_module from scrapy.utils.spider import iter_spider_classes from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError -from scrapy.utils.conf import arglist_to_dict -from scrapy.utils.python import without_none_values +from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli def _import_file(filepath): @@ -43,7 +42,7 @@ class Command(ScrapyCommand): ScrapyCommand.add_options(self, parser) parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", help="set spider argument (may be repeated)") - parser.add_option("-o", "--output", metavar="FILE", + parser.add_option("-o", "--output", metavar="FILE", action="append", help="dump scraped items into FILE (use - for stdout)") parser.add_option("-t", "--output-format", metavar="FORMAT", help="format to use for dumping items with -o") @@ -55,19 +54,8 @@ class Command(ScrapyCommand): except ValueError: raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) if opts.output: - if opts.output == '-': - self.settings.set('FEED_URI', 'stdout:', priority='cmdline') - else: - self.settings.set('FEED_URI', opts.output, priority='cmdline') - feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS')) - if not opts.output_format: - opts.output_format = os.path.splitext(opts.output)[1].replace(".", "") - if opts.output_format not in feed_exporters: - raise UsageError("Unrecognized output format '%s', set one" - " using the '-t' switch or as a file extension" - " from the supported list %s" % (opts.output_format, - tuple(feed_exporters))) - self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline') + feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format) + self.settings.set('FEEDS', feeds, priority='cmdline') def run(self, args, opts): if len(args) != 1: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index f1b101780..108b6d35c 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -4,24 +4,27 @@ Feed Exports extension See documentation in docs/topics/feed-exports.rst """ +import logging import os import sys -import logging -from tempfile import NamedTemporaryFile +import warnings from datetime import datetime -from urllib.parse import urlparse, unquote +from tempfile import NamedTemporaryFile +from urllib.parse import unquote, urlparse -from zope.interface import Interface, implementer from twisted.internet import defer, threads from w3lib.url import file_uri_to_path +from zope.interface import implementer, Interface from scrapy import signals -from scrapy.utils.ftp import ftp_store_file -from scrapy.exceptions import NotConfigured -from scrapy.utils.misc import create_instance, load_object -from scrapy.utils.log import failure_to_exc_info -from scrapy.utils.python import without_none_values +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore +from scrapy.utils.conf import feed_complete_default_values_from_settings +from scrapy.utils.ftp import ftp_store_file +from scrapy.utils.log import failure_to_exc_info +from scrapy.utils.misc import create_instance, load_object +from scrapy.utils.python import without_none_values + logger = logging.getLogger(__name__) @@ -98,8 +101,6 @@ class S3FeedStorage(BlockingFeedStorage): from scrapy.utils.project import get_project_settings settings = get_project_settings() if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings: - import warnings - from scrapy.exceptions import ScrapyDeprecationWarning warnings.warn( "Initialising `scrapy.extensions.feedexport.S3FeedStorage` " "without AWS keys is deprecated. Please supply credentials or " @@ -178,88 +179,117 @@ class FTPFeedStorage(BlockingFeedStorage): ) -class SpiderSlot(object): - def __init__(self, file, exporter, storage, uri): +class _FeedSlot(object): + def __init__(self, file, exporter, storage, uri, format, store_empty): self.file = file self.exporter = exporter self.storage = storage + # feed params self.uri = uri + self.format = format + self.store_empty = store_empty + # flags self.itemcount = 0 + self._exporting = False + + def start_exporting(self): + if not self._exporting: + self.exporter.start_exporting() + self._exporting = True + + def finish_exporting(self): + if self._exporting: + self.exporter.finish_exporting() + self._exporting = False class FeedExporter(object): - def __init__(self, settings): - self.settings = settings - if not settings['FEED_URI']: - raise NotConfigured - self.urifmt = str(settings['FEED_URI']) - self.format = settings['FEED_FORMAT'].lower() - self.export_encoding = settings['FEED_EXPORT_ENCODING'] - self.storages = self._load_components('FEED_STORAGES') - self.exporters = self._load_components('FEED_EXPORTERS') - if not self._storage_supported(self.urifmt): - raise NotConfigured - if not self._exporter_supported(self.format): - raise NotConfigured - self.store_empty = settings.getbool('FEED_STORE_EMPTY') - self._exporting = False - self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None - self.indent = None - if settings.get('FEED_EXPORT_INDENT') is not None: - self.indent = settings.getint('FEED_EXPORT_INDENT') - uripar = settings['FEED_URI_PARAMS'] - self._uripar = load_object(uripar) if uripar else lambda x, y: None - @classmethod def from_crawler(cls, crawler): - o = cls(crawler.settings) - o.crawler = crawler - crawler.signals.connect(o.open_spider, signals.spider_opened) - crawler.signals.connect(o.close_spider, signals.spider_closed) - crawler.signals.connect(o.item_scraped, signals.item_scraped) - return o + exporter = cls(crawler) + crawler.signals.connect(exporter.open_spider, signals.spider_opened) + crawler.signals.connect(exporter.close_spider, signals.spider_closed) + crawler.signals.connect(exporter.item_scraped, signals.item_scraped) + return exporter + + def __init__(self, crawler): + self.crawler = crawler + self.settings = crawler.settings + self.feeds = {} + self.slots = [] + + if not self.settings['FEEDS'] and not self.settings['FEED_URI']: + raise NotConfigured + + # Begin: Backward compatibility for FEED_URI and FEED_FORMAT settings + if self.settings['FEED_URI']: + warnings.warn( + 'The `FEED_URI` and `FEED_FORMAT` settings have been deprecated in favor of ' + 'the `FEEDS` setting. Please see the `FEEDS` setting docs for more details', + category=ScrapyDeprecationWarning, stacklevel=2, + ) + uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects + feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')} + self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings) + # End: Backward compatibility for FEED_URI and FEED_FORMAT settings + + # 'FEEDS' setting takes precedence over 'FEED_URI' + for uri, feed in self.settings.getdict('FEEDS').items(): + uri = str(uri) # handle pathlib.Path objects + self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings) + + self.storages = self._load_components('FEED_STORAGES') + self.exporters = self._load_components('FEED_EXPORTERS') + for uri, feed in self.feeds.items(): + if not self._storage_supported(uri): + raise NotConfigured + if not self._exporter_supported(feed['format']): + raise NotConfigured def open_spider(self, spider): - uri = self.urifmt % self._get_uri_params(spider) - storage = self._get_storage(uri) - file = storage.open(spider) - exporter = self._get_exporter(file, fields_to_export=self.export_fields, - encoding=self.export_encoding, indent=self.indent) - if self.store_empty: - exporter.start_exporting() - self._exporting = True - self.slot = SpiderSlot(file, exporter, storage, uri) + for uri, feed in self.feeds.items(): + uri = uri % self._get_uri_params(spider, feed['uri_params']) + storage = self._get_storage(uri) + file = storage.open(spider) + exporter = self._get_exporter( + file=file, + format=feed['format'], + fields_to_export=feed['fields'], + encoding=feed['encoding'], + indent=feed['indent'], + ) + slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty']) + self.slots.append(slot) + if slot.store_empty: + slot.start_exporting() def close_spider(self, spider): - slot = self.slot - if not slot.itemcount and not self.store_empty: - # We need to call slot.storage.store nonetheless to get the file - # properly closed. - return defer.maybeDeferred(slot.storage.store, slot.file) - if self._exporting: - slot.exporter.finish_exporting() - self._exporting = False - logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" - log_args = {'format': self.format, - 'itemcount': slot.itemcount, - 'uri': slot.uri} - d = defer.maybeDeferred(slot.storage.store, slot.file) - d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args, - extra={'spider': spider})) - d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args, - exc_info=failure_to_exc_info(f), - extra={'spider': spider})) - return d + deferred_list = [] + for slot in self.slots: + if not slot.itemcount and not slot.store_empty: + # We need to call slot.storage.store nonetheless to get the file + # properly closed. + return defer.maybeDeferred(slot.storage.store, slot.file) + slot.finish_exporting() + logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s" + log_args = {'format': slot.format, + 'itemcount': slot.itemcount, + 'uri': slot.uri} + d = defer.maybeDeferred(slot.storage.store, slot.file) + d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args, + extra={'spider': spider})) + d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args, + exc_info=failure_to_exc_info(f), + extra={'spider': spider})) + deferred_list.append(d) + return defer.DeferredList(deferred_list) if deferred_list else None def item_scraped(self, item, spider): - slot = self.slot - if not self._exporting: - slot.exporter.start_exporting() - self._exporting = True - slot.exporter.export_item(item) - slot.itemcount += 1 - return item + for slot in self.slots: + slot.start_exporting() + slot.exporter.export_item(item) + slot.itemcount += 1 def _load_components(self, setting_prefix): conf = without_none_values(self.settings.getwithbase(setting_prefix)) @@ -295,17 +325,18 @@ class FeedExporter(object): objcls, self.settings, getattr(self, 'crawler', None), *args, **kwargs) - def _get_exporter(self, *args, **kwargs): - return self._get_instance(self.exporters[self.format], *args, **kwargs) + def _get_exporter(self, file, format, *args, **kwargs): + return self._get_instance(self.exporters[format], file, *args, **kwargs) def _get_storage(self, uri): return self._get_instance(self.storages[urlparse(uri).scheme], uri) - def _get_uri_params(self, spider): + def _get_uri_params(self, spider, uri_params): params = {} for k in dir(spider): params[k] = getattr(spider, k) ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-') params['time'] = ts - self._uripar(params, spider) + uripar_function = load_object(uri_params) if uri_params else lambda x, y: None + uripar_function(params, spider) return params diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index f8a0457ce..077317c81 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -133,9 +133,8 @@ EXTENSIONS_BASE = { } FEED_TEMPDIR = None -FEED_URI = None +FEEDS = {} FEED_URI_PARAMS = None # a function to extend uri arguments -FEED_FORMAT = 'jsonlines' FEED_STORE_EMPTY = False FEED_EXPORT_ENCODING = None FEED_EXPORT_FIELDS = None diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 23306ca28..e01027491 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,9 +1,12 @@ +import numbers import os import sys -import numbers +import warnings from configparser import ConfigParser from operator import itemgetter +from scrapy.exceptions import ScrapyDeprecationWarning, UsageError + from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values @@ -106,3 +109,66 @@ def get_sources(use_closest=True): if use_closest: sources.append(closest_scrapy_cfg()) return sources + + +def feed_complete_default_values_from_settings(feed, settings): + out = feed.copy() + if 'encoding' not in out: + out['encoding'] = settings['FEED_EXPORT_ENCODING'] + if 'fields' not in out: + out['fields'] = settings.getlist('FEED_EXPORT_FIELDS') or None + if 'indent' not in out: + out['indent'] = None if settings['FEED_EXPORT_INDENT'] is None else settings.getint('FEED_EXPORT_INDENT') + if 'store_empty' not in out: + out['store_empty'] = settings.getbool('FEED_STORE_EMPTY') + if 'uri_params' not in out: + out['uri_params'] = settings['FEED_URI_PARAMS'] + return out + + +def feed_process_params_from_cli(settings, output, output_format=None): + """ + Receives feed export params (from the 'crawl' or 'runspider' commands), + checks for inconsistencies in their quantities and returns a dictionary + suitable to be used as the FEEDS setting. + """ + valid_output_formats = without_none_values( + settings.getwithbase('FEED_EXPORTERS') + ).keys() + + def check_valid_format(output_format): + if output_format not in valid_output_formats: + raise UsageError("Unrecognized output format '%s', set one after a" + " colon using the -o option (i.e. -o :)" + " or as a file extension, from the supported list %s" % + (output_format, tuple(valid_output_formats))) + + if output_format: + if len(output) == 1: + check_valid_format(output_format) + warnings.warn('The -t command line option is deprecated in favor' + ' of specifying the output format within the -o' + ' option, please check the -o option docs for more details', + category=ScrapyDeprecationWarning, stacklevel=2) + return {output[0]: {'format': output_format}} + else: + raise UsageError('The -t command line option cannot be used if multiple' + ' output files are specified with the -o option') + + result = {} + for element in output: + try: + feed_uri, feed_format = element.rsplit(':', 1) + except ValueError: + feed_uri = element + feed_format = os.path.splitext(element)[1].replace('.', '') + else: + if feed_uri == '-': + feed_uri = 'stdout:' + check_valid_format(feed_format) + result[feed_uri] = {'format': feed_format} + + # FEEDS setting should take precedence over the -o and -t CLI options + result.update(settings.getdict('FEEDS')) + + return result diff --git a/tests/test_commands.py b/tests/test_commands.py index 3612b70c9..24a341759 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,22 +1,46 @@ import inspect +import json +import optparse import os -import sys import subprocess +import sys import tempfile +from contextlib import contextmanager from os.path import exists, join, abspath from shutil import rmtree, copytree from tempfile import mkdtemp -from contextlib import contextmanager from threading import Timer from twisted.trial import unittest import scrapy +from scrapy.commands import ScrapyCommand +from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv + from tests.test_crawler import ExceptionSpider, NoRequestsSpider +class CommandSettings(unittest.TestCase): + + def setUp(self): + self.command = ScrapyCommand() + self.command.settings = Settings() + self.parser = optparse.OptionParser( + formatter=optparse.TitledHelpFormatter(), + conflict_handler='resolve', + ) + self.command.add_options(self.parser) + + def test_settings_json_string(self): + feeds_json = '{"data.json": {"format": "json"}, "data.xml": {"format": "xml"}}' + opts, args = self.parser.parse_args(args=['-s', 'FEEDS={}'.format(feeds_json), 'spider.py']) + self.command.process_options(args, opts) + self.assertIsInstance(self.command.settings['FEEDS'], scrapy.settings.BaseSettings) + self.assertEqual(dict(self.command.settings['FEEDS']), json.loads(feeds_json)) + + class ProjectTest(unittest.TestCase): project_name = 'testproject' @@ -34,7 +58,7 @@ class ProjectTest(unittest.TestCase): with tempfile.TemporaryFile() as out: args = (sys.executable, '-m', 'scrapy.cmdline') + new_args return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd, - env=self.env, **kwargs) + env=self.env, **kwargs) def proc(self, *new_args, **popen_kwargs): args = (sys.executable, '-m', 'scrapy.cmdline') + new_args @@ -310,6 +334,6 @@ class BenchCommandTest(CommandTest): def test_run(self): _, _, log = self.proc('bench', '-s', 'LOGSTATS_INTERVAL=0.001', - '-s', 'CLOSESPIDER_TIMEOUT=0.01') + '-s', 'CLOSESPIDER_TIMEOUT=0.01') self.assertIn('INFO: Crawled', log) self.assertNotIn('Unhandled Error', log) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2ca57c19d..08e8dfc41 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,32 +1,34 @@ -import os import csv import json -import warnings -import tempfile +import os +import random import shutil import string +import tempfile +import warnings from io import BytesIO from pathlib import Path +from string import ascii_letters, digits from unittest import mock from urllib.parse import urljoin, urlparse, quote from urllib.request import pathname2url -from zope.interface.verify import verifyObject -from twisted.trial import unittest +import lxml.etree from twisted.internet import defer -from scrapy.crawler import CrawlerRunner -from scrapy.settings import Settings -from tests.mockserver import MockServer +from twisted.trial import unittest from w3lib.url import path_to_file_uri +from zope.interface.verify import verifyObject import scrapy +from scrapy.crawler import CrawlerRunner from scrapy.exporters import CsvItemExporter -from scrapy.extensions.feedexport import ( - IFeedStorage, FileFeedStorage, FTPFeedStorage, - S3FeedStorage, StdoutFeedStorage, - BlockingFeedStorage) -from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler +from scrapy.extensions.feedexport import (BlockingFeedStorage, FileFeedStorage, FTPFeedStorage, + IFeedStorage, S3FeedStorage, StdoutFeedStorage) +from scrapy.settings import Settings from scrapy.utils.python import to_unicode +from scrapy.utils.test import assert_aws_environ, get_crawler, get_s3_content_and_delete + +from tests.mockserver import MockServer class FileFeedStorageTest(unittest.TestCase): @@ -395,29 +397,41 @@ class FeedExportTest(unittest.TestCase): egg = scrapy.Field() baz = scrapy.Field() + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def _random_temp_filename(self): + chars = [random.choice(ascii_letters + digits) for _ in range(15)] + filename = ''.join(chars) + return os.path.join(self.temp_dir, filename) + @defer.inlineCallbacks - def run_and_export(self, spider_cls, settings=None): + def run_and_export(self, spider_cls, settings): """ Run spider with specified settings; return exported data. """ - tmpdir = tempfile.mkdtemp() - res_path = os.path.join(tmpdir, 'res') - res_uri = urljoin('file:', pathname2url(res_path)) - defaults = { - 'FEED_URI': res_uri, - 'FEED_FORMAT': 'csv', - 'FEED_PATH': res_path + + FEEDS = settings.get('FEEDS') or {} + settings['FEEDS'] = { + urljoin('file:', pathname2url(str(file_path))): feed + for file_path, feed in FEEDS.items() } - defaults.update(settings or {}) + + content = {} try: with MockServer() as s: - runner = CrawlerRunner(Settings(defaults)) + runner = CrawlerRunner(Settings(settings)) spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) - with open(str(defaults['FEED_PATH']), 'rb') as f: - content = f.read() + for file_path, feed in FEEDS.items(): + with open(str(file_path), 'rb') as f: + content[feed['format']] = f.read() finally: - shutil.rmtree(tmpdir) + for file_path in FEEDS.keys(): + os.remove(str(file_path)) defer.returnValue(content) @@ -453,10 +467,14 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None, ordered=True): settings = settings or {} - settings.update({'FEED_FORMAT': 'csv'}) + settings.update({ + 'FEEDS': { + self._random_temp_filename(): {'format': 'csv'}, + }, + }) data = yield self.exported_data(items, settings) - reader = csv.DictReader(to_unicode(data).splitlines()) + reader = csv.DictReader(to_unicode(data['csv']).splitlines()) got_rows = list(reader) if ordered: self.assertEqual(reader.fieldnames, header) @@ -468,51 +486,87 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): settings = settings or {} - settings.update({'FEED_FORMAT': 'jl'}) + settings.update({ + 'FEEDS': { + self._random_temp_filename(): {'format': 'jl'}, + }, + }) data = yield self.exported_data(items, settings) - parsed = [json.loads(to_unicode(line)) for line in data.splitlines()] + parsed = [json.loads(to_unicode(line)) for line in data['jl'].splitlines()] rows = [{k: v for k, v in row.items() if v} for row in rows] self.assertEqual(rows, parsed) @defer.inlineCallbacks def assertExportedXml(self, items, rows, settings=None): settings = settings or {} - settings.update({'FEED_FORMAT': 'xml'}) + settings.update({ + 'FEEDS': { + self._random_temp_filename(): {'format': 'xml'}, + }, + }) data = yield self.exported_data(items, settings) rows = [{k: v for k, v in row.items() if v} for row in rows] - import lxml.etree - root = lxml.etree.fromstring(data) + root = lxml.etree.fromstring(data['xml']) got_rows = [{e.tag: e.text for e in it} for it in root.findall('item')] self.assertEqual(rows, got_rows) + @defer.inlineCallbacks + def assertExportedMultiple(self, items, rows, settings=None): + settings = settings or {} + settings.update({ + 'FEEDS': { + self._random_temp_filename(): {'format': 'xml'}, + self._random_temp_filename(): {'format': 'json'}, + }, + }) + data = yield self.exported_data(items, settings) + rows = [{k: v for k, v in row.items() if v} for row in rows] + # XML + root = lxml.etree.fromstring(data['xml']) + xml_rows = [{e.tag: e.text for e in it} for it in root.findall('item')] + self.assertEqual(rows, xml_rows) + # JSON + json_rows = json.loads(to_unicode(data['json'])) + self.assertEqual(rows, json_rows) + def _load_until_eof(self, data, load_func): - bytes_output = BytesIO(data) result = [] - while True: - try: - result.append(load_func(bytes_output)) - except EOFError: - break + with tempfile.TemporaryFile() as temp: + temp.write(data) + temp.seek(0) + while True: + try: + result.append(load_func(temp)) + except EOFError: + break return result @defer.inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} - settings.update({'FEED_FORMAT': 'pickle'}) + settings.update({ + 'FEEDS': { + self._random_temp_filename(): {'format': 'pickle'}, + }, + }) data = yield self.exported_data(items, settings) expected = [{k: v for k, v in row.items() if v} for row in rows] import pickle - result = self._load_until_eof(data, load_func=pickle.load) + result = self._load_until_eof(data['pickle'], load_func=pickle.load) self.assertEqual(expected, result) @defer.inlineCallbacks def assertExportedMarshal(self, items, rows, settings=None): settings = settings or {} - settings.update({'FEED_FORMAT': 'marshal'}) + settings.update({ + 'FEEDS': { + self._random_temp_filename(): {'format': 'marshal'}, + }, + }) data = yield self.exported_data(items, settings) expected = [{k: v for k, v in row.items() if v} for row in rows] import marshal - result = self._load_until_eof(data, load_func=marshal.load) + result = self._load_until_eof(data['marshal'], load_func=marshal.load) self.assertEqual(expected, result) @defer.inlineCallbacks @@ -521,6 +575,8 @@ class FeedExportTest(unittest.TestCase): yield self.assertExportedJsonLines(items, rows, settings) yield self.assertExportedXml(items, rows, settings) yield self.assertExportedPickle(items, rows, settings) + yield self.assertExportedMarshal(items, rows, settings) + yield self.assertExportedMultiple(items, rows, settings) @defer.inlineCallbacks def test_export_items(self): @@ -538,15 +594,14 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def test_export_no_items_not_store_empty(self): - formats = ('json', - 'jsonlines', - 'xml', - 'csv',) - - for fmt in formats: - settings = {'FEED_FORMAT': fmt} + for fmt in ('json', 'jsonlines', 'xml', 'csv'): + settings = { + 'FEEDS': { + self._random_temp_filename(): {'format': fmt}, + }, + } data = yield self.exported_no_data(settings) - self.assertEqual(data, b'') + self.assertEqual(data[fmt], b'') @defer.inlineCallbacks def test_export_no_items_store_empty(self): @@ -558,9 +613,15 @@ class FeedExportTest(unittest.TestCase): ) for fmt, expctd in formats: - settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None} + settings = { + 'FEEDS': { + self._random_temp_filename(): {'format': fmt}, + }, + 'FEED_STORE_EMPTY': True, + 'FEED_EXPORT_INDENT': None, + } data = yield self.exported_no_data(settings) - self.assertEqual(data, expctd) + self.assertEqual(data[fmt], expctd) @defer.inlineCallbacks def test_export_multiple_item_classes(self): @@ -581,9 +642,9 @@ class FeedExportTest(unittest.TestCase): header = self.MyItem.fields.keys() rows_csv = [ {'egg': 'spam1', 'foo': 'bar1', 'baz': ''}, - {'egg': '', 'foo': 'bar2', 'baz': ''}, + {'egg': '', 'foo': 'bar2', 'baz': ''}, {'egg': 'spam3', 'foo': 'bar3', 'baz': 'quux3'}, - {'egg': 'spam4', 'foo': '', 'baz': ''}, + {'egg': 'spam4', 'foo': '', 'baz': ''}, ] rows_jl = [dict(row) for row in items] yield self.assertExportedCsv(items, header, rows_csv, ordered=False) @@ -598,10 +659,10 @@ class FeedExportTest(unittest.TestCase): header = ["foo", "baz", "hello"] settings = {'FEED_EXPORT_FIELDS': header} rows = [ - {'foo': 'bar1', 'baz': '', 'hello': ''}, - {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, + {'foo': 'bar1', 'baz': '', 'hello': ''}, + {'foo': 'bar2', 'baz': '', 'hello': 'world2'}, {'foo': 'bar3', 'baz': 'quux3', 'hello': ''}, - {'foo': '', 'baz': '', 'hello': 'world4'}, + {'foo': '', 'baz': '', 'hello': 'world4'}, ] yield self.assertExported(items, header, rows, settings=settings, ordered=True) @@ -663,10 +724,15 @@ class FeedExportTest(unittest.TestCase): 'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'), } - for format, expected in formats.items(): - settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT': None} + for fmt, expected in formats.items(): + settings = { + 'FEEDS': { + self._random_temp_filename(): {'format': fmt}, + }, + 'FEED_EXPORT_INDENT': None, + } data = yield self.exported_data(items, settings) - self.assertEqual(expected, data) + self.assertEqual(expected, data[fmt]) formats = { 'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'), @@ -675,11 +741,53 @@ class FeedExportTest(unittest.TestCase): 'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'), } - settings = {'FEED_EXPORT_INDENT': None, 'FEED_EXPORT_ENCODING': 'latin-1'} - for format, expected in formats.items(): - settings['FEED_FORMAT'] = format + for fmt, expected in formats.items(): + settings = { + 'FEEDS': { + self._random_temp_filename(): {'format': fmt}, + }, + 'FEED_EXPORT_INDENT': None, + 'FEED_EXPORT_ENCODING': 'latin-1', + } data = yield self.exported_data(items, settings) - self.assertEqual(expected, data) + self.assertEqual(expected, data[fmt]) + + @defer.inlineCallbacks + def test_export_multiple_configs(self): + items = [dict({'foo': u'FOO', 'bar': u'BAR'})] + + formats = { + 'json': u'[\n{"bar": "BAR"}\n]'.encode('utf-8'), + 'xml': u'\n\n \n FOO\n \n'.encode('latin-1'), + 'csv': u'bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'), + } + + settings = { + 'FEEDS': { + self._random_temp_filename(): { + 'format': 'json', + 'indent': 0, + 'fields': ['bar'], + 'encoding': 'utf-8', + }, + self._random_temp_filename(): { + 'format': 'xml', + 'indent': 2, + 'fields': ['foo'], + 'encoding': 'latin-1', + }, + self._random_temp_filename(): { + 'format': 'csv', + 'indent': None, + 'fields': ['bar', 'foo'], + 'encoding': 'utf-8', + }, + }, + } + + data = yield self.exported_data(items, settings) + for fmt, expected in formats.items(): + self.assertEqual(expected, data[fmt]) @defer.inlineCallbacks def test_export_indentation(self): @@ -827,33 +935,38 @@ class FeedExportTest(unittest.TestCase): ] for row in test_cases: - settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']} + settings = { + 'FEEDS': { + self._random_temp_filename(): { + 'format': row['format'], + 'indent': row['indent'], + }, + }, + } data = yield self.exported_data(items, settings) - print(row['format'], row['indent']) - self.assertEqual(row['expected'], data) + self.assertEqual(row['expected'], data[row['format']]) @defer.inlineCallbacks def test_init_exporters_storages_with_crawler(self): settings = { - 'FEED_EXPORTERS': {'csv': 'tests.test_feedexport.' - 'FromCrawlerCsvItemExporter'}, - 'FEED_STORAGES': {'file': 'tests.test_feedexport.' - 'FromCrawlerFileFeedStorage'}, + 'FEED_EXPORTERS': {'csv': 'tests.test_feedexport.FromCrawlerCsvItemExporter'}, + 'FEED_STORAGES': {'file': 'tests.test_feedexport.FromCrawlerFileFeedStorage'}, + 'FEEDS': { + self._random_temp_filename(): {'format': 'csv'}, + }, } - yield self.exported_data({}, settings) + yield self.exported_data(items=[], settings=settings) self.assertTrue(FromCrawlerCsvItemExporter.init_with_crawler) self.assertTrue(FromCrawlerFileFeedStorage.init_with_crawler) @defer.inlineCallbacks def test_pathlib_uri(self): - tmpdir = tempfile.mkdtemp() - feed_uri = Path(tmpdir) / 'res' + feed_path = Path(self._random_temp_filename()) settings = { - 'FEED_FORMAT': 'csv', 'FEED_STORE_EMPTY': True, - 'FEED_URI': feed_uri, - 'FEED_PATH': feed_uri + 'FEEDS': { + feed_path: {'format': 'csv'} + }, } data = yield self.exported_no_data(settings) - self.assertEqual(data, b'') - shutil.rmtree(tmpdir, ignore_errors=True) + self.assertEqual(data['csv'], b'') diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 61e110845..f064a646c 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -1,7 +1,9 @@ import unittest +import warnings -from scrapy.settings import BaseSettings -from scrapy.utils.conf import build_component_list, arglist_to_dict +from scrapy.exceptions import UsageError, ScrapyDeprecationWarning +from scrapy.settings import BaseSettings, Settings +from scrapy.utils.conf import build_component_list, arglist_to_dict, feed_process_params_from_cli class BuildComponentListTest(unittest.TestCase): @@ -90,5 +92,49 @@ class UtilsConfTestCase(unittest.TestCase): {'arg1': 'val1', 'arg2': 'val2'}) +class FeedExportConfigTestCase(unittest.TestCase): + + def test_feed_export_config_invalid_format(self): + settings = Settings() + self.assertRaises(UsageError, feed_process_params_from_cli, settings, ['items.dat'], 'noformat') + + def test_feed_export_config_mismatch(self): + settings = Settings() + self.assertRaises( + UsageError, + feed_process_params_from_cli, settings, ['items1.dat', 'items2.dat'], 'noformat' + ) + + def test_feed_export_config_backward_compatible(self): + with warnings.catch_warnings(record=True) as cw: + settings = Settings() + self.assertEqual( + {'items.dat': {'format': 'csv'}}, + feed_process_params_from_cli(settings, ['items.dat'], 'csv') + ) + self.assertEqual(cw[0].category, ScrapyDeprecationWarning) + + def test_feed_export_config_explicit_formats(self): + settings = Settings() + self.assertEqual( + {'items_1.dat': {'format': 'json'}, 'items_2.dat': {'format': 'xml'}, 'items_3.dat': {'format': 'csv'}}, + feed_process_params_from_cli(settings, ['items_1.dat:json', 'items_2.dat:xml', 'items_3.dat:csv']) + ) + + def test_feed_export_config_implicit_formats(self): + settings = Settings() + self.assertEqual( + {'items_1.json': {'format': 'json'}, 'items_2.xml': {'format': 'xml'}, 'items_3.csv': {'format': 'csv'}}, + feed_process_params_from_cli(settings, ['items_1.json', 'items_2.xml', 'items_3.csv']) + ) + + def test_feed_export_config_stdout(self): + settings = Settings() + self.assertEqual( + {'stdout:': {'format': 'pickle'}}, + feed_process_params_from_cli(settings, ['-:pickle']) + ) + + if __name__ == "__main__": unittest.main() From c2c6ea376ca2a1a0634946f95690c03fefb9990b Mon Sep 17 00:00:00 2001 From: nyov Date: Wed, 4 Mar 2020 21:30:32 +0000 Subject: [PATCH 112/281] Remove obsolete DEPRECATED_SETTINGS (deprecated.py) --- scrapy/cmdline.py | 2 -- scrapy/settings/deprecated.py | 23 ----------------------- 2 files changed, 25 deletions(-) delete mode 100644 scrapy/settings/deprecated.py diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index ec78f7c91..a4ec7c8ae 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -12,7 +12,6 @@ from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import inside_project, get_project_settings from scrapy.utils.python import garbage_collect -from scrapy.settings.deprecated import check_deprecated_settings def _iter_command_classes(module_name): @@ -118,7 +117,6 @@ def execute(argv=None, settings=None): pass else: settings['EDITOR'] = editor - check_deprecated_settings(settings) inproject = inside_project() cmds = _get_commands_dict(settings, inproject) diff --git a/scrapy/settings/deprecated.py b/scrapy/settings/deprecated.py deleted file mode 100644 index f6f878725..000000000 --- a/scrapy/settings/deprecated.py +++ /dev/null @@ -1,23 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning - -DEPRECATED_SETTINGS = [ - ('TRACK_REFS', 'no longer needed (trackref is always enabled)'), - ('RESPONSE_CLASSES', 'no longer supported'), - ('DEFAULT_RESPONSE_ENCODING', 'no longer supported'), - ('BOT_VERSION', 'no longer used (user agent defaults to Scrapy now)'), - ('ENCODING_ALIASES', 'no longer needed (encoding discovery uses w3lib now)'), - ('STATS_ENABLED', 'no longer supported (change STATS_CLASS instead)'), - ('SQLITE_DB', 'no longer supported'), - ('AUTOTHROTTLE_MIN_DOWNLOAD_DELAY', 'use DOWNLOAD_DELAY instead'), - ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'), -] - - -def check_deprecated_settings(settings): - deprecated = [x for x in DEPRECATED_SETTINGS if settings[x[0]] is not None] - if deprecated: - msg = "You are using the following settings which are deprecated or obsolete" - msg += " (ask scrapy-users@googlegroups.com for alternatives):" - msg = msg + "\n " + "\n ".join("%s: %s" % x for x in deprecated) - warnings.warn(msg, ScrapyDeprecationWarning) From 915e363db5cb208de9043b61015b79c91ed0a6bb Mon Sep 17 00:00:00 2001 From: nyov Date: Sat, 7 Mar 2020 18:03:25 +0000 Subject: [PATCH 113/281] Remove a 'twisted.test.proto_helpers' deprecation warning --- tests/test_webclient.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 99a998a46..6253d5c3f 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -9,7 +9,12 @@ import OpenSSL.SSL from twisted.trial import unittest from twisted.web import server, static, util, resource from twisted.internet import reactor, defer -from twisted.test.proto_helpers import StringTransport +try: + from twisted.internet.testing import StringTransport +except ImportError: + # deprecated in Twisted 19.7.0 + # (remove once we bump our requirement past that version) + from twisted.test.proto_helpers import StringTransport from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory from twisted.internet.defer import inlineCallbacks From 9d9dea0d69709ef0f7aef67ddba1bd7bda25d273 Mon Sep 17 00:00:00 2001 From: Lukas Anzinger Date: Sat, 7 Mar 2020 19:54:25 +0100 Subject: [PATCH 114/281] Fix handling of None in allowed_domains. Nones in allowed_domains ought to be ignored and there are also tests for that scenario. This commit fixes the handling of None and also the accompanying tests which are now executed again. --- scrapy/spidermiddlewares/offsite.py | 8 ++++++-- tests/test_spidermiddleware_offsite.py | 18 +++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 232e96cbb..36f809699 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -54,12 +54,16 @@ class OffsiteMiddleware(object): if not allowed_domains: return re.compile('') # allow all by default url_pattern = re.compile("^https?://.*$") + domains = [] for domain in allowed_domains: - if url_pattern.match(domain): + if domain is None: + continue + elif url_pattern.match(domain): message = ("allowed_domains accepts only domains, not URLs. " "Ignoring URL entry %s in allowed_domains." % domain) warnings.warn(message, URLWarning) - domains = [re.escape(d) for d in allowed_domains if d is not None] + else: + domains.append(re.escape(domain)) regex = r'^(.*\.)?(%s)$' % '|'.join(domains) return re.compile(regex) diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 7511aa568..51c328943 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -55,21 +55,21 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): class TestOffsiteMiddleware3(TestOffsiteMiddleware2): - def _get_spider(self): - return Spider('foo') + def _get_spiderargs(self): + return dict(name='foo') class TestOffsiteMiddleware4(TestOffsiteMiddleware3): - def _get_spider(self): - bad_hostname = urlparse('http:////scrapytest.org').hostname - return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname]) + def _get_spiderargs(self): + bad_hostname = urlparse('http:////scrapytest.org').hostname + return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname]) def test_process_spider_output(self): - res = Response('http://scrapytest.org') - reqs = [Request('http://scrapytest.org/1')] - out = list(self.mw.process_spider_output(res, reqs, self.spider)) - self.assertEqual(out, reqs) + res = Response('http://scrapytest.org') + reqs = [Request('http://scrapytest.org/1')] + out = list(self.mw.process_spider_output(res, reqs, self.spider)) + self.assertEqual(out, reqs) class TestOffsiteMiddleware5(TestOffsiteMiddleware4): From 91a78eef3ee9de033e66db55c49321b2cc43740e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 8 Mar 2020 22:32:17 -0300 Subject: [PATCH 115/281] Pass callback results as dicts instead of tuples --- scrapy/core/downloader/handlers/http11.py | 56 ++++++++++++++++------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 190ae1d3b..e904cbc05 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -384,7 +384,13 @@ class ScrapyAgent(object): def _cb_bodyready(self, txresponse, request): # deliverBody hangs for responses without body if txresponse.length == 0: - return txresponse, b'', None, None + return { + "txresponse": txresponse, + "body": b"", + "flags": None, + "certificate": None, + "ip_address": None, + } maxsize = request.meta.get('download_maxsize', self._maxsize) warnsize = request.meta.get('download_warnsize', self._warnsize) @@ -420,12 +426,18 @@ class ScrapyAgent(object): return d def _cb_bodydone(self, result, request, url): - txresponse, body, flags, certificate, ip_address = result - status = int(txresponse.code) - headers = Headers(txresponse.headers.getAllRawHeaders()) - respcls = responsetypes.from_args(headers=headers, url=url, body=body) - return respcls(url=url, status=status, headers=headers, body=body, - flags=flags, certificate=certificate, ip_address=ip_address) + status = int(result["txresponse"].code) + headers = Headers(result["txresponse"].headers.getAllRawHeaders()) + respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) + return respcls( + url=url, + status=status, + headers=headers, + body=result["body"], + flags=result["flags"], + certificate=result["certificate"], + ip_address=result["ip_address"], + ) @implementer(IBodyProducer) @@ -501,22 +513,34 @@ class _ResponseReader(protocol.Protocol): body = self._bodybuf.getvalue() if reason.check(ResponseDone): - self._finished.callback( - (self._txresponse, body, None, self._certificate, self._ip_address) - ) + self._finished.callback({ + "txresponse": self._txresponse, + "body": body, + "flags": None, + "certificate": self._certificate, + "ip_address": self._ip_address, + }) return if reason.check(PotentialDataLoss): - self._finished.callback( - (self._txresponse, body, ['partial'], self._certificate, self._ip_address) - ) + self._finished.callback({ + "txresponse": self._txresponse, + "body": body, + "flags": ["partial"], + "certificate": self._certificate, + "ip_address": self._ip_address, + }) return if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons): if not self._fail_on_dataloss: - self._finished.callback( - (self._txresponse, body, ['dataloss'], self._certificate, self._ip_address) - ) + self._finished.callback({ + "txresponse": self._txresponse, + "body": body, + "flags": ["dataloss"], + "certificate": self._certificate, + "ip_address": self._ip_address, + }) return elif not self._fail_on_dataloss_warned: From 1785095707dec53647c835c0b0861b220e8495af Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Mar 2020 20:41:59 -0300 Subject: [PATCH 116/281] Remove single-use variable --- scrapy/core/downloader/handlers/http11.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index e904cbc05..a5b03a62b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -426,12 +426,11 @@ class ScrapyAgent(object): return d def _cb_bodydone(self, result, request, url): - status = int(result["txresponse"].code) headers = Headers(result["txresponse"].headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) return respcls( url=url, - status=status, + status=int(result["txresponse"].code), headers=headers, body=result["body"], flags=result["flags"], From 49156f2ecb0197c96e3889805a233b9a626c6d65 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Mar 2020 20:45:54 -0300 Subject: [PATCH 117/281] [doc] Feed exports: full local path as example --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 6d6ba33c9..9e5968a29 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -249,7 +249,7 @@ For instance:: 'fields': None, 'indent': 4, }, - 'items.xml': { + '/home/user/documents/items.xml': { 'format': 'xml', 'fields': ['name', 'price'], 'encoding': 'latin1', From f3bab819ab92cc0750c9b73141abdcbc8da7c4ac Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Mar 2020 20:56:25 -0300 Subject: [PATCH 118/281] Add tests for scrapy.utils.conf.feed_complete_default_values_from_settings --- tests/test_utils_conf.py | 45 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index f064a646c..332120021 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -3,7 +3,12 @@ import warnings from scrapy.exceptions import UsageError, ScrapyDeprecationWarning from scrapy.settings import BaseSettings, Settings -from scrapy.utils.conf import build_component_list, arglist_to_dict, feed_process_params_from_cli +from scrapy.utils.conf import ( + arglist_to_dict, + build_component_list, + feed_complete_default_values_from_settings, + feed_process_params_from_cli +) class BuildComponentListTest(unittest.TestCase): @@ -135,6 +140,44 @@ class FeedExportConfigTestCase(unittest.TestCase): feed_process_params_from_cli(settings, ['-:pickle']) ) + def test_feed_complete_default_values_from_settings_empty(self): + feed = {} + settings = Settings({ + "FEED_EXPORT_ENCODING": "custom encoding", + "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], + "FEED_EXPORT_INDENT": 42, + "FEED_STORE_EMPTY": True, + "FEED_URI_PARAMS": (1, 2, 3, 4), + }) + new_feed = feed_complete_default_values_from_settings(feed, settings) + self.assertEqual(new_feed, { + "encoding": "custom encoding", + "fields": ["f1", "f2", "f3"], + "indent": 42, + "store_empty": True, + "uri_params": (1, 2, 3, 4), + }) + + def test_feed_complete_default_values_from_settings_non_empty(self): + feed = { + "encoding": "other encoding", + "fields": None, + } + settings = Settings({ + "FEED_EXPORT_ENCODING": "custom encoding", + "FEED_EXPORT_FIELDS": ["f1", "f2", "f3"], + "FEED_EXPORT_INDENT": 42, + "FEED_STORE_EMPTY": True, + }) + new_feed = feed_complete_default_values_from_settings(feed, settings) + self.assertEqual(new_feed, { + "encoding": "other encoding", + "fields": None, + "indent": 42, + "store_empty": True, + "uri_params": None, + }) + if __name__ == "__main__": unittest.main() From c886a70eae35e628040da0504af3fd9aaa6aea75 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Mar 2020 21:06:51 -0300 Subject: [PATCH 119/281] Use dict.setdefault in scrapy.utils.conf.feed_complete_default_values_from_settings --- scrapy/utils/conf.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index e01027491..5921f82bf 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -113,16 +113,14 @@ def get_sources(use_closest=True): def feed_complete_default_values_from_settings(feed, settings): out = feed.copy() - if 'encoding' not in out: - out['encoding'] = settings['FEED_EXPORT_ENCODING'] - if 'fields' not in out: - out['fields'] = settings.getlist('FEED_EXPORT_FIELDS') or None - if 'indent' not in out: - out['indent'] = None if settings['FEED_EXPORT_INDENT'] is None else settings.getint('FEED_EXPORT_INDENT') - if 'store_empty' not in out: - out['store_empty'] = settings.getbool('FEED_STORE_EMPTY') - if 'uri_params' not in out: - out['uri_params'] = settings['FEED_URI_PARAMS'] + out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"]) + out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None) + out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY")) + out.setdefault("uri_params", settings["FEED_URI_PARAMS"]) + if settings["FEED_EXPORT_INDENT"] is None: + out.setdefault("indent", None) + else: + out.setdefault("indent", settings.getint("FEED_EXPORT_INDENT")) return out From 8d30dc08882e3b97dbaa17b3254de708c358d0d2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 12 Mar 2020 09:36:15 -0300 Subject: [PATCH 120/281] Response.follow_all: return empty generators for empty sequences --- scrapy/http/response/text.py | 8 +++++--- tests/test_http_response.py | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 33a485328..2f0f3820c 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -188,9 +188,11 @@ class TextResponse(Response): selectors from which links cannot be obtained (for instance, anchor tags without an ``href`` attribute) """ - arg_count = len(list(filter(None, (urls, css, xpath)))) - if arg_count != 1: - raise ValueError('Please supply exactly one of the following arguments: urls, css, xpath') + arguments = [x for x in (urls, css, xpath) if x is not None] + if len(arguments) != 1: + raise ValueError( + "Please supply exactly one of the following arguments: urls, css, xpath" + ) if not urls: if css: urls = self.css(css) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index be17dfd6b..eafc3560e 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -215,6 +215,10 @@ class BaseResponseTest(unittest.TestCase): links = map(Link, absolute) self._assert_followed_all_urls(links, absolute) + def test_follow_all_empty(self): + r = self.response_class("http://example.com") + self.assertEqual([], list(r.follow_all([]))) + def test_follow_all_invalid(self): r = self.response_class("http://example.com") if self.response_class == Response: From 3b0820d747e11a1a7722f0777baf923607fc0485 Mon Sep 17 00:00:00 2001 From: nyov Date: Thu, 12 Mar 2020 19:15:49 +0000 Subject: [PATCH 121/281] Deprecate Spider.make_requests_from_url, part 2 (#4412) --- scrapy/spiders/__init__.py | 6 ++++++ tests/test_spider.py | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 9429f6cb2..ba1c866f8 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -78,6 +78,12 @@ class Spider(object_ref): def make_requests_from_url(self, url): """ This method is deprecated. """ + warnings.warn( + "Spider.make_requests_from_url method is deprecated: " + "it will be removed and not be called by the default " + "Spider.start_requests method in future Scrapy releases. " + "Please override Spider.start_requests method instead." + ) return Request(url, dont_filter=True) def parse(self, response): diff --git a/tests/test_spider.py b/tests/test_spider.py index 317a27076..bb00c8f42 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -602,13 +602,19 @@ class DeprecationTest(unittest.TestCase): self.assertEqual(len(list(spider1.start_requests())), 1) self.assertEqual(len(w), 0) + # spider without overridden make_requests_from_url method + # should issue a warning when called directly + request = spider1.make_requests_from_url("http://www.example.com") + self.assertTrue(isinstance(request, Request)) + self.assertEqual(len(w), 1) + # spider with overridden make_requests_from_url issues a warning, # but the method still works spider2 = MySpider5() requests = list(spider2.start_requests()) self.assertEqual(len(requests), 1) self.assertEqual(requests[0].url, 'http://example.com/foo') - self.assertEqual(len(w), 1) + self.assertEqual(len(w), 2) class NoParseMethodSpiderTest(unittest.TestCase): From ccc4d88779cf2827431ef9e73f976f755c04fe0e Mon Sep 17 00:00:00 2001 From: Lukas Anzinger Date: Thu, 12 Mar 2020 20:42:14 +0100 Subject: [PATCH 122/281] Ignore a domain in allowed_domains with port and issue a warning (#4413) --- scrapy/spidermiddlewares/offsite.py | 11 ++++++++++- tests/test_spidermiddleware_offsite.py | 15 +++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 36f809699..2fab572e6 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -53,7 +53,8 @@ class OffsiteMiddleware(object): allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: return re.compile('') # allow all by default - url_pattern = re.compile("^https?://.*$") + url_pattern = re.compile(r"^https?://.*$") + port_pattern = re.compile(r":\d+$") domains = [] for domain in allowed_domains: if domain is None: @@ -62,6 +63,10 @@ class OffsiteMiddleware(object): message = ("allowed_domains accepts only domains, not URLs. " "Ignoring URL entry %s in allowed_domains." % domain) warnings.warn(message, URLWarning) + elif port_pattern.search(domain): + message = ("allowed_domains accepts only domains without ports. " + "Ignoring entry %s in allowed_domains." % domain) + warnings.warn(message, PortWarning) else: domains.append(re.escape(domain)) regex = r'^(.*\.)?(%s)$' % '|'.join(domains) @@ -74,3 +79,7 @@ class OffsiteMiddleware(object): class URLWarning(Warning): pass + + +class PortWarning(Warning): + pass diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 51c328943..b96807bc2 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -4,7 +4,7 @@ import warnings from scrapy.http import Response, Request from scrapy.spiders import Spider -from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning +from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning, PortWarning from scrapy.utils.test import get_crawler @@ -26,7 +26,8 @@ class TestOffsiteMiddleware(TestCase): Request('http://scrapy.org/1'), Request('http://sub.scrapy.org/1'), Request('http://offsite.tld/letmepass', dont_filter=True), - Request('http://scrapy.test.org/')] + Request('http://scrapy.test.org/'), + Request('http://scrapy.test.org:8000/')] offsite_reqs = [Request('http://scrapy2.org'), Request('http://offsite.tld/'), Request('http://offsite.tld/scrapytest.org'), @@ -80,3 +81,13 @@ class TestOffsiteMiddleware5(TestOffsiteMiddleware4): warnings.simplefilter("always") self.mw.get_host_regex(self.spider) assert issubclass(w[-1].category, URLWarning) + + +class TestOffsiteMiddleware6(TestOffsiteMiddleware4): + + def test_get_host_regex(self): + self.spider.allowed_domains = ['scrapytest.org:8000', 'scrapy.org', 'scrapy.test.org'] + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + self.mw.get_host_regex(self.spider) + assert issubclass(w[-1].category, PortWarning) From 3f6cdcabceff5c5b2ac2935f8073b6efe817645b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 13 Mar 2020 13:25:53 +0100 Subject: [PATCH 123/281] Restrict pytest to versions prior to 5.4 --- tests/requirements-py3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index d97c4b8ee..d207c5fb0 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -2,7 +2,7 @@ jmespath mitmproxy; python_version >= '3.6' mitmproxy<4.0.0; python_version < '3.6' -pytest +pytest < 5.4 pytest-cov pytest-twisted >= 1.11 pytest-xdist From f9bf4b8d4dd64a1d65e949927b8ea7ad34e756d3 Mon Sep 17 00:00:00 2001 From: Aditya Kumar Date: Sat, 14 Mar 2020 15:09:00 +0530 Subject: [PATCH 124/281] Remove all top-level imports for twisted.internet.reactor (#4406) --- docs/topics/settings.rst | 61 ++++++++++++++++++++++- scrapy/core/downloader/__init__.py | 3 +- scrapy/core/downloader/handlers/ftp.py | 2 +- scrapy/core/downloader/handlers/http10.py | 3 +- scrapy/core/downloader/handlers/http11.py | 6 ++- scrapy/extensions/closespider.py | 3 +- scrapy/mail.py | 5 +- scrapy/utils/benchserver.py | 2 +- scrapy/utils/testproc.py | 3 +- scrapy/utils/testsite.py | 3 +- 10 files changed, 78 insertions(+), 13 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a70023efa..c01202a10 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1473,7 +1473,66 @@ If a reactor is already installed, :meth:`CrawlerRunner.__init__ ` raises :exc:`Exception` if the installed reactor does not match the -:setting:`TWISTED_REACTOR` setting. +:setting:`TWISTED_REACTOR` setting; therfore, having top-level +:mod:`~twisted.internet.reactor` imports in project files and imported +third-party libraries will make Scrapy raise :exc:`Exception` when +it checks which reactor is installed. + +In order to use the reactor installed by Scrapy:: + + import scrapy + from twisted.internet import reactor + + + class QuotesSpider(scrapy.Spider): + name = 'quotes' + + def __init__(self, *args, **kwargs): + self.timeout = int(kwargs.pop('timeout', '60')) + super(QuotesSpider, self).__init__(*args, **kwargs) + + def start_requests(self): + reactor.callLater(self.timeout, self.stop) + + urls = ['http://quotes.toscrape.com/page/1'] + for url in urls: + yield scrapy.Request(url=url, callback=self.parse) + + def parse(self, response): + for quote in response.css('div.quote'): + yield {'text': quote.css('span.text::text').get()} + + def stop(self): + self.crawler.engine.close_spider(self, 'timeout') + + +which raises :exc:`Exception`, becomes:: + + import scrapy + + + class QuotesSpider(scrapy.Spider): + name = 'quotes' + + def __init__(self, *args, **kwargs): + self.timeout = int(kwargs.pop('timeout', '60')) + super(QuotesSpider, self).__init__(*args, **kwargs) + + def start_requests(self): + from twisted.internet import reactor + reactor.callLater(self.timeout, self.stop) + + urls = ['http://quotes.toscrape.com/page/1'] + for url in urls: + yield scrapy.Request(url=url, callback=self.parse) + + def parse(self, response): + for quote in response.css('div.quote'): + yield {'text': quote.css('span.text::text').get()} + + def stop(self): + self.crawler.engine.close_spider(self, 'timeout') + The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which means that Scrapy will not attempt to install any specific reactor, and the diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 5a2fdadf5..644be121f 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -3,7 +3,7 @@ from time import time from datetime import datetime from collections import deque -from twisted.internet import reactor, defer, task +from twisted.internet import defer, task from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached @@ -133,6 +133,7 @@ class Downloader(object): return deferred def _process_queue(self, spider, slot): + from twisted.internet import reactor if slot.latercall and slot.latercall.active(): return diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 1681c6df8..432cb1831 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -32,7 +32,6 @@ import re from io import BytesIO from urllib.parse import unquote -from twisted.internet import reactor from twisted.internet.protocol import ClientCreator, Protocol from twisted.protocols.ftp import CommandFailed, FTPClient @@ -81,6 +80,7 @@ class FTPDownloadHandler: return cls(crawler.settings) def download_request(self, request, spider): + from twisted.internet import reactor parsed_url = urlparse_cached(request) user = request.meta.get("ftp_user", self.default_user) password = request.meta.get("ftp_password", self.default_password) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index d4aa51bd1..c0146a0a6 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,7 +1,5 @@ """Download handlers for http and https schemes """ -from twisted.internet import reactor - from scrapy.utils.misc import create_instance, load_object from scrapy.utils.python import to_unicode @@ -26,6 +24,7 @@ class HTTP10DownloadHandler: return factory.deferred def _connect(self, factory): + from twisted.internet import reactor host, port = to_unicode(factory.host), factory.port if factory.scheme == b'https': client_context_factory = create_instance( diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 93951d3b5..04a8d617a 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -8,7 +8,7 @@ from io import BytesIO from time import time from urllib.parse import urldefrag -from twisted.internet import defer, protocol, reactor, ssl +from twisted.internet import defer, protocol, ssl from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI @@ -33,6 +33,7 @@ class HTTP11DownloadHandler: lazy = False def __init__(self, settings, crawler=None): + from twisted.internet import reactor self._pool = HTTPConnectionPool(reactor, persistent=True) self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self._pool._factory.noisy = False @@ -81,6 +82,7 @@ class HTTP11DownloadHandler: return agent.download_request(request) def close(self): + from twisted.internet import reactor d = self._pool.closeCachedConnections() # closeCachedConnections will hang on network or server issues, so # we'll manually timeout the deferred. @@ -284,6 +286,7 @@ class ScrapyAgent(object): self._txresponse = None def _get_agent(self, request, timeout): + from twisted.internet import reactor bindaddress = request.meta.get('bindaddress') or self._bindAddress proxy = request.meta.get('proxy') if proxy: @@ -326,6 +329,7 @@ class ScrapyAgent(object): ) def download_request(self, request): + from twisted.internet import reactor timeout = request.meta.get('download_timeout') or self._connectTimeout agent = self._get_agent(request, timeout) diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index afb2ed049..260b2e86e 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -6,8 +6,6 @@ See documentation in docs/topics/extensions.rst from collections import defaultdict -from twisted.internet import reactor - from scrapy import signals from scrapy.exceptions import NotConfigured @@ -54,6 +52,7 @@ class CloseSpider(object): self.crawler.engine.close_spider(spider, 'closespider_pagecount') def spider_opened(self, spider): + from twisted.internet import reactor self.task = reactor.callLater(self.close_on['timeout'], self.crawler.engine.close_spider, spider, reason='closespider_timeout') diff --git a/scrapy/mail.py b/scrapy/mail.py index 9655b8114..b2a24a3db 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -12,7 +12,7 @@ from email.mime.text import MIMEText from email.utils import COMMASPACE, formatdate from io import BytesIO -from twisted.internet import defer, reactor, ssl +from twisted.internet import defer, ssl from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes @@ -28,7 +28,6 @@ def _to_bytes_or_none(text): class MailSender(object): - def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): self.smtphost = smtphost @@ -47,6 +46,7 @@ class MailSender(object): settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL')) def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None): + from twisted.internet import reactor if attachs: msg = MIMEMultipart() else: @@ -111,6 +111,7 @@ class MailSender(object): def _sendmail(self, to_addrs, msg): # Import twisted.mail here because it is not available in python3 + from twisted.internet import reactor from twisted.mail.smtp import ESMTPSenderFactory msg = BytesIO(msg) d = defer.Deferred() diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index cdbe21942..9d8d64612 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -3,7 +3,6 @@ from urllib.parse import urlencode from twisted.web.server import Site from twisted.web.resource import Resource -from twisted.internet import reactor class Root(Resource): @@ -34,6 +33,7 @@ def _getarg(request, name, default=None, type=str): if __name__ == '__main__': + from twisted.internet import reactor root = Root() factory = Site(root) httpPort = reactor.listenTCP(8998, Site(root)) diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 0f15cf60a..37803b287 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -1,7 +1,7 @@ import sys import os -from twisted.internet import reactor, defer, protocol +from twisted.internet import defer, protocol class ProcessTest(object): @@ -11,6 +11,7 @@ class ProcessTest(object): cwd = os.getcwd() # trial chdirs to temp dir def execute(self, args, check_code=True, settings=None): + from twisted.internet import reactor env = os.environ.copy() if settings is not None: env['SCRAPY_SETTINGS_MODULE'] = settings diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 6f5c21624..9e1598805 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -1,12 +1,12 @@ from urllib.parse import urljoin -from twisted.internet import reactor from twisted.web import server, resource, static, util class SiteTest(object): def setUp(self): + from twisted.internet import reactor super(SiteTest, self).setUp() self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") self.baseurl = "http://localhost:%d/" % self.site.getHost().port @@ -38,6 +38,7 @@ def test_site(): if __name__ == '__main__': + from twisted.internet import reactor port = reactor.listenTCP(0, test_site(), interface="127.0.0.1") print("http://localhost:%d/" % port.getHost().port) reactor.run() From e5711127b162ce78e6c1b60ef209792cd6eef4e6 Mon Sep 17 00:00:00 2001 From: elacuesta Date: Mon, 16 Mar 2020 15:43:02 -0300 Subject: [PATCH 125/281] Remove deprecated ChunkedTransferMiddleware (#4431) --- scrapy/downloadermiddlewares/chunked.py | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 scrapy/downloadermiddlewares/chunked.py diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py deleted file mode 100644 index 6748d0265..000000000 --- a/scrapy/downloadermiddlewares/chunked.py +++ /dev/null @@ -1,21 +0,0 @@ -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.http import decode_chunked_transfer - - -warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, " - "chunked transfers are supported by default.", - ScrapyDeprecationWarning, stacklevel=2) - - -class ChunkedTransferMiddleware(object): - """This middleware adds support for chunked transfer encoding, as - documented in: https://en.wikipedia.org/wiki/Chunked_transfer_encoding - """ - - def process_response(self, request, response, spider): - if response.headers.get('Transfer-Encoding') == 'chunked': - body = decode_chunked_transfer(response.body) - return response.replace(body=body) - return response From dfbe1d95071acfbba159cea051530749b1684460 Mon Sep 17 00:00:00 2001 From: elacuesta Date: Mon, 16 Mar 2020 16:12:46 -0300 Subject: [PATCH 126/281] Remove object base class (#4430) --- docs/topics/exporters.rst | 2 +- docs/topics/extensions.rst | 2 +- docs/topics/item-pipeline.rst | 10 +++++----- docs/topics/leaks.rst | 2 +- docs/topics/settings.rst | 2 +- docs/topics/stats.rst | 2 +- scrapy/commands/__init__.py | 2 +- scrapy/commands/bench.py | 2 +- scrapy/contracts/__init__.py | 4 ++-- scrapy/core/downloader/__init__.py | 4 ++-- scrapy/core/downloader/handlers/http11.py | 4 ++-- scrapy/core/engine.py | 4 ++-- scrapy/core/scheduler.py | 2 +- scrapy/core/scraper.py | 4 ++-- scrapy/downloadermiddlewares/ajaxcrawl.py | 2 +- scrapy/downloadermiddlewares/cookies.py | 2 +- scrapy/downloadermiddlewares/decompression.py | 2 +- scrapy/downloadermiddlewares/defaultheaders.py | 2 +- .../downloadermiddlewares/downloadtimeout.py | 2 +- scrapy/downloadermiddlewares/httpauth.py | 2 +- scrapy/downloadermiddlewares/httpcache.py | 2 +- .../downloadermiddlewares/httpcompression.py | 2 +- scrapy/downloadermiddlewares/httpproxy.py | 2 +- scrapy/downloadermiddlewares/redirect.py | 2 +- scrapy/downloadermiddlewares/retry.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 2 +- scrapy/downloadermiddlewares/stats.py | 2 +- scrapy/downloadermiddlewares/useragent.py | 2 +- scrapy/dupefilters.py | 2 +- scrapy/exporters.py | 2 +- scrapy/extensions/closespider.py | 2 +- scrapy/extensions/corestats.py | 2 +- scrapy/extensions/debug.py | 4 ++-- scrapy/extensions/feedexport.py | 10 +++++----- scrapy/extensions/httpcache.py | 8 ++++---- scrapy/extensions/logstats.py | 2 +- scrapy/extensions/memdebug.py | 2 +- scrapy/extensions/memusage.py | 2 +- scrapy/extensions/spiderstate.py | 2 +- scrapy/extensions/statsmailer.py | 2 +- scrapy/extensions/throttle.py | 2 +- scrapy/http/cookies.py | 8 ++++---- scrapy/link.py | 2 +- scrapy/linkextractors/__init__.py | 2 +- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/loader/__init__.py | 2 +- scrapy/loader/processors.py | 12 ++++++------ scrapy/logformatter.py | 2 +- scrapy/mail.py | 2 +- scrapy/middleware.py | 2 +- scrapy/pipelines/files.py | 8 ++++---- scrapy/pipelines/media.py | 4 ++-- scrapy/pqueues.py | 6 +++--- scrapy/responsetypes.py | 2 +- scrapy/settings/__init__.py | 2 +- scrapy/shell.py | 2 +- scrapy/signalmanager.py | 2 +- scrapy/spiderloader.py | 2 +- scrapy/spidermiddlewares/depth.py | 2 +- scrapy/spidermiddlewares/httperror.py | 2 +- scrapy/spidermiddlewares/offsite.py | 2 +- scrapy/spidermiddlewares/referer.py | 4 ++-- scrapy/spidermiddlewares/urllength.py | 2 +- scrapy/spiders/crawl.py | 2 +- scrapy/statscollectors.py | 2 +- .../project/module/middlewares.py.tmpl | 4 ++-- .../templates/project/module/pipelines.py.tmpl | 2 +- scrapy/utils/datatypes.py | 2 +- scrapy/utils/deprecate.py | 2 +- scrapy/utils/iterators.py | 2 +- scrapy/utils/log.py | 2 +- scrapy/utils/python.py | 4 ++-- scrapy/utils/reactor.py | 2 +- scrapy/utils/sitemap.py | 2 +- scrapy/utils/testproc.py | 2 +- scrapy/utils/testsite.py | 2 +- scrapy/utils/trackref.py | 5 ++--- tests/pipelines.py | 4 ++-- tests/test_cmdline/extensions.py | 4 ++-- .../test_spider/pipelines.py | 4 ++-- tests/test_command_parse.py | 2 +- tests/test_contracts.py | 2 +- tests/test_crawler.py | 2 +- tests/test_downloadermiddleware.py | 2 +- tests/test_dupefilters.py | 2 +- tests/test_engine.py | 2 +- tests/test_feedexport.py | 2 +- tests/test_item.py | 2 +- tests/test_loader.py | 2 +- tests/test_logformatter.py | 2 +- tests/test_middleware.py | 8 ++++---- tests/test_request_cb_kwargs.py | 4 ++-- tests/test_scheduler.py | 6 +++--- tests/test_spidermiddleware_referer.py | 18 +++++++++--------- tests/test_squeues.py | 2 +- tests/test_utils_deprecate.py | 6 +++--- tests/test_utils_python.py | 8 ++++---- tests/test_utils_reqser.py | 2 +- 98 files changed, 155 insertions(+), 156 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index d411e2eed..e52682690 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -42,7 +42,7 @@ value of one of their fields:: from scrapy.exporters import XmlItemExporter - class PerYearXmlExportPipeline(object): + class PerYearXmlExportPipeline: """Distribute items across multiple XML files according to their 'year' field""" def open_spider(self, spider): diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index dc057f6b6..94fd2e36e 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -107,7 +107,7 @@ Here is the code of such extension:: logger = logging.getLogger(__name__) - class SpiderOpenCloseLogging(object): + class SpiderOpenCloseLogging: def __init__(self, item_count): self.item_count = item_count diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 98e2506e5..533f84630 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -81,7 +81,7 @@ contain a price:: from scrapy.exceptions import DropItem - class PricePipeline(object): + class PricePipeline: vat_factor = 1.15 @@ -103,7 +103,7 @@ format:: import json - class JsonWriterPipeline(object): + class JsonWriterPipeline: def open_spider(self, spider): self.file = open('items.jl', 'w') @@ -132,7 +132,7 @@ method and how to clean up the resources properly.:: import pymongo - class MongoPipeline(object): + class MongoPipeline: collection_name = 'scrapy_items' @@ -180,7 +180,7 @@ it saves the screenshot to a file and adds filename to the item. from urllib.parse import quote - class ScreenshotPipeline(object): + class ScreenshotPipeline: """Pipeline that uses Splash to render screenshot of every Scrapy item.""" @@ -219,7 +219,7 @@ returns multiples items with the same id:: from scrapy.exceptions import DropItem - class DuplicatesPipeline(object): + class DuplicatesPipeline: def __init__(self): self.ids_seen = set() diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index c0c83fc84..4ee447065 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -170,7 +170,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module. .. class:: object_ref - Inherit from this class (instead of object) if you want to track live + Inherit from this class if you want to track live instances with the ``trackref`` module. .. function:: print_live_refs(class_name, ignore=NoneType) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c01202a10..dc6843d75 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -124,7 +124,7 @@ Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings` attribute of the Crawler that is passed to ``from_crawler`` method in extensions, middlewares and item pipelines:: - class MyExtension(object): + class MyExtension: def __init__(self, log_is_enabled=False): if log_is_enabled: print("log is enabled!") diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 3dd829ebe..af848b402 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -32,7 +32,7 @@ Common Stats Collector uses Access the stats collector through the :attr:`~scrapy.crawler.Crawler.stats` attribute. Here is an example of an extension that access stats:: - class ExtensionThatAccessStats(object): + class ExtensionThatAccessStats: def __init__(self, stats): self.stats = stats diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 0b24193c2..a573a03d9 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -9,7 +9,7 @@ from scrapy.utils.conf import arglist_to_dict from scrapy.exceptions import UsageError -class ScrapyCommand(object): +class ScrapyCommand: requires_project = False crawler_process = None diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 7bbe362e7..c9f3b38e0 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -25,7 +25,7 @@ class Command(ScrapyCommand): self.crawler_process.start() -class _BenchServer(object): +class _BenchServer: def __enter__(self): from scrapy.utils.test import get_testenv diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 7b6591d86..41d4f25b2 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -9,7 +9,7 @@ from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_spec -class ContractsManager(object): +class ContractsManager: contracts = {} def __init__(self, contracts): @@ -107,7 +107,7 @@ class ContractsManager(object): request.errback = eb_wrapper -class Contract(object): +class Contract: """ Abstract class for contracts """ request_cls = None diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 644be121f..36aca4dae 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -13,7 +13,7 @@ from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.core.downloader.handlers import DownloadHandlers -class Slot(object): +class Slot: """Downloader slot""" def __init__(self, concurrency, delay, randomize_delay): @@ -66,7 +66,7 @@ def _get_concurrency_delay(concurrency, spider, settings): return concurrency, delay -class Downloader(object): +class Downloader: DOWNLOAD_SLOT = 'download_slot' diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 04a8d617a..c970909d7 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -268,7 +268,7 @@ class ScrapyProxyAgent(Agent): ) -class ScrapyAgent(object): +class ScrapyAgent: _Agent = Agent _ProxyAgent = ScrapyProxyAgent @@ -432,7 +432,7 @@ class ScrapyAgent(object): @implementer(IBodyProducer) -class _RequestBodyProducer(object): +class _RequestBodyProducer: def __init__(self, body): self.body = body diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 6ab8cde6b..74f03344e 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -21,7 +21,7 @@ from scrapy.utils.log import logformatter_adapter, failure_to_exc_info logger = logging.getLogger(__name__) -class Slot(object): +class Slot: def __init__(self, start_requests, close_if_idle, nextcall, scheduler): self.closing = False @@ -53,7 +53,7 @@ class Slot(object): self.closing.callback(None) -class ExecutionEngine(object): +class ExecutionEngine: def __init__(self, crawler, spider_closed_callback): self.crawler = crawler diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index c96b9b719..a18c26b17 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -14,7 +14,7 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning logger = logging.getLogger(__name__) -class Scheduler(object): +class Scheduler: """ Scrapy Scheduler. It allows to enqueue requests and then get a next request to download. Scheduler is also handling duplication diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 41f015017..3e4826216 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -21,7 +21,7 @@ from scrapy.core.spidermw import SpiderMiddlewareManager logger = logging.getLogger(__name__) -class Slot(object): +class Slot: """Scraper slot (one per running spider)""" MIN_RESPONSE_SIZE = 1024 @@ -62,7 +62,7 @@ class Slot(object): return self.active_size > self.max_active_size -class Scraper(object): +class Scraper: def __init__(self, crawler): self.slot = None diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 16b046e99..ad7a81e6b 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -11,7 +11,7 @@ from scrapy.http import HtmlResponse logger = logging.getLogger(__name__) -class AjaxCrawlMiddleware(object): +class AjaxCrawlMiddleware: """ Handle 'AJAX crawlable' pages marked as crawlable via meta tag. For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index d8dabdf13..d57f04bc3 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -10,7 +10,7 @@ from scrapy.utils.python import to_unicode logger = logging.getLogger(__name__) -class CookiesMiddleware(object): +class CookiesMiddleware: """This middleware enables working with sites that need cookies""" def __init__(self, debug=False): diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index fcea38ef5..0fcf8fb8c 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -16,7 +16,7 @@ from scrapy.responsetypes import responsetypes logger = logging.getLogger(__name__) -class DecompressionMiddleware(object): +class DecompressionMiddleware: """ This middleware tries to recognise and extract the possibly compressed responses that may arrive. """ diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 93fe97673..f67961881 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst from scrapy.utils.python import without_none_values -class DefaultHeadersMiddleware(object): +class DefaultHeadersMiddleware: def __init__(self, headers): self._headers = headers diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index 18123cfce..d373a22df 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst from scrapy import signals -class DownloadTimeoutMiddleware(object): +class DownloadTimeoutMiddleware: def __init__(self, timeout=180): self._timeout = timeout diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 7aa7a62bc..089bf0d85 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -9,7 +9,7 @@ from w3lib.http import basic_auth_header from scrapy import signals -class HttpAuthMiddleware(object): +class HttpAuthMiddleware: """Set Basic HTTP Authorization header (http_user and http_pass spider class attributes)""" diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 4e06f8236..6db57bd8b 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -17,7 +17,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.utils.misc import load_object -class HttpCacheMiddleware(object): +class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 0010b2a8f..727c41466 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -15,7 +15,7 @@ except ImportError: pass -class HttpCompressionMiddleware(object): +class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" @classmethod diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 814ce78fe..da89d3e9b 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -7,7 +7,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes -class HttpProxyMiddleware(object): +class HttpProxyMiddleware: def __init__(self, auth_encoding='latin-1'): self.auth_encoding = auth_encoding diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 08cff8a55..09ee8377e 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -11,7 +11,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured logger = logging.getLogger(__name__) -class BaseRedirectMiddleware(object): +class BaseRedirectMiddleware: enabled_setting = 'REDIRECT_ENABLED' diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 7ab5b6e62..bbf5fca05 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -25,7 +25,7 @@ from scrapy.utils.python import global_object_name logger = logging.getLogger(__name__) -class RetryMiddleware(object): +class RetryMiddleware: # IOError is raised by the HttpCompression middleware when trying to # decompress an empty response diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 251706c50..7f18b2bf2 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -16,7 +16,7 @@ from scrapy.utils.misc import load_object logger = logging.getLogger(__name__) -class RobotsTxtMiddleware(object): +class RobotsTxtMiddleware: DOWNLOAD_PRIORITY = 1000 def __init__(self, crawler): diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index ef0aafce0..46a2ad397 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -4,7 +4,7 @@ from scrapy.utils.response import response_httprepr from scrapy.utils.python import global_object_name -class DownloaderStats(object): +class DownloaderStats: def __init__(self, stats): self.stats = stats diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index d24750c69..3ee7bd129 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -3,7 +3,7 @@ from scrapy import signals -class UserAgentMiddleware(object): +class UserAgentMiddleware: """This middleware allows spiders to override the user_agent""" def __init__(self, user_agent='Scrapy'): diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index a36c8304f..d74c8ed36 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -5,7 +5,7 @@ from scrapy.utils.job import job_dir from scrapy.utils.request import referer_str, request_fingerprint -class BaseDupeFilter(object): +class BaseDupeFilter: @classmethod def from_settings(cls, settings): diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 2e20a7180..0cb6cef98 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -21,7 +21,7 @@ __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', 'JsonItemExporter', 'MarshalItemExporter'] -class BaseItemExporter(object): +class BaseItemExporter: def __init__(self, *, dont_fail=False, **kwargs): self._kwargs = kwargs diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 260b2e86e..e3f212bef 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -10,7 +10,7 @@ from scrapy import signals from scrapy.exceptions import NotConfigured -class CloseSpider(object): +class CloseSpider: def __init__(self, crawler): self.crawler = crawler diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 20adfbe4b..389cb65bc 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -6,7 +6,7 @@ from datetime import datetime from scrapy import signals -class CoreStats(object): +class CoreStats: def __init__(self, stats): self.stats = stats diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 625e13249..586399784 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -17,7 +17,7 @@ from scrapy.utils.trackref import format_live_refs logger = logging.getLogger(__name__) -class StackTraceDump(object): +class StackTraceDump: def __init__(self, crawler=None): self.crawler = crawler @@ -52,7 +52,7 @@ class StackTraceDump(object): return dumps -class Debugger(object): +class Debugger: def __init__(self): try: signal.signal(signal.SIGUSR2, self._enter_debugger) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 108b6d35c..998d2a5d1 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -44,7 +44,7 @@ class IFeedStorage(Interface): @implementer(IFeedStorage) -class BlockingFeedStorage(object): +class BlockingFeedStorage: def open(self, spider): path = spider.crawler.settings['FEED_TEMPDIR'] @@ -61,7 +61,7 @@ class BlockingFeedStorage(object): @implementer(IFeedStorage) -class StdoutFeedStorage(object): +class StdoutFeedStorage: def __init__(self, uri, _stdout=None): if not _stdout: @@ -76,7 +76,7 @@ class StdoutFeedStorage(object): @implementer(IFeedStorage) -class FileFeedStorage(object): +class FileFeedStorage: def __init__(self, uri): self.path = file_uri_to_path(uri) @@ -179,7 +179,7 @@ class FTPFeedStorage(BlockingFeedStorage): ) -class _FeedSlot(object): +class _FeedSlot: def __init__(self, file, exporter, storage, uri, format, store_empty): self.file = file self.exporter = exporter @@ -203,7 +203,7 @@ class _FeedSlot(object): self._exporting = False -class FeedExporter(object): +class FeedExporter: @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 91850683f..8546628a8 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -20,7 +20,7 @@ from scrapy.utils.request import request_fingerprint logger = logging.getLogger(__name__) -class DummyPolicy(object): +class DummyPolicy: def __init__(self, settings): self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') @@ -39,7 +39,7 @@ class DummyPolicy(object): return True -class RFC2616Policy(object): +class RFC2616Policy: MAXAGE = 3600 * 24 * 365 # one year @@ -213,7 +213,7 @@ class RFC2616Policy(object): return currentage -class DbmCacheStorage(object): +class DbmCacheStorage: def __init__(self, settings): self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True) @@ -270,7 +270,7 @@ class DbmCacheStorage(object): return request_fingerprint(request) -class FilesystemCacheStorage(object): +class FilesystemCacheStorage: def __init__(self, settings): self.cachedir = data_path(settings['HTTPCACHE_DIR']) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index b685e7b19..0be2831a1 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -8,7 +8,7 @@ from scrapy import signals logger = logging.getLogger(__name__) -class LogStats(object): +class LogStats: """Log basic scraping stats periodically""" def __init__(self, stats, interval=60.0): diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index 892aa8a86..dc8cdbb1d 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -11,7 +11,7 @@ from scrapy.exceptions import NotConfigured from scrapy.utils.trackref import live_refs -class MemoryDebugger(object): +class MemoryDebugger: def __init__(self, stats): self.stats = stats diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 14e0fb32d..a0540bf8f 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -19,7 +19,7 @@ from scrapy.utils.engine import get_engine_status logger = logging.getLogger(__name__) -class MemoryUsage(object): +class MemoryUsage: def __init__(self, crawler): if not crawler.settings.getbool('MEMUSAGE_ENABLED'): diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 2c8e46914..2e5ff569f 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -6,7 +6,7 @@ from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir -class SpiderState(object): +class SpiderState: """Store and load spider state during a scraping job""" def __init__(self, jobdir=None): diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 6a982195d..320f13b29 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -8,7 +8,7 @@ from scrapy import signals from scrapy.mail import MailSender from scrapy.exceptions import NotConfigured -class StatsMailer(object): +class StatsMailer: def __init__(self, stats, recipients, mail): self.stats = stats diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 198d4bbb0..56e5ad2d2 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -6,7 +6,7 @@ from scrapy import signals logger = logging.getLogger(__name__) -class AutoThrottle(object): +class AutoThrottle: def __init__(self, crawler): self.crawler = crawler diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 0903fd4f8..3e810992c 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -5,7 +5,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode -class CookieJar(object): +class CookieJar: def __init__(self, policy=None, check_expired_frequency=10000): self.policy = policy or DefaultCookiePolicy() self.jar = _CookieJar(self.policy) @@ -100,7 +100,7 @@ def potential_domain_matches(domain): return matches + ['.' + d for d in matches] -class _DummyLock(object): +class _DummyLock: def acquire(self): pass @@ -108,7 +108,7 @@ class _DummyLock(object): pass -class WrappedRequest(object): +class WrappedRequest: """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class see http://docs.python.org/library/urllib2.html#urllib2.Request @@ -178,7 +178,7 @@ class WrappedRequest(object): self.request.headers.appendlist(name, value) -class WrappedResponse(object): +class WrappedResponse: def __init__(self, response): self.response = response diff --git a/scrapy/link.py b/scrapy/link.py index a809c5ca4..7cb0765cc 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -6,7 +6,7 @@ its documentation in: docs/topics/link-extractors.rst """ -class Link(object): +class Link: """Link objects represent an extracted link by the LinkExtractor.""" __slots__ = ['url', 'text', 'fragment', 'nofollow'] diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index bdeab3a75..6afe867b5 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -49,7 +49,7 @@ _matches = lambda url, regexs: any(r.search(url) for r in regexs) _is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'} -class FilteringLinkExtractor(object): +class FilteringLinkExtractor: _csstranslator = HTMLTranslator() diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index ab82e1915..fbac1dc59 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -27,7 +27,7 @@ def _nons(tag): return tag -class LxmlParserLinkExtractor(object): +class LxmlParserLinkExtractor: def __init__(self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False): self.scan_tag = tag if callable(tag) else lambda t: t == tag diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 7cf67e29e..21c4fb376 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -25,7 +25,7 @@ def unbound_method(method): return method -class ItemLoader(object): +class ItemLoader: default_item_class = Item default_input_processor = Identity() diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py index 02c625acc..a7be65609 100644 --- a/scrapy/loader/processors.py +++ b/scrapy/loader/processors.py @@ -9,7 +9,7 @@ from scrapy.utils.misc import arg_to_iter from scrapy.loader.common import wrap_loader_context -class MapCompose(object): +class MapCompose: def __init__(self, *functions, **default_loader_context): self.functions = functions @@ -36,7 +36,7 @@ class MapCompose(object): return values -class Compose(object): +class Compose: def __init__(self, *functions, **default_loader_context): self.functions = functions @@ -61,7 +61,7 @@ class Compose(object): return value -class TakeFirst(object): +class TakeFirst: def __call__(self, values): for value in values: @@ -69,13 +69,13 @@ class TakeFirst(object): return value -class Identity(object): +class Identity: def __call__(self, values): return values -class SelectJmes(object): +class SelectJmes: """ Query the input string for the jmespath (given at instantiation), and return the answer @@ -95,7 +95,7 @@ class SelectJmes(object): return self.compiled_path.search(value) -class Join(object): +class Join: def __init__(self, separator=u' '): self.separator = separator diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 14cec44a6..219145f13 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -14,7 +14,7 @@ DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s" DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s" -class LogFormatter(object): +class LogFormatter: """Class for generating log messages for different actions. All methods must return a dictionary listing the parameters ``level``, ``msg`` diff --git a/scrapy/mail.py b/scrapy/mail.py index b2a24a3db..9d186f4f3 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -27,7 +27,7 @@ def _to_bytes_or_none(text): return to_bytes(text) -class MailSender(object): +class MailSender: def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): self.smtphost = smtphost diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 53fa435bb..5040378ea 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -9,7 +9,7 @@ from scrapy.utils.defer import process_parallel, process_chain, process_chain_bo logger = logging.getLogger(__name__) -class MiddlewareManager(object): +class MiddlewareManager: """Base class for implementing middleware managers""" component_name = 'foo middleware' diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 9b7445755..101bf5fbc 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -37,7 +37,7 @@ class FileException(Exception): """General media error exception""" -class FSFilesStore(object): +class FSFilesStore: def __init__(self, basedir): if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -75,7 +75,7 @@ class FSFilesStore(object): seen.add(dirname) -class S3FilesStore(object): +class S3FilesStore: AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None AWS_ENDPOINT_URL = None @@ -213,7 +213,7 @@ class S3FilesStore(object): return extra -class GCSFilesStore(object): +class GCSFilesStore: GCS_PROJECT_ID = None @@ -259,7 +259,7 @@ class GCSFilesStore(object): ) -class FTPFilesStore(object): +class FTPFilesStore: FTP_USERNAME = None FTP_PASSWORD = None diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index c174addf9..562d9ee32 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -14,11 +14,11 @@ from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) -class MediaPipeline(object): +class MediaPipeline: LOG_FAILED_RESULTS = True - class SpiderInfo(object): + class SpiderInfo: def __init__(self, spider): self.spider = spider self.downloading = set() diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 1afe58dab..e13d389ee 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -110,7 +110,7 @@ class ScrapyPriorityQueue: return sum(len(x) for x in self.queues.values()) if self.queues else 0 -class DownloaderInterface(object): +class DownloaderInterface: def __init__(self, crawler): self.downloader = crawler.engine.downloader @@ -129,8 +129,8 @@ class DownloaderInterface(object): return len(self.downloader.slots[slot].active) -class DownloaderAwarePriorityQueue(object): - """ PriorityQueue which takes Downloader activity in account: +class DownloaderAwarePriorityQueue: + """ PriorityQueue which takes Downloader activity into account: domains (slots) with the least amount of active downloads are dequeued first. """ diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 64bf93e86..ad89d9d22 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -11,7 +11,7 @@ from scrapy.utils.misc import load_object from scrapy.utils.python import binary_is_text, to_bytes, to_unicode -class ResponseTypes(object): +class ResponseTypes: CLASSES = { 'text/html': 'scrapy.http.HtmlResponse', diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b6133619c..b9a13c018 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -28,7 +28,7 @@ def get_settings_priority(priority): return priority -class SettingsAttribute(object): +class SettingsAttribute: """Class for storing data related to settings attributes. diff --git a/scrapy/shell.py b/scrapy/shell.py index a5e140484..08ce89481 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -24,7 +24,7 @@ from scrapy.utils.conf import get_config from scrapy.utils.console import DEFAULT_PYTHON_SHELLS -class Shell(object): +class Shell: relevant_classes = (Crawler, Spider, Request, Response, BaseItem, Settings) diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 481d97e9a..54eb7cfa3 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -2,7 +2,7 @@ from pydispatch import dispatcher from scrapy.utils import signal as _signal -class SignalManager(object): +class SignalManager: def __init__(self, sender=dispatcher.Anonymous): self.sender = sender diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 048e84e4f..3be5aaec5 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -11,7 +11,7 @@ from scrapy.utils.spider import iter_spider_classes @implementer(ISpiderLoader) -class SpiderLoader(object): +class SpiderLoader: """ SpiderLoader is a class which locates and loads spiders in a Scrapy project. diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 34a87f2df..fa7f5bef9 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -11,7 +11,7 @@ from scrapy.http import Request logger = logging.getLogger(__name__) -class DepthMiddleware(object): +class DepthMiddleware: def __init__(self, maxdepth, stats, verbose_stats=False, prio=1): self.maxdepth = maxdepth diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index def697c2b..375042340 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -18,7 +18,7 @@ class HttpError(IgnoreRequest): super(HttpError, self).__init__(*args, **kwargs) -class HttpErrorMiddleware(object): +class HttpErrorMiddleware: @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 2fab572e6..a006f3177 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -14,7 +14,7 @@ from scrapy.utils.httpobj import urlparse_cached logger = logging.getLogger(__name__) -class OffsiteMiddleware(object): +class OffsiteMiddleware: def __init__(self, stats): self.stats = stats diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index dce2b3598..3784de885 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -28,7 +28,7 @@ POLICY_UNSAFE_URL = "unsafe-url" POLICY_SCRAPY_DEFAULT = "scrapy-default" -class ReferrerPolicy(object): +class ReferrerPolicy: NOREFERRER_SCHEMES = LOCAL_SCHEMES @@ -284,7 +284,7 @@ def _load_policy_class(policy, warning_only=False): return None -class RefererMiddleware(object): +class RefererMiddleware: def __init__(self, settings=None): self.default_policy = DefaultReferrerPolicy diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index a904635d8..5be1f80cb 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -12,7 +12,7 @@ from scrapy.exceptions import NotConfigured logger = logging.getLogger(__name__) -class UrlLengthMiddleware(object): +class UrlLengthMiddleware: def __init__(self, maxlength): self.maxlength = maxlength diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index a2c364c0e..d76a96451 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -34,7 +34,7 @@ def _get_method(method, spider): _default_link_extractor = LinkExtractor() -class Rule(object): +class Rule: def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None, errback=None): diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index f0bfaed34..579c60180 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -7,7 +7,7 @@ import logging logger = logging.getLogger(__name__) -class StatsCollector(object): +class StatsCollector: def __init__(self, crawler): self._dump = crawler.settings.getbool('STATS_DUMP') diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 97b5db2e1..b3e58ff94 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -8,7 +8,7 @@ from scrapy import signals -class ${ProjectName}SpiderMiddleware(object): +class ${ProjectName}SpiderMiddleware: # Not all methods need to be defined. If a method is not defined, # scrapy acts as if the spider middleware does not modify the # passed objects. @@ -56,7 +56,7 @@ class ${ProjectName}SpiderMiddleware(object): spider.logger.info('Spider opened: %s' % spider.name) -class ${ProjectName}DownloaderMiddleware(object): +class ${ProjectName}DownloaderMiddleware: # Not all methods need to be defined. If a method is not defined, # scrapy acts as if the downloader middleware does not modify the # passed objects. diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index fb641d447..4876526a9 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -6,6 +6,6 @@ # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html -class ${ProjectName}Pipeline(object): +class ${ProjectName}Pipeline: def process_item(self, item, spider): return item diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 175f92d77..f59f4cc55 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -109,7 +109,7 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): return None # key is not weak-referenceable, it's not cached -class SequenceExclude(object): +class SequenceExclude: """Object to test if an item is NOT within some sequence.""" def __init__(self, seq): diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 2d3db431d..69334a918 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -144,7 +144,7 @@ def method_is_overridden(subclass, base_class, method_name): Return True if a method named ``method_name`` of a ``base_class`` is overridden in a ``subclass``. - >>> class Base(object): + >>> class Base: ... def foo(self): ... pass >>> class Sub1(Base): diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 7849174fb..b71419111 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -52,7 +52,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): yield xs.xpath(selxpath)[0] -class _StreamReader(object): +class _StreamReader: def __init__(self, obj): self._ptr = 0 diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index afef2c93f..5998dc33b 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -152,7 +152,7 @@ def log_scrapy_info(settings): logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) -class StreamLogger(object): +class StreamLogger: """Fake file-like stream object that redirects writes to a logger instance Taken from: diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index e95a4648e..3d02d9478 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -223,7 +223,7 @@ def get_spec(func): >>> get_spec(re.match) (['pattern', 'string'], {'flags': 0}) - >>> class Test(object): + >>> class Test: ... def __call__(self, val): ... pass ... def method(self, val, flags=0): @@ -272,7 +272,7 @@ def equal_attributes(obj1, obj2, attributes): return True -class WeakKeyCache(object): +class WeakKeyCache: def __init__(self, default_factory): self.default_factory = default_factory diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 17d6b2857..5308812d6 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -24,7 +24,7 @@ def listen_tcp(portrange, host, factory): raise -class CallLaterOnce(object): +class CallLaterOnce: """Schedule a function to be called in the next reactor loop, but only if it hasn't been already scheduled since the last time it ran. """ diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 2f10cf4de..a57a0c291 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -10,7 +10,7 @@ from urllib.parse import urljoin import lxml.etree -class Sitemap(object): +class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 37803b287..a63c9a942 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -4,7 +4,7 @@ import os from twisted.internet import defer, protocol -class ProcessTest(object): +class ProcessTest: command = None prefix = [sys.executable, '-m', 'scrapy.cmdline'] diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 9e1598805..66930ad2c 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -3,7 +3,7 @@ from urllib.parse import urljoin from twisted.web import server, resource, static, util -class SiteTest(object): +class SiteTest: def setUp(self): from twisted.internet import reactor diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 4842b95df..baed5c536 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -19,9 +19,8 @@ NoneType = type(None) live_refs = defaultdict(weakref.WeakKeyDictionary) -class object_ref(object): - """Inherit from this class (instead of object) to a keep a record of live - instances""" +class object_ref: + """Inherit from this class to a keep a record of live instances""" __slots__ = () diff --git a/tests/pipelines.py b/tests/pipelines.py index de4894c32..cf677cc17 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -3,7 +3,7 @@ Some pipelines used for testing """ -class ZeroDivisionErrorPipeline(object): +class ZeroDivisionErrorPipeline: def open_spider(self, spider): a = 1 / 0 @@ -12,7 +12,7 @@ class ZeroDivisionErrorPipeline(object): return item -class ProcessWithZeroDivisionErrorPipiline(object): +class ProcessWithZeroDivisionErrorPipiline: def process_item(self, item, spider): 1 / 0 diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py index c64e87d81..6504b4d2c 100644 --- a/tests/test_cmdline/extensions.py +++ b/tests/test_cmdline/extensions.py @@ -1,7 +1,7 @@ """A test extension used to check the settings loading order""" -class TestExtension(object): +class TestExtension: def __init__(self, settings): settings.set('TEST1', "%s + %s" % (settings['TEST1'], 'started')) @@ -11,5 +11,5 @@ class TestExtension(object): return cls(crawler.settings) -class DummyExtension(object): +class DummyExtension: pass diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py index ce916f699..bd1f9cd8c 100644 --- a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py +++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py @@ -1,4 +1,4 @@ -class TestSpiderPipeline(object): +class TestSpiderPipeline: def open_spider(self, spider): pass @@ -7,7 +7,7 @@ class TestSpiderPipeline(object): return item -class TestSpiderExceptionPipeline(object): +class TestSpiderExceptionPipeline: def open_spider(self, spider): raise Exception('exception') diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 5bf92b71a..85a24d0bc 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -89,7 +89,7 @@ class MyBadCrawlSpider(CrawlSpider): f.write(""" import logging -class MyPipeline(object): +class MyPipeline: component_name = 'my_pipeline' def process_item(self, item, spider): diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 11d41c1fe..d1ce80f9d 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -25,7 +25,7 @@ class TestItem(Item): url = Field() -class ResponseMock(object): +class ResponseMock: url = 'http://scrapy.org' diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 37a069611..169e763f0 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -126,7 +126,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0) -class SpiderLoaderWithWrongInterface(object): +class SpiderLoaderWithWrongInterface: def unneeded_method(self): pass diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 3dd4f2351..a9190c62b 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -106,7 +106,7 @@ class ResponseFromProcessRequestTest(ManagerTestCase): def test_download_func_not_called(self): resp = Response('http://example.com/index.html') - class ResponseMiddleware(object): + class ResponseMiddleware: def process_request(self, request, spider): return resp diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 9e24d86dd..ea0e664be 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -35,7 +35,7 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter): return df -class DirectDupeFilter(object): +class DirectDupeFilter: method = 'n/a' diff --git a/tests/test_engine.py b/tests/test_engine.py index 25dee7c1f..5b7a4e676 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -96,7 +96,7 @@ def start_test_site(debug=False): return port -class CrawlerRun(object): +class CrawlerRun: """A class to run the crawler and keep track of events occurred""" def __init__(self, spider_class): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 08e8dfc41..c5589e52f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -373,7 +373,7 @@ class StdoutFeedStorageTest(unittest.TestCase): self.assertEqual(out.getvalue(), b"content") -class FromCrawlerMixin(object): +class FromCrawlerMixin: init_with_crawler = False @classmethod diff --git a/tests/test_item.py b/tests/test_item.py index f70632d57..4017f6e84 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -213,7 +213,7 @@ class ItemTest(unittest.TestCase): class B(A): pass - class C(object): + class C: fields = {'load': Field(default='C')} not_allowed = Field(default='not_allowed') save = Field(default='C') diff --git a/tests/test_loader.py b/tests/test_loader.py index 579a85ff6..701d568dc 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -456,7 +456,7 @@ class BasicItemLoaderTest(unittest.TestCase): [u'marta', u'other'], Compose(float)) -class InitializationTestMixin(object): +class InitializationTestMixin: item_class = None diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index bf9fbe5e4..cd6cb8016 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -170,7 +170,7 @@ class SkipMessagesLogFormatter(LogFormatter): return None -class DropSomeItemsPipeline(object): +class DropSomeItemsPipeline: drop = True def process_item(self, item, spider): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index ebf817c7e..3af514bb0 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -5,7 +5,7 @@ from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager -class M1(object): +class M1: def open_spider(self, spider): pass @@ -17,7 +17,7 @@ class M1(object): pass -class M2(object): +class M2: def open_spider(self, spider): pass @@ -28,13 +28,13 @@ class M2(object): pass -class M3(object): +class M3: def process(self, response, request, spider): pass -class MOff(object): +class MOff: def open_spider(self, spider): pass diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index a5cdc0de0..a3ddd50f4 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -8,7 +8,7 @@ from tests.spiders import MockServerSpider from tests.mockserver import MockServer -class InjectArgumentsDownloaderMiddleware(object): +class InjectArgumentsDownloaderMiddleware: """ Make sure downloader middlewares are able to update the keyword arguments """ @@ -23,7 +23,7 @@ class InjectArgumentsDownloaderMiddleware(object): return response -class InjectArgumentsSpiderMiddleware(object): +class InjectArgumentsSpiderMiddleware: """ Make sure spider middlewares are able to update the keyword arguments """ diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 13c297084..00568aee9 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -20,7 +20,7 @@ MockEngine = collections.namedtuple('MockEngine', ['downloader']) MockSlot = collections.namedtuple('MockSlot', ['active']) -class MockDownloader(object): +class MockDownloader: def __init__(self): self.slots = dict() @@ -57,7 +57,7 @@ class MockCrawler(Crawler): self.engine = MockEngine(downloader=MockDownloader()) -class SchedulerHandler(object): +class SchedulerHandler: priority_queue_cls = None jobdir = None @@ -245,7 +245,7 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots): return True -class DownloaderAwareSchedulerTestMixin(object): +class DownloaderAwareSchedulerTestMixin: priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' reopen = False diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 7cc17600c..4c6ede70b 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -47,7 +47,7 @@ class TestRefererMiddleware(TestCase): self.assertEqual(out[0].headers.get('Referer'), referrer) -class MixinDefault(object): +class MixinDefault: """ Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade @@ -72,7 +72,7 @@ class MixinDefault(object): ] -class MixinNoReferrer(object): +class MixinNoReferrer: scenarii = [ ('https://example.com/page.html', 'https://example.com/', None), ('http://www.example.com/', 'https://scrapy.org/', None), @@ -82,7 +82,7 @@ class MixinNoReferrer(object): ] -class MixinNoReferrerWhenDowngrade(object): +class MixinNoReferrerWhenDowngrade: scenarii = [ # TLS to TLS: send non-empty referrer ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), @@ -111,7 +111,7 @@ class MixinNoReferrerWhenDowngrade(object): ] -class MixinSameOrigin(object): +class MixinSameOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), @@ -144,7 +144,7 @@ class MixinSameOrigin(object): ] -class MixinOrigin(object): +class MixinOrigin: scenarii = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), @@ -157,7 +157,7 @@ class MixinOrigin(object): ] -class MixinStrictOrigin(object): +class MixinStrictOrigin: scenarii = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), @@ -176,7 +176,7 @@ class MixinStrictOrigin(object): ] -class MixinOriginWhenCrossOrigin(object): +class MixinOriginWhenCrossOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), @@ -211,7 +211,7 @@ class MixinOriginWhenCrossOrigin(object): ] -class MixinStrictOriginWhenCrossOrigin(object): +class MixinStrictOriginWhenCrossOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), @@ -255,7 +255,7 @@ class MixinStrictOriginWhenCrossOrigin(object): ] -class MixinUnsafeUrl(object): +class MixinUnsafeUrl: scenarii = [ # TLS to TLS: send referrer ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), diff --git a/tests/test_squeues.py b/tests/test_squeues.py index f0f3dd4c6..5ad8035f7 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -36,7 +36,7 @@ def nonserializable_object_test(self): self.assertRaises(ValueError, q.push, lambda x: x) else: # Use a different unpickleable object - class A(object): + class A: pass a = A() diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index b3a90d314..b17e17f2f 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -11,7 +11,7 @@ class MyWarning(UserWarning): pass -class SomeBaseClass(object): +class SomeBaseClass: pass @@ -155,7 +155,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): class OutdatedUserClass1a(DeprecatedName): pass - class UnrelatedClass(object): + class UnrelatedClass: pass class OldStyleClass: @@ -191,7 +191,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): class OutdatedUserClass2a(DeprecatedName): pass - class UnrelatedClass(object): + class UnrelatedClass: pass class OldStyleClass: diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index ec5b4c596..8cb8df15b 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -73,7 +73,7 @@ class ToBytesTest(unittest.TestCase): class MemoizedMethodTest(unittest.TestCase): def test_memoizemethod_noargs(self): - class A(object): + class A: @memoizemethod_noargs def cached(self): @@ -153,7 +153,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) def test_weakkeycache(self): - class _Weakme(object): + class _Weakme: pass _values = count() @@ -176,14 +176,14 @@ class UtilsPythonTestCase(unittest.TestCase): def f2(a, b=None, c=None): pass - class A(object): + class A: def __init__(self, a, b, c): pass def method(self, a, b, c): pass - class Callable(object): + class Callable: def __call__(self, a, b, c): pass diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 06d9c004c..c7572f02c 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -126,7 +126,7 @@ class RequestSerializationTest(unittest.TestCase): self.assertRaises(ValueError, request_to_dict, r) -class TestSpiderMixin(object): +class TestSpiderMixin: def __mixin_callback(self, response): pass From 533131a30fa944688dc54dd82a581739d1ed247c Mon Sep 17 00:00:00 2001 From: sakshamb2113 <44064539+sakshamb2113@users.noreply.github.com> Date: Tue, 17 Mar 2020 14:42:49 +0530 Subject: [PATCH 127/281] Remove Guppy-specific code and documentation (#4343) --- docs/topics/leaks.rst | 64 +++---------------------------------- scrapy/extensions/telnet.py | 6 ---- 2 files changed, 5 insertions(+), 65 deletions(-) diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 4ee447065..ceb708c7e 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -17,8 +17,8 @@ what is known as a "memory leak". To help debugging memory leaks, Scrapy provides a built-in mechanism for tracking objects references called :ref:`trackref `, -and you can also use a third-party library called :ref:`Guppy -` for more advanced memory debugging (see below for more +and you can also use a third-party library called :ref:`muppy +` for more advanced memory debugging (see below for more info). Both mechanisms must be used from the :ref:`Telnet Console `. @@ -193,9 +193,9 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module. ``None`` if none is found. Use :func:`print_live_refs` first to get a list of all tracked live objects per class name. -.. _topics-leaks-guppy: +.. _topics-leaks-muppy: -Debugging memory leaks with Guppy +Debugging memory leaks with muppy ================================= ``trackref`` provides a very convenient mechanism for tracking down memory @@ -203,63 +203,9 @@ leaks, but it only keeps track of the objects that are more likely to cause memory leaks (Requests, Responses, Items, and Selectors). However, there are other cases where the memory leaks could come from other (more or less obscure) objects. If this is your case, and you can't find your leaks using ``trackref``, -you still have another resource: the `Guppy library`_. -If you're using Python3, see :ref:`topics-leaks-muppy`. +you still have another resource: the muppy library. -.. _Guppy library: https://pypi.org/project/guppy/ -If you use ``pip``, you can install Guppy with the following command:: - - pip install guppy - -The telnet console also comes with a built-in shortcut (``hpy``) for accessing -Guppy heap objects. Here's an example to view all Python objects available in -the heap using Guppy: - ->>> x = hpy.heap() ->>> x.bytype -Partition of a set of 297033 objects. Total size = 52587824 bytes. - Index Count % Size % Cumulative % Type - 0 22307 8 16423880 31 16423880 31 dict - 1 122285 41 12441544 24 28865424 55 str - 2 68346 23 5966696 11 34832120 66 tuple - 3 227 0 5836528 11 40668648 77 unicode - 4 2461 1 2222272 4 42890920 82 type - 5 16870 6 2024400 4 44915320 85 function - 6 13949 5 1673880 3 46589200 89 types.CodeType - 7 13422 5 1653104 3 48242304 92 list - 8 3735 1 1173680 2 49415984 94 _sre.SRE_Pattern - 9 1209 0 456936 1 49872920 95 scrapy.http.headers.Headers -<1676 more rows. Type e.g. '_.more' to view.> - -You can see that most space is used by dicts. Then, if you want to see from -which attribute those dicts are referenced, you could do: - ->>> x.bytype[0].byvia -Partition of a set of 22307 objects. Total size = 16423880 bytes. - Index Count % Size % Cumulative % Referred Via: - 0 10982 49 9416336 57 9416336 57 '.__dict__' - 1 1820 8 2681504 16 12097840 74 '.__dict__', '.func_globals' - 2 3097 14 1122904 7 13220744 80 - 3 990 4 277200 2 13497944 82 "['cookies']" - 4 987 4 276360 2 13774304 84 "['cache']" - 5 985 4 275800 2 14050104 86 "['meta']" - 6 897 4 251160 2 14301264 87 '[2]' - 7 1 0 196888 1 14498152 88 "['moduleDict']", "['modules']" - 8 672 3 188160 1 14686312 89 "['cb_kwargs']" - 9 27 0 155016 1 14841328 90 '[1]' -<333 more rows. Type e.g. '_.more' to view.> - -As you can see, the Guppy module is very powerful but also requires some deep -knowledge about Python internals. For more info about Guppy, refer to the -`Guppy documentation`_. - -.. _Guppy documentation: http://guppy-pe.sourceforge.net/ - -.. _topics-leaks-muppy: - -Debugging memory leaks with muppy -================================= You can use muppy from `Pympler`_. .. _Pympler: https://pypi.org/project/Pympler/ diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 26b214ee2..04ffd7235 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -26,11 +26,6 @@ from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp from scrapy.utils.decorators import defers -try: - import guppy - hpy = guppy.hpy() -except ImportError: - hpy = None logger = logging.getLogger(__name__) @@ -110,7 +105,6 @@ class TelnetConsole(protocol.ServerFactory): 'est': lambda: print_engine_status(self.crawler.engine), 'p': pprint.pprint, 'prefs': print_live_refs, - 'hpy': hpy, 'help': "This is Scrapy telnet console. For more info see: " "https://docs.scrapy.org/en/latest/topics/telnetconsole.html", } From 9ab45325ff6746f5c5325940fbbff1e020dd999a Mon Sep 17 00:00:00 2001 From: "Matsievskiy S.V" Date: Tue, 17 Mar 2020 18:45:00 +0300 Subject: [PATCH 128/281] edit zsh completion - Fix bug introduced in https://github.com/scrapy/scrapy/pull/4291 - Enforce `[command] [options] [arguments]` syntax. Do not allow options after arguments - Exclude already used option aliases from completion list --- extras/scrapy_zsh_completion | 76 ++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion index 33f46eda8..e2f2dc82b 100644 --- a/extras/scrapy_zsh_completion +++ b/extras/scrapy_zsh_completion @@ -14,40 +14,40 @@ _scrapy() { ;; args) case $words[1] in - bench) + (bench) _scrapy_glb_opts ;; - fetch) + (fetch) local options=( '--headers[print response HTTP headers instead of body]' '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]' - '--spider[use this spider]:spider:_scrapy_spiders' + '--spider=[use this spider]:spider:_scrapy_spiders' '1::URL:_httpie_urls' ) _scrapy_glb_opts $options ;; - genspider) + (genspider) local options=( - {-l,--list}'[List available templates]' - {-e,--edit}'[Edit spider after creating it]' + {'(--list)-l','(-l)--list'}'[List available templates]' + {'(--edit)-e','(-e)--edit'}'[Edit spider after creating it]' '--force[If the spider already exists, overwrite it with the template]' - {-d,--dump=}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)' - {-t,--template=}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)' + {'(--dump)-d','(-d)--dump='}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)' + {'(--template)-t','(-t)--template='}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)' '1:name:(NAME)' '2:domain:_httpie_urls' ) _scrapy_glb_opts $options ;; - runspider) + (runspider) local options=( - {-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files' - {-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)' + {'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files' + {'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)' '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' '1:spider file:_files -g \*.py' ) _scrapy_glb_opts $options ;; - settings) + (settings) local options=( '--get=[print raw setting value]:option:(SETTING)' '--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)' @@ -57,77 +57,77 @@ _scrapy() { ) _scrapy_glb_opts $options ;; - shell) + (shell) local options=( '-c[evaluate the code in the shell, print the result and exit]:code:(CODE)' '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]' - '--spider[use this spider]:spider:_scrapy_spiders' + '--spider=[use this spider]:spider:_scrapy_spiders' '::file:_files -g \*.html' '::URL:_httpie_urls' ) _scrapy_glb_opts $options ;; - startproject) + (startproject) local options=( '1:name:(NAME)' '2:dir:_dir_list' ) _scrapy_glb_opts $options ;; - version) + (version) local options=( - {-v,--verbose}'[also display twisted/python/platform info (useful for bug reports)]' + {'(--verbose)-v','(-v)--verbose'}'[also display twisted/python/platform info (useful for bug reports)]' ) _scrapy_glb_opts $options ;; - view) + (view) local options=( '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]' - '--spider[use this spider]:spider:_scrapy_spiders' + '--spider=[use this spider]:spider:_scrapy_spiders' '1:URL:_httpie_urls' ) _scrapy_glb_opts $options ;; - check) + (check) local options=( - '(- 1 *)'{-l,--list}'[only list contracts, without checking them]' - {-v,--verbose}'[print contract tests for all spiders]' + {'(--list)-l','(-l)--list'}'[only list contracts, without checking them]' + {'(--verbose)-v','(-v)--verbose'}'[print contract tests for all spiders]' '1:spider:_scrapy_spiders' ) _scrapy_glb_opts $options ;; - crawl) + (crawl) local options=( - {-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files' - {-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)' + {'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files' + {'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)' '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' '1:spider:_scrapy_spiders' ) _scrapy_glb_opts $options ;; - edit) + (edit) local options=( - '1:spider:_scrapy_spiders' + '1:spider:_scrapy_spiders' ) _scrapy_glb_opts $options ;; - list) + (list) _scrapy_glb_opts ;; - parse) + (parse) local options=( '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' - '--spider[use this spider without looking for one]:spider:_scrapy_spiders' + '--spider=[use this spider without looking for one]:spider:_scrapy_spiders' '--pipelines[process items through pipelines]' "--nolinks[don't show links to follow (extracted requests)]" "--noitems[don't show scraped items]" '--nocolour[avoid using pygments to colorize the output]' - {-r,--rules}'[use CrawlSpider rules to discover the callback]' - {-c,--callback=}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)' - {-m,--meta=}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)' + {'(--rules)-r','(-r)--rules'}'[use CrawlSpider rules to discover the callback]' + {'(--callback)-c','(-c)--callback'}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)' + {'(--meta)-m','(-m)--meta='}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)' '--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)' - {-d,--depth=}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)' - {-v,--verbose}'[print each depth level one by one]' + {'(--depth)-d','(-d)--depth='}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)' + {'(--verbose)-v','(-v)--verbose'}'[print each depth level one by one]' '1:URL:_httpie_urls' ) _scrapy_glb_opts $options @@ -162,7 +162,7 @@ _scrapy_cmds() { if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then commands=(${commands[@]} ${project_commands[@]}) fi - _describe -t common-commands 'common commands' commands + _describe -t common-commands 'common commands' commands && ret=0 } _scrapy_glb_opts() { @@ -172,13 +172,13 @@ _scrapy_glb_opts() { '(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files' '--pidfile=[write process ID to FILE]:file:_files' '--profile=[write python cProfile stats to FILE]:file:_files' - '(--nolog)'{-L,--loglevel=}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)' + {'(--loglevel --nolog)-L','(-L --nolog)--loglevel='}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)' '(-L --loglevel --logfile)--nolog[disable logging completely]' '--pdb[enable pdb on failure]' '*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)' ) options=(${options[@]} "$@") - _arguments $options + _arguments -A "-*" $options && ret=0 } _httpie_urls() { From 6c747953f97d60e5668cff1af3f091979ceaaa83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 18 Mar 2020 18:33:41 +0100 Subject: [PATCH 129/281] Cover 2.0.1 in the release notes (#4437) --- docs/news.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index dd5e00223..e9b7140cd 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,22 @@ Release notes ============= +.. _release-2.0.1: + +Scrapy 2.0.1 (2020-03-18) +------------------------- + +* :meth:`Response.follow_all ` now supports + an empty URL iterable as input (:issue:`4408`, :issue:`4420`) + +* Removed top-level :mod:`~twisted.internet.reactor` imports to prevent + errors about the wrong Twisted reactor being installed when setting a + different Twisted reactor using :setting:`TWISTED_REACTOR` (:issue:`4401`, + :issue:`4406`) + +* Fixed tests (:issue:`4422`) + + .. _release-2.0.0: Scrapy 2.0.0 (2020-03-03) From ca08e04198b94bd9583704f86316b57af3408adc Mon Sep 17 00:00:00 2001 From: Aditya Date: Fri, 20 Mar 2020 02:31:35 +0530 Subject: [PATCH 130/281] [docs] update redirect links python2 -> python3 --- docs/topics/downloader-middleware.rst | 5 ++--- docs/topics/email.rst | 2 +- docs/topics/exporters.rst | 8 ++++---- docs/topics/extensions.rst | 2 +- docs/topics/items.rst | 6 +++--- docs/topics/logging.rst | 16 ++++++++-------- docs/topics/request-response.rst | 10 +++++----- docs/topics/selectors.rst | 2 +- docs/topics/settings.rst | 6 +++--- docs/topics/spider-middleware.rst | 6 +++--- 10 files changed, 31 insertions(+), 32 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 73648994d..61a3806fb 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -739,7 +739,7 @@ HttpProxyMiddleware This middleware sets the HTTP proxy to use for requests, by setting the ``proxy`` meta value for :class:`~scrapy.http.Request` objects. - Like the Python standard library modules `urllib`_ and `urllib2`_, it obeys + Like the Python standard library module `urllib.request`_, it obeys the following environment variables: * ``http_proxy`` @@ -751,8 +751,7 @@ HttpProxyMiddleware Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` environment variables, and it will also ignore ``no_proxy`` environment variable. -.. _urllib: https://docs.python.org/2/library/urllib.html -.. _urllib2: https://docs.python.org/2/library/urllib2.html +.. _urllib.request: https://docs.python.org/3/library/urllib.request.html RedirectMiddleware ------------------ diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 72bf52227..aed3deb2e 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -15,7 +15,7 @@ IO of the crawler. It also provides a simple API for sending attachments and it's very easy to configure, with a few :ref:`settings `. -.. _smtplib: https://docs.python.org/2/library/smtplib.html +.. _smtplib: https://docs.python.org/3/library/smtplib.html Quick example ============= diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index e52682690..4ba8714bd 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -320,7 +320,7 @@ CsvItemExporter Color TV,1200 DVD player,200 -.. _csv.writer: https://docs.python.org/2/library/csv.html#csv.writer +.. _csv.writer: https://docs.python.org/3/library/csv.html#csv.writer PickleItemExporter ------------------ @@ -342,7 +342,7 @@ PickleItemExporter Pickle isn't a human readable format, so no output examples are provided. -.. _pickle module documentation: https://docs.python.org/2/library/pickle.html +.. _pickle module documentation: https://docs.python.org/3/library/pickle.html PprintItemExporter ------------------ @@ -393,7 +393,7 @@ JsonItemExporter stream-friendly format, consider using :class:`JsonLinesItemExporter` instead, or splitting the output in multiple chunks. -.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder +.. _JSONEncoder: https://docs.python.org/3/library/json.html#json.JSONEncoder JsonLinesItemExporter --------------------- @@ -417,7 +417,7 @@ JsonLinesItemExporter Unlike the one produced by :class:`JsonItemExporter`, the format produced by this exporter is well suited for serializing large amounts of data. -.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder +.. _JSONEncoder: https://docs.python.org/3/library/json.html#json.JSONEncoder MarshalItemExporter ------------------- diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 94fd2e36e..f57e37e6f 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -372,5 +372,5 @@ For more info see `Debugging in Python`_. This extension only works on POSIX-compliant platforms (i.e. not Windows). -.. _Python debugger: https://docs.python.org/2/library/pdb.html +.. _Python debugger: https://docs.python.org/3/library/pdb.html .. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/ diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 44643cb67..36731571e 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -24,7 +24,7 @@ serialization can be customized using Item fields metadata, :mod:`trackref` tracks Item instances to help find memory leaks (see :ref:`topics-leaks-trackrefs`), etc. -.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict +.. _dictionary-like: https://docs.python.org/3/library/stdtypes.html#dict .. _topics-items-declaring: @@ -249,7 +249,7 @@ Item objects :class:`Field` objects used in the :ref:`Item declaration `. -.. _dict API: https://docs.python.org/2/library/stdtypes.html#dict +.. _dict API: https://docs.python.org/3/library/stdtypes.html#dict Field objects ============= @@ -262,7 +262,7 @@ Field objects to support the :ref:`item declaration syntax ` based on class attributes. -.. _dict: https://docs.python.org/2/library/stdtypes.html#dict +.. _dict: https://docs.python.org/3/library/stdtypes.html#dict Other classes related to Item diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index d4d22d889..a85e1a769 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -83,10 +83,10 @@ path:: .. seealso:: - Module logging, `HowTo `_ + Module logging, `HowTo `_ Basic Logging Tutorial - Module logging, `Loggers `_ + Module logging, `Loggers `_ Further documentation on loggers .. _topics-logging-from-spiders: @@ -166,13 +166,13 @@ possible levels listed in :ref:`topics-logging-levels`. :setting:`LOG_FORMAT` and :setting:`LOG_DATEFORMAT` specify formatting strings used as layouts for all messages. Those strings can contain any placeholders listed in `logging's logrecord attributes docs -`_ and +`_ and `datetime's strftime and strptime directives -`_ +`_ respectively. If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy -component that prints the log. It is unset by default, hence logs contain the +component that prints the log. It is unset by default, hence logs contain the Scrapy component responsible for that log output. Command-line options @@ -190,7 +190,7 @@ to override some of the Scrapy settings regarding logging. .. seealso:: - Module `logging.handlers `_ + Module `logging.handlers `_ Further documentation on available handlers .. _custom-log-formats: @@ -201,7 +201,7 @@ Custom Log Formats A custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class and making :setting:`LOG_FORMATTER` point to your new class. - + .. autoclass:: scrapy.logformatter.LogFormatter :members: @@ -276,6 +276,6 @@ scrapy.utils.log module Refer to :ref:`run-from-script` for more details about using Scrapy this way. -.. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig +.. _logging.basicConfig(): https://docs.python.org/3/library/logging.html#logging.basicConfig diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index b2a60ff39..6c5a08409 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -189,7 +189,7 @@ Request objects ``copy()`` or ``replace()`` methods, and can also be accessed, in your spider, from the ``response.cb_kwargs`` attribute. - .. _shallow copied: https://docs.python.org/2/library/copy.html + .. _shallow copied: https://docs.python.org/3/library/copy.html .. method:: Request.copy() @@ -706,7 +706,7 @@ Response objects A :class:`twisted.internet.ssl.Certificate` object representing the server's SSL certificate. - + Only populated for ``https`` responses, ``None`` otherwise. .. method:: Response.copy() @@ -724,17 +724,17 @@ Response objects Constructs an absolute url by combining the Response's :attr:`url` with a possible relative url. - This is a wrapper over `urlparse.urljoin`_, it's merely an alias for + This is a wrapper over `urllib.parse.urljoin`_, it's merely an alias for making this call:: - urlparse.urljoin(response.url, url) + urllib.parse.urljoin(response.url, url) .. automethod:: Response.follow .. automethod:: Response.follow_all -.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin +.. _urllib.parse.urljoin: https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urljoin .. _topics-request-response-ref-response-subclasses: diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 1f7802c98..0f90b28c0 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -36,7 +36,7 @@ defines selectors to associate those styles with specific HTML elements. .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ .. _lxml: https://lxml.de/ -.. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html +.. _ElementTree: https://docs.python.org/3/library/xml.etree.elementtree.html .. _XPath: https://www.w3.org/TR/xpath/all/ .. _CSS: https://www.w3.org/TR/selectors .. _parsel: https://parsel.readthedocs.io/en/latest/ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index dc6843d75..d78a6253e 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -28,7 +28,7 @@ The value of ``SCRAPY_SETTINGS_MODULE`` should be in Python path syntax, e.g. ``myproject.settings``. Note that the settings module should be on the Python `import search path`_. -.. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path +.. _import search path: https://docs.python.org/3/tutorial/modules.html#the-module-search-path .. _populating-settings: @@ -902,7 +902,7 @@ Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'`` String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available placeholders. -.. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes +.. _Python logging documentation: https://docs.python.org/3/library/logging.html#logrecord-attributes .. setting:: LOG_DATEFORMAT @@ -915,7 +915,7 @@ String for formatting date/time, expansion of the ``%(asctime)s`` placeholder in :setting:`LOG_FORMAT`. Refer to the `Python datetime documentation`_ for the whole list of available directives. -.. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior +.. _Python datetime documentation: https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior .. setting:: LOG_FORMATTER diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 0e8210130..3d7450c86 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -173,18 +173,18 @@ object gives you access, for example, to the :ref:`settings `. :type spider: :class:`~scrapy.spiders.Spider` object .. method:: from_crawler(cls, crawler) - + If present, this classmethod is called to create a middleware instance from a :class:`~scrapy.crawler.Crawler`. It must return a new instance of the middleware. Crawler object provides access to all Scrapy core components like settings and signals; it is a way for middleware to access them and hook its functionality into Scrapy. - + :param crawler: crawler that uses this middleware :type crawler: :class:`~scrapy.crawler.Crawler` object -.. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception +.. _Exception: https://docs.python.org/3/library/exceptions.html#Exception .. _topics-spider-middleware-ref: From f37b1bdc5616f67460c645e26c49f9d5b34e3631 Mon Sep 17 00:00:00 2001 From: Aditya Date: Fri, 20 Mar 2020 05:22:51 +0530 Subject: [PATCH 131/281] [docs] update redirect links to python3 --- docs/intro/tutorial.rst | 10 +++++----- docs/topics/contracts.rst | 4 +--- docs/topics/downloader-middleware.rst | 11 +++-------- docs/topics/dynamic-content.rst | 10 ++++------ docs/topics/email.rst | 4 +--- docs/topics/exporters.rst | 20 ++++++-------------- docs/topics/extensions.rst | 3 +-- docs/topics/items.rst | 21 ++++++--------------- docs/topics/logging.rst | 15 +++++---------- docs/topics/request-response.rst | 8 ++------ docs/topics/selectors.rst | 3 +-- docs/topics/spider-middleware.rst | 6 +----- docs/topics/spiders.rst | 4 +--- docs/topics/telnetconsole.rst | 11 ++++------- scrapy/item.py | 4 +--- 15 files changed, 42 insertions(+), 92 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 1768badbb..ab6fd4829 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -25,16 +25,16 @@ Scrapy. If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource. If you're new to programming and want to start with Python, the following books -may be useful to you: +may be useful to you: * `Automate the Boring Stuff With Python`_ -* `How To Think Like a Computer Scientist`_ +* `How To Think Like a Computer Scientist`_ -* `Learn Python 3 The Hard Way`_ +* `Learn Python 3 The Hard Way`_ You can also take a look at `this list of Python resources for non-programmers`_, -as well as the `suggested resources in the learnpython-subreddit`_. +as well as the `suggested resources in the learnpython-subreddit`_. .. _Python: https://www.python.org/ .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers @@ -62,7 +62,7 @@ This will create a ``tutorial`` directory with the following contents:: __init__.py items.py # project items definition file - + middlewares.py # project middlewares file pipelines.py # project pipelines file diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 43db8f101..319f577bc 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -136,7 +136,7 @@ Detecting check runs ==================== When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is -set to the ``true`` string. You can use `os.environ`_ to perform any change to +set to the ``true`` string. You can use :data:`os.environ` to perform any change to your spiders or your settings when ``scrapy check`` is used:: import os @@ -148,5 +148,3 @@ your spiders or your settings when ``scrapy check`` is used:: def __init__(self): if os.environ.get('SCRAPY_CHECK'): pass # Do some scraper adjustments when a check is running - -.. _os.environ: https://docs.python.org/3/library/os.html#os.environ diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 61a3806fb..d7ec53bfa 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -739,7 +739,7 @@ HttpProxyMiddleware This middleware sets the HTTP proxy to use for requests, by setting the ``proxy`` meta value for :class:`~scrapy.http.Request` objects. - Like the Python standard library module `urllib.request`_, it obeys + Like the Python standard library module :mod:`urllib.request`, it obeys the following environment variables: * ``http_proxy`` @@ -751,8 +751,6 @@ HttpProxyMiddleware Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` environment variables, and it will also ignore ``no_proxy`` environment variable. -.. _urllib.request: https://docs.python.org/3/library/urllib.request.html - RedirectMiddleware ------------------ @@ -982,7 +980,7 @@ RobotsTxtMiddleware Scrapy ships with support for the following robots.txt_ parsers: * :ref:`Protego ` (default) - * :ref:`RobotFileParser ` + * :class:`~urllib.robotparser.RobotFileParser` * :ref:`Reppy ` * :ref:`Robotexclusionrulesparser ` @@ -1030,13 +1028,10 @@ Based on `Protego `_: Scrapy uses this parser by default. -.. _python-robotfileparser: - RobotFileParser ~~~~~~~~~~~~~~~ -Based on `RobotFileParser -`_: +Based on :class:`~urllib.robotparser.RobotFileParser`: * is Python's built-in robots.txt_ parser diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index b98133676..22bcac268 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -115,7 +115,7 @@ data from it depends on the type of response: - If the response is HTML or XML, use :ref:`selectors ` as usual. -- If the response is JSON, use `json.loads`_ to load the desired data from +- If the response is JSON, use :func:`json.loads` to load the desired data from :attr:`response.text `:: data = json.loads(response.text) @@ -130,7 +130,7 @@ data from it depends on the type of response: - If the response is JavaScript, or HTML with a ``