diff --git a/.travis.yml b/.travis.yml index cb54c8008..bcbf75a43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,8 @@ matrix: python: 3.8 - env: TOXENV=flake8 python: 3.8 + - env: TOXENV=pylint + python: 3.8 - env: TOXENV=docs python: 3.7 # Keep in sync with .readthedocs.yml diff --git a/docs/conf.py b/docs/conf.py index 813417bae..8ab38a090 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # Scrapy documentation build configuration file, created by # sphinx-quickstart on Mon Nov 24 12:02:52 2008. # diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 3b85bfe8a..495111b56 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -184,6 +184,18 @@ data from it: >>> json.loads(json_data) {'field': 'value'} +- chompjs_ provides an API to parse JavaScript objects into a :class:`dict`. + + For example, if the JavaScript code contains + ``var data = {field: "value", secondField: "second value"};`` + you can extract that data as follows: + + >>> import chompjs + >>> javascript = response.css('script::text').get() + >>> data = chompjs.parse_js_object(javascript) + >>> data + {'field': 'value', 'secondField': 'second value'} + - Otherwise, use js2xml_ to convert the JavaScript code into an XML document that you can parse using :ref:`selectors `. @@ -241,6 +253,7 @@ along with `scrapy-selenium`_ for seamless integration. .. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29 +.. _chompjs: https://github.com/Nykakin/chompjs .. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets .. _curl: https://curl.haxx.se/ .. _headless browser: https://en.wikipedia.org/wiki/Headless_browser diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 024f46466..15a83f453 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -834,11 +834,6 @@ TextResponse objects .. automethod:: TextResponse.follow_all - .. method:: TextResponse.body_as_unicode() - - The same as :attr:`text`, but available as a method. This method is - kept for backward compatibility; please prefer ``response.text``. - HtmlResponse objects -------------------- diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 8661f86a0..7fe63a7b0 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -112,7 +112,7 @@ engine_started Sent when the Scrapy engine has started crawling. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. .. note:: This signal may be fired *after* the :signal:`spider_opened` signal, depending on how the spider was started. So **don't** rely on this signal @@ -127,7 +127,7 @@ engine_stopped Sent when the Scrapy engine is stopped (for example, when a crawling process has finished). - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. Item signals ------------ @@ -149,7 +149,7 @@ item_scraped Sent when an item has been scraped, after it has passed all the :ref:`topics-item-pipeline` stages (without being dropped). - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param item: the item scraped :type item: dict or :class:`~scrapy.item.Item` object @@ -169,7 +169,7 @@ item_dropped Sent after an item has been dropped from the :ref:`topics-item-pipeline` when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param item: the item dropped from the :ref:`topics-item-pipeline` :type item: dict or :class:`~scrapy.item.Item` object @@ -194,7 +194,7 @@ item_error Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises an exception), except :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param item: the item dropped from the :ref:`topics-item-pipeline` :type item: dict or :class:`~scrapy.item.Item` object @@ -220,7 +220,7 @@ spider_closed Sent after a spider has been closed. This can be used to release per-spider resources reserved on :signal:`spider_opened`. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param spider: the spider which has been closed :type spider: :class:`~scrapy.spiders.Spider` object @@ -244,7 +244,7 @@ spider_opened reserve per-spider resources, but can be used for any task that needs to be performed when a spider is opened. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param spider: the spider which has been opened :type spider: :class:`~scrapy.spiders.Spider` object @@ -268,7 +268,7 @@ spider_idle You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to prevent the spider from being closed. - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param spider: the spider which has gone idle :type spider: :class:`~scrapy.spiders.Spider` object @@ -287,7 +287,7 @@ spider_error Sent when a spider callback generates an error (i.e. raises an exception). - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param failure: the exception raised :type failure: twisted.python.failure.Failure @@ -310,7 +310,7 @@ request_scheduled Sent when the engine schedules a :class:`~scrapy.http.Request`, to be downloaded later. - The signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler :type request: :class:`~scrapy.http.Request` object @@ -327,7 +327,7 @@ request_dropped Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be downloaded later, is rejected by the scheduler. - The signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler :type request: :class:`~scrapy.http.Request` object @@ -343,7 +343,7 @@ request_reached_downloader Sent when a :class:`~scrapy.http.Request` reached downloader. - The signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param request: the request that reached downloader :type request: :class:`~scrapy.http.Request` object @@ -370,6 +370,29 @@ request_left_downloader :param spider: the spider that yielded the request :type spider: :class:`~scrapy.spiders.Spider` object +bytes_received +~~~~~~~~~~~~~~ + +.. signal:: bytes_received +.. function:: bytes_received(data, request, spider) + + Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is + received for a specific request. This signal might be fired multiple + times for the same request, with partial data each time. For instance, + a possible scenario for a 25 kb response would be two signals fired + with 10 kb of data, and a final one with 5 kb of data. + + This signal does not support returning deferreds from its handlers. + + :param data: the data received by the download handler + :type spider: :class:`bytes` object + + :param request: the request that generated the response + :type request: :class:`~scrapy.http.Request` object + + :param spider: the spider associated with the response + :type spider: :class:`~scrapy.spiders.Spider` object + Response signals ---------------- @@ -382,7 +405,7 @@ response_received Sent when the engine receives a new :class:`~scrapy.http.Response` from the downloader. - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param response: the response received :type response: :class:`~scrapy.http.Response` object @@ -401,7 +424,7 @@ response_downloaded Sent by the downloader right after a ``HTTPResponse`` is downloaded. - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param response: the response downloaded :type response: :class:`~scrapy.http.Response` object diff --git a/docs/utils/linkfix.py b/docs/utils/linkfix.py index 6290adbe2..9acfc3b23 100755 --- a/docs/utils/linkfix.py +++ b/docs/utils/linkfix.py @@ -14,50 +14,57 @@ Author: dufferzafar import re -# Used for remembering the file (and its contents) -# so we don't have to open the same file again. -_filename = None -_contents = None -# A regex that matches standard linkcheck output lines -line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') +def main(): -# Read lines from the linkcheck output file -try: - with open("build/linkcheck/output.txt") as out: - output_lines = out.readlines() -except IOError: - print("linkcheck output not found; please run linkcheck first.") - exit(1) + # Used for remembering the file (and its contents) + # so we don't have to open the same file again. + _filename = None + _contents = None -# For every line, fix the respective file -for line in output_lines: - match = re.match(line_re, line) + # A regex that matches standard linkcheck output lines + line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))') - if match: - newfilename = match.group(1) - errortype = match.group(2) + # Read lines from the linkcheck output file + try: + with open("build/linkcheck/output.txt") as out: + output_lines = out.readlines() + except IOError: + print("linkcheck output not found; please run linkcheck first.") + exit(1) - # Broken links can't be fixed and - # I am not sure what do with the local ones. - if errortype.lower() in ["broken", "local"]: - print("Not Fixed: " + line) + # For every line, fix the respective file + for line in output_lines: + match = re.match(line_re, line) + + if match: + newfilename = match.group(1) + errortype = match.group(2) + + # Broken links can't be fixed and + # I am not sure what do with the local ones. + if errortype.lower() in ["broken", "local"]: + print("Not Fixed: " + line) + else: + # If this is a new file + if newfilename != _filename: + + # Update the previous file + if _filename: + with open(_filename, "w") as _file: + _file.write(_contents) + + _filename = newfilename + + # Read the new file to memory + with open(_filename) as _file: + _contents = _file.read() + + _contents = _contents.replace(match.group(3), match.group(4)) else: - # If this is a new file - if newfilename != _filename: + # We don't understand what the current line means! + print("Not Understood: " + line) - # Update the previous file - if _filename: - with open(_filename, "w") as _file: - _file.write(_contents) - _filename = newfilename - - # Read the new file to memory - with open(_filename) as _file: - _contents = _file.read() - - _contents = _contents.replace(match.group(3), match.group(4)) - else: - # We don't understand what the current line means! - print("Not Understood: " + line) +if __name__ == '__main__': + main() diff --git a/pylintrc b/pylintrc new file mode 100644 index 000000000..129c7bf7d --- /dev/null +++ b/pylintrc @@ -0,0 +1,113 @@ +[MASTER] +persistent=no +jobs=1 # >1 hides results + +[MESSAGES CONTROL] +disable=abstract-method, + anomalous-backslash-in-string, + arguments-differ, + attribute-defined-outside-init, + bad-classmethod-argument, + bad-continuation, + bad-indentation, + bad-mcs-classmethod-argument, + bad-super-call, + bad-whitespace, + bare-except, + blacklisted-name, + broad-except, + c-extension-no-member, + catching-non-exception, + cell-var-from-loop, + comparison-with-callable, + consider-iterating-dictionary, + consider-using-in, + consider-using-set-comprehension, + consider-using-sys-exit, + cyclic-import, + dangerous-default-value, + deprecated-method, + deprecated-module, + duplicate-code, # https://github.com/PyCQA/pylint/issues/214 + eval-used, + expression-not-assigned, + fixme, + function-redefined, + global-statement, + import-error, + import-outside-toplevel, + import-self, + inconsistent-return-statements, + inherit-non-class, + invalid-name, + invalid-overridden-method, + isinstance-second-argument-not-valid-type, + keyword-arg-before-vararg, + line-too-long, + logging-format-interpolation, + logging-not-lazy, + lost-exception, + method-hidden, + misplaced-comparison-constant, + missing-docstring, + missing-final-newline, + multiple-imports, + multiple-statements, + no-else-continue, + no-else-raise, + no-else-return, + no-init, + no-member, + no-method-argument, + no-name-in-module, + no-self-argument, + no-self-use, + no-value-for-parameter, + not-an-iterable, + not-callable, + pointless-statement, + pointless-string-statement, + protected-access, + redefined-argument-from-local, + redefined-builtin, + redefined-outer-name, + reimported, + signature-differs, + singleton-comparison, + super-init-not-called, + superfluous-parens, + too-few-public-methods, + too-many-ancestors, + too-many-arguments, + too-many-branches, + too-many-format-args, + too-many-function-args, + too-many-instance-attributes, + too-many-lines, + too-many-locals, + too-many-public-methods, + too-many-return-statements, + trailing-newlines, + trailing-whitespace, + unbalanced-tuple-unpacking, + undefined-variable, + undefined-loop-variable, + unexpected-special-method-signature, + ungrouped-imports, + unidiomatic-typecheck, + unnecessary-comprehension, + unnecessary-lambda, + unnecessary-pass, + unreachable, + unsubscriptable-object, + unused-argument, + unused-import, + unused-variable, + unused-wildcard-import, + used-before-assignment, + useless-object-inheritance, # Required for Python 2 support + useless-return, + useless-super-delegation, + wildcard-import, + wrong-import-order, + wrong-import-position diff --git a/pytest.ini b/pytest.ini index 053793853..8177fedd3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -31,52 +31,51 @@ flake8-ignore = extras/qps-bench-server.py E501 extras/qpsclient.py E501 E501 # scrapy/commands - scrapy/commands/__init__.py E128 E501 + scrapy/commands/__init__.py E501 scrapy/commands/check.py E501 scrapy/commands/crawl.py E501 scrapy/commands/edit.py E501 - scrapy/commands/fetch.py E501 E128 - scrapy/commands/genspider.py E128 E501 - scrapy/commands/parse.py E128 E501 + scrapy/commands/fetch.py E501 + scrapy/commands/genspider.py E501 + scrapy/commands/parse.py E501 scrapy/commands/runspider.py E501 - scrapy/commands/settings.py E128 - scrapy/commands/shell.py E128 E501 - scrapy/commands/startproject.py E501 E128 - scrapy/commands/version.py E501 E128 + scrapy/commands/settings.py E501 + scrapy/commands/shell.py E501 + scrapy/commands/startproject.py E501 + scrapy/commands/version.py E501 # scrapy/contracts scrapy/contracts/__init__.py E501 - scrapy/contracts/default.py E128 # scrapy/core - scrapy/core/engine.py E501 E128 + scrapy/core/engine.py E501 scrapy/core/scheduler.py E501 - scrapy/core/scraper.py E501 E128 - scrapy/core/spidermw.py E501 E126 + scrapy/core/scraper.py E501 + scrapy/core/spidermw.py E501 scrapy/core/downloader/__init__.py E501 - scrapy/core/downloader/contextfactory.py E501 E128 E126 + scrapy/core/downloader/contextfactory.py E501 scrapy/core/downloader/middleware.py E501 scrapy/core/downloader/tls.py E501 - scrapy/core/downloader/webclient.py E501 E128 E126 + scrapy/core/downloader/webclient.py E501 scrapy/core/downloader/handlers/__init__.py E501 - scrapy/core/downloader/handlers/ftp.py E501 E128 + scrapy/core/downloader/handlers/ftp.py E501 scrapy/core/downloader/handlers/http10.py E501 scrapy/core/downloader/handlers/http11.py E501 - scrapy/core/downloader/handlers/s3.py E501 E128 E126 + scrapy/core/downloader/handlers/s3.py E501 # scrapy/downloadermiddlewares scrapy/downloadermiddlewares/ajaxcrawl.py E501 scrapy/downloadermiddlewares/decompression.py E501 scrapy/downloadermiddlewares/defaultheaders.py E501 - scrapy/downloadermiddlewares/httpcache.py E501 E126 - scrapy/downloadermiddlewares/httpcompression.py E501 E128 + scrapy/downloadermiddlewares/httpcache.py E501 + scrapy/downloadermiddlewares/httpcompression.py E501 scrapy/downloadermiddlewares/httpproxy.py E501 scrapy/downloadermiddlewares/redirect.py E501 - scrapy/downloadermiddlewares/retry.py E501 E126 + scrapy/downloadermiddlewares/retry.py E501 scrapy/downloadermiddlewares/robotstxt.py E501 scrapy/downloadermiddlewares/stats.py E501 # scrapy/extensions - scrapy/extensions/closespider.py E501 E128 + scrapy/extensions/closespider.py E501 scrapy/extensions/corestats.py E501 - scrapy/extensions/feedexport.py E128 E501 - scrapy/extensions/httpcache.py E128 E501 + scrapy/extensions/feedexport.py E501 + scrapy/extensions/httpcache.py E501 scrapy/extensions/memdebug.py E501 scrapy/extensions/spiderstate.py E501 scrapy/extensions/telnet.py E501 @@ -87,25 +86,25 @@ flake8-ignore = scrapy/http/request/__init__.py E501 scrapy/http/request/form.py E501 scrapy/http/request/json_request.py E501 - scrapy/http/response/__init__.py E501 E128 - scrapy/http/response/text.py E501 E128 + scrapy/http/response/__init__.py E501 + scrapy/http/response/text.py E501 # scrapy/linkextractors scrapy/linkextractors/__init__.py E501 E402 scrapy/linkextractors/lxmlhtml.py E501 # scrapy/loader - scrapy/loader/__init__.py E501 E128 + scrapy/loader/__init__.py E501 scrapy/loader/processors.py E501 # scrapy/pipelines scrapy/pipelines/__init__.py E501 - scrapy/pipelines/files.py E116 E501 + scrapy/pipelines/files.py E501 scrapy/pipelines/images.py E501 scrapy/pipelines/media.py E501 # scrapy/selector scrapy/selector/__init__.py F403 - scrapy/selector/unified.py E501 E111 + scrapy/selector/unified.py E501 # scrapy/settings scrapy/settings/__init__.py E501 - scrapy/settings/default_settings.py E501 E114 E116 + scrapy/settings/default_settings.py E501 scrapy/settings/deprecated.py E501 # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 @@ -123,13 +122,13 @@ flake8-ignore = scrapy/utils/conf.py E402 E501 scrapy/utils/datatypes.py E501 scrapy/utils/decorators.py E501 - scrapy/utils/defer.py E501 E128 + scrapy/utils/defer.py E501 scrapy/utils/deprecate.py E501 scrapy/utils/gz.py E501 scrapy/utils/http.py F403 scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 - scrapy/utils/log.py E128 E501 + scrapy/utils/log.py E501 scrapy/utils/markup.py F403 scrapy/utils/misc.py E501 scrapy/utils/multipart.py F403 @@ -138,13 +137,13 @@ flake8-ignore = scrapy/utils/reactor.py E501 scrapy/utils/reqser.py E501 scrapy/utils/request.py E501 - scrapy/utils/response.py E501 E128 - scrapy/utils/signal.py E501 E128 + scrapy/utils/response.py E501 + scrapy/utils/signal.py E501 scrapy/utils/sitemap.py E501 scrapy/utils/spider.py E501 scrapy/utils/ssl.py E501 scrapy/utils/test.py E501 - scrapy/utils/url.py E501 F403 E128 F405 + scrapy/utils/url.py E501 F403 F405 # scrapy scrapy/__init__.py E402 E501 scrapy/cmdline.py E501 @@ -153,99 +152,95 @@ flake8-ignore = scrapy/exceptions.py E501 scrapy/exporters.py E501 scrapy/interfaces.py E501 - scrapy/item.py E501 E128 + scrapy/item.py E501 scrapy/link.py E501 scrapy/logformatter.py E501 - scrapy/mail.py E402 E128 E501 - scrapy/middleware.py E128 E501 + scrapy/mail.py E402 E501 + scrapy/middleware.py E501 scrapy/pqueues.py E501 scrapy/resolver.py E501 - scrapy/responsetypes.py E128 E501 + scrapy/responsetypes.py E501 scrapy/robotstxt.py E501 scrapy/shell.py E501 scrapy/signalmanager.py E501 - scrapy/spiderloader.py F841 E501 E126 - scrapy/squeues.py E128 + scrapy/spiderloader.py E501 + scrapy/squeues.py E501 scrapy/statscollectors.py E501 # tests tests/__init__.py E402 E501 - tests/mockserver.py E501 E126 - tests/pipelines.py F841 + tests/mockserver.py E501 tests/spiders.py E501 tests/test_closespider.py E501 tests/test_command_fetch.py E501 - tests/test_command_parse.py E501 E128 - tests/test_command_shell.py E501 E128 - tests/test_commands.py E128 E501 - tests/test_contracts.py E501 E128 - tests/test_crawl.py E501 E741 - tests/test_crawler.py F841 E501 - tests/test_dependencies.py F841 E501 - tests/test_downloader_handlers.py E128 E501 E126 + tests/test_command_parse.py E501 + tests/test_command_shell.py E501 + tests/test_commands.py E501 + tests/test_contracts.py E501 + tests/test_crawl.py E501 + tests/test_crawler.py E501 + tests/test_dependencies.py E501 + tests/test_downloader_handlers.py E501 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 - tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126 + tests/test_downloadermiddleware_cookies.py E501 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 tests/test_downloadermiddleware_httpcache.py E501 - tests/test_downloadermiddleware_httpcompression.py E501 E126 + tests/test_downloadermiddleware_httpcompression.py E501 tests/test_downloadermiddleware_decompression.py E501 - tests/test_downloadermiddleware_httpproxy.py E501 E128 - tests/test_downloadermiddleware_redirect.py E501 E128 - tests/test_downloadermiddleware_retry.py E501 E128 E126 + tests/test_downloadermiddleware_httpproxy.py E501 + tests/test_downloadermiddleware_redirect.py E501 + tests/test_downloadermiddleware_retry.py E501 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E501 E741 E128 - tests/test_engine.py E501 E128 - tests/test_exporters.py E501 E128 - tests/test_extension_telnet.py F841 - tests/test_feedexport.py E501 F841 + tests/test_dupefilters.py E501 + tests/test_engine.py E501 + tests/test_exporters.py E501 + tests/test_feedexport.py E501 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 - tests/test_http_request.py E402 E501 E128 E128 E126 - tests/test_http_response.py E501 E128 - tests/test_item.py E128 F841 + tests/test_http_request.py E402 E501 + tests/test_http_response.py E501 + tests/test_item.py E501 tests/test_link.py E501 - tests/test_linkextractors.py E501 E128 - tests/test_loader.py E501 E741 E128 E117 - tests/test_logformatter.py E128 E501 - tests/test_mail.py E128 E501 - tests/test_middleware.py E501 E128 - tests/test_pipeline_crawl.py E501 E128 E126 + tests/test_linkextractors.py E501 + tests/test_loader.py E501 E741 + tests/test_logformatter.py E501 + tests/test_mail.py E501 + tests/test_middleware.py E501 + tests/test_pipeline_crawl.py E501 tests/test_pipeline_files.py E501 - tests/test_pipeline_images.py F841 E501 - tests/test_pipeline_media.py E501 E741 E128 - tests/test_proxy_connect.py E501 E741 + tests/test_pipeline_images.py E501 + tests/test_pipeline_media.py E501 + tests/test_proxy_connect.py E501 tests/test_request_cb_kwargs.py E501 tests/test_responsetypes.py E501 tests/test_robotstxt_interface.py E501 E501 - tests/test_scheduler.py E501 E126 + tests/test_scheduler.py E501 tests/test_selector.py E501 tests/test_spider.py E501 tests/test_spidermiddleware.py E501 - tests/test_spidermiddleware_httperror.py E128 E501 E121 - tests/test_spidermiddleware_offsite.py E501 E128 E111 + tests/test_spidermiddleware_httperror.py E501 + tests/test_spidermiddleware_offsite.py E501 tests/test_spidermiddleware_output_chain.py E501 - tests/test_spidermiddleware_referer.py E501 F841 E501 E121 - tests/test_squeues.py E501 E741 + tests/test_spidermiddleware_referer.py E501 + tests/test_squeues.py E501 tests/test_utils_asyncio.py E501 - tests/test_utils_conf.py E501 E128 + tests/test_utils_conf.py E501 tests/test_utils_curl.py E501 tests/test_utils_datatypes.py E402 E501 - tests/test_utils_defer.py E501 F841 - tests/test_utils_deprecate.py F841 E501 - tests/test_utils_http.py E501 E128 - tests/test_utils_iterators.py E501 E128 - tests/test_utils_log.py E741 + tests/test_utils_defer.py E501 + tests/test_utils_deprecate.py E501 + tests/test_utils_http.py E501 + tests/test_utils_iterators.py E501 tests/test_utils_python.py E501 - tests/test_utils_reqser.py E501 E128 - tests/test_utils_request.py E501 E128 + tests/test_utils_reqser.py E501 + tests/test_utils_request.py E501 tests/test_utils_response.py E501 - tests/test_utils_signal.py E741 F841 - tests/test_utils_sitemap.py E128 E501 - tests/test_utils_url.py E501 E501 E126 - tests/test_webclient.py E501 E128 E402 E126 + tests/test_utils_sitemap.py E501 + tests/test_utils_url.py E501 E501 + tests/test_webclient.py E501 E402 tests/test_cmdline/__init__.py E501 - tests/test_settings/__init__.py E501 E128 - tests/test_spiderloader/__init__.py E128 E501 + tests/test_settings/__init__.py E501 + tests/test_spiderloader/__init__.py E501 tests/test_utils_misc/__init__.py E501 diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 0ddbf2ca0..338fc9b0e 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -70,8 +70,7 @@ class ScrapyCommand: help="write process ID to FILE") group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE", help="set/override setting (may be repeated)") - group.add_option("--pdb", action="store_true", - help="enable pdb on failure") + group.add_option("--pdb", action="store_true", help="enable pdb on failure") parser.add_option_group(group) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 506d1f1b7..063195f50 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -27,8 +27,8 @@ class Command(ScrapyCommand): parser.add_option("--spider", dest="spider", help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", help="print response HTTP headers instead of body") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", - default=False, help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2e837abed..abf3b7a5c 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -36,15 +36,15 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("-l", "--list", dest="list", action="store_true", - help="List available templates") + help="List available templates") parser.add_option("-e", "--edit", dest="edit", action="store_true", - help="Edit spider after creating it") + help="Edit spider after creating it") parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE", - help="Dump template to standard output") + help="Dump template to standard output") parser.add_option("-t", "--template", dest="template", default="basic", - help="Uses a custom template.") + help="Uses a custom template.") parser.add_option("--force", dest="force", action="store_true", - help="If the spider already exists, overwrite it with the template") + help="If the spider already exists, overwrite it with the template") def run(self, args, opts): if opts.list: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 1cefed106..d9ab2126a 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -33,29 +33,29 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("--spider", dest="spider", default=None, - help="use this spider without looking for one") + help="use this spider without looking for one") parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE", - help="set spider argument (may be repeated)") + help="set spider argument (may be repeated)") parser.add_option("--pipelines", action="store_true", - help="process items through pipelines") + help="process items through pipelines") parser.add_option("--nolinks", dest="nolinks", action="store_true", - help="don't show links to follow (extracted requests)") + help="don't show links to follow (extracted requests)") parser.add_option("--noitems", dest="noitems", action="store_true", - help="don't show scraped items") + help="don't show scraped items") parser.add_option("--nocolour", dest="nocolour", action="store_true", - help="avoid using pygments to colorize the output") + help="avoid using pygments to colorize the output") parser.add_option("-r", "--rules", dest="rules", action="store_true", - help="use CrawlSpider rules to discover the callback") + help="use CrawlSpider rules to discover the callback") parser.add_option("-c", "--callback", dest="callback", - help="use this callback for parsing, instead looking for a callback") + help="use this callback for parsing, instead looking for a callback") parser.add_option("-m", "--meta", dest="meta", - help="inject extra meta into the Request, it must be a valid raw json string") + help="inject extra meta into the Request, it must be a valid raw json string") parser.add_option("--cbkwargs", dest="cbkwargs", - help="inject extra callback kwargs into the Request, it must be a valid raw json string") + help="inject extra callback kwargs into the Request, it must be a valid raw json string") parser.add_option("-d", "--depth", dest="depth", type="int", default=1, - help="maximum depth for parsing requests [default: %default]") + help="maximum depth for parsing requests [default: %default]") parser.add_option("-v", "--verbose", dest="verbose", action="store_true", - help="print each depth level one by one") + help="print each depth level one by one") @property def max_level(self): diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 603bafb9f..8d49e440f 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -19,15 +19,15 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("--get", dest="get", metavar="SETTING", - help="print raw setting value") + help="print raw setting value") parser.add_option("--getbool", dest="getbool", metavar="SETTING", - help="print setting value, interpreted as a boolean") + help="print setting value, interpreted as a boolean") parser.add_option("--getint", dest="getint", metavar="SETTING", - help="print setting value, interpreted as an integer") + help="print setting value, interpreted as an integer") parser.add_option("--getfloat", dest="getfloat", metavar="SETTING", - help="print setting value, interpreted as a float") + help="print setting value, interpreted as a float") parser.add_option("--getlist", dest="getlist", metavar="SETTING", - help="print setting value, interpreted as a list") + help="print setting value, interpreted as a list") def run(self, args, opts): settings = self.crawler_process.settings diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 5946f21e8..d1944df3d 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -34,11 +34,11 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("-c", dest="code", - help="evaluate the code in the shell, print the result and exit") + help="evaluate the code in the shell, print the result and exit") parser.add_option("--spider", dest="spider", - help="use this spider") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", - default=False, help="do not handle HTTP 3xx status codes and print response as-is") + help="use this spider") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False, + help="do not handle HTTP 3xx status codes and print response as-is") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index b123e5c84..ebe3a9c2c 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -102,10 +102,8 @@ class Command(ScrapyCommand): move(join(project_dir, 'module'), join(project_dir, project_name)) for paths in TEMPLATES_TO_RENDER: path = join(*paths) - tplfile = join(project_dir, - string.Template(path).substitute(project_name=project_name)) - render_templatefile(tplfile, project_name=project_name, - ProjectName=string_camelcase(project_name)) + tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name)) + render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name)) print("New Scrapy project '%s', using template directory '%s', " "created in:" % (project_name, self.templates_dir)) print(" %s\n" % abspath(project_dir)) diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 1516c5997..d0ea72a67 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -17,7 +17,7 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) parser.add_option("--verbose", "-v", dest="verbose", action="store_true", - help="also display twisted/python/platform info (useful for bug reports)") + help="also display twisted/python/platform info (useful for bug reports)") def run(self, args, opts): if opts.verbose: diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 6e023ebcc..452242d47 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -46,11 +46,12 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # # * getattr() for `_ssl_method` attribute for context factories # not calling super(..., self).__init__ - return CertificateOptions(verify=False, - method=getattr(self, 'method', - getattr(self, '_ssl_method', None)), - fixBrokenPeers=True, - acceptableCiphers=self.tls_ciphers) + return CertificateOptions( + verify=False, + method=getattr(self, 'method', getattr(self, '_ssl_method', None)), + fixBrokenPeers=True, + acceptableCiphers=self.tls_ciphers, + ) # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() @@ -86,8 +87,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): # # This means that a website like https://www.cacert.org will be rejected # by default, since CAcert.org CA certificate is seldom shipped. - return optionsForClientTLS(hostname.decode("ascii"), - trustRoot=platformTrust(), - extraCertificateOptions={ - 'method': self._ssl_method, - }) + return optionsForClientTLS( + hostname=hostname.decode("ascii"), + trustRoot=platformTrust(), + extraCertificateOptions={'method': self._ssl_method}, + ) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 94b55c347..3ef129587 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -86,10 +86,9 @@ class FTPDownloadHandler: password = request.meta.get("ftp_password", self.default_password) passive_mode = 1 if bool(request.meta.get("ftp_passive", self.passive_mode)) else 0 - creator = ClientCreator(reactor, FTPClient, user, password, - passive=passive_mode) - return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient, - request, unquote(parsed_url.path)) + creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode) + dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) + return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path)) def gotClient(self, client, request, filepath): self.client = client diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9be8ffdfb..c21491f52 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -18,6 +18,7 @@ from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from zope.interface import implementer +from scrapy import signals from scrapy.core.downloader.tls import openssl_methods from scrapy.core.downloader.webclient import _parse from scrapy.exceptions import ScrapyDeprecationWarning @@ -34,6 +35,8 @@ class HTTP11DownloadHandler: lazy = False def __init__(self, settings, crawler=None): + self._crawler = crawler + from twisted.internet import reactor self._pool = HTTPConnectionPool(reactor, persistent=True) self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') @@ -79,6 +82,7 @@ class HTTP11DownloadHandler: maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), fail_on_dataloss=self._fail_on_dataloss, + crawler=self._crawler, ) return agent.download_request(request) @@ -276,7 +280,7 @@ class ScrapyAgent: _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, - maxsize=0, warnsize=0, fail_on_dataloss=True): + maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress @@ -285,6 +289,7 @@ class ScrapyAgent: self._warnsize = warnsize self._fail_on_dataloss = fail_on_dataloss self._txresponse = None + self._crawler = crawler def _get_agent(self, request, timeout): from twisted.internet import reactor @@ -407,7 +412,15 @@ class ScrapyAgent: d = defer.Deferred(_cancel) txresponse.deliverBody( - _ResponseReader(d, txresponse, request, maxsize, warnsize, fail_on_dataloss) + _ResponseReader( + finished=d, + txresponse=txresponse, + request=request, + maxsize=maxsize, + warnsize=warnsize, + fail_on_dataloss=fail_on_dataloss, + crawler=self._crawler, + ) ) # save response for timeouts @@ -449,7 +462,7 @@ class _RequestBodyProducer: class _ResponseReader(protocol.Protocol): - def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss): + def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler): self._finished = finished self._txresponse = txresponse self._request = request @@ -462,6 +475,7 @@ class _ResponseReader(protocol.Protocol): self._bytes_received = 0 self._certificate = None self._ip_address = None + self._crawler = crawler def connectionMade(self): if self._certificate is None: @@ -479,6 +493,13 @@ class _ResponseReader(protocol.Protocol): self._bodybuf.write(bodyBytes) self._bytes_received += len(bodyBytes) + self._crawler.signals.send_catch_log( + signal=signals.bytes_received, + data=bodyBytes, + request=self._request, + spider=self._crawler.spider, + ) + if self._maxsize and self._bytes_received > self._maxsize: logger.error("Received (%(bytes)s) bytes larger than download " "max size (%(maxsize)s) in request %(request)s.", diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 40a1fa48e..8f63ad974 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -100,11 +100,12 @@ class S3DownloadHandler: url=url, headers=awsrequest.headers.items()) else: signed_headers = self.conn.make_request( - method=request.method, - bucket=bucket, - key=unquote(p.path), - query_args=unquote(p.query), - headers=request.headers, - data=request.body) + method=request.method, + bucket=bucket, + key=unquote(p.path), + query_args=unquote(p.query), + headers=request.headers, + data=request.body, + ) request = request.replace(url=url, headers=signed_headers) return self._download_http(request, spider) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index a90a77b2b..355045d74 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -88,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient): self.transport.stopProducing() self.factory.noPage( - defer.TimeoutError("Getting %s took longer than %s seconds." % - (self.factory.url, self.factory.timeout))) + defer.TimeoutError("Getting %s took longer than %s seconds." + % (self.factory.url, self.factory.timeout))) class ScrapyHTTPClientFactory(HTTPClientFactory): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 324d21716..de0da4b70 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -217,11 +217,9 @@ class ExecutionEngine: self.slot.nextcall.schedule() def schedule(self, request, spider): - self.signals.send_catch_log(signal=signals.request_scheduled, - request=request, spider=spider) + self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider) if not self.slot.scheduler.enqueue_request(request): - self.signals.send_catch_log(signal=signals.request_dropped, - request=request, spider=spider) + self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider) def download(self, request, spider): d = self._download(request, spider) @@ -247,8 +245,8 @@ class ExecutionEngine: logkws = self.logformatter.crawled(request, response, spider) if logkws is not None: logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) - self.signals.send_catch_log(signal=signals.response_received, - response=response, request=request, spider=spider) + self.signals.send_catch_log(signals.response_received, + response=response, request=request, spider=spider) return response def _on_complete(_): @@ -286,8 +284,7 @@ class ExecutionEngine: next loop and this function is guaranteed to be called (at least) once again for this spider. """ - res = self.signals.send_catch_log(signal=signals.spider_idle, - spider=spider, dont_log=DontCloseSpider) + res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider) if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res): return diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 87d08cab7..35264a92b 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -19,7 +19,7 @@ def _isiterable(possible_iterator): def _fname(f): - return "%s.%s".format( + return "{}.{}".format( f.__self__.__class__.__name__, f.__func__.__name__ ) diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index ad7a81e6b..4e12a5044 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import re import logging diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index bbf5fca05..6d11af5b2 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -12,9 +12,15 @@ once the spider has finished crawling all regular (non failed) pages. import logging from twisted.internet import defer -from twisted.internet.error import TimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost, TCPTimedOutError +from twisted.internet.error import ( + ConnectError, + ConnectionDone, + ConnectionLost, + ConnectionRefusedError, + DNSLookupError, + TCPTimedOutError, + TimeoutError, +) from twisted.web.client import ResponseFailed from scrapy.exceptions import NotConfigured diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 0cb6cef98..349a9586b 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -250,7 +250,7 @@ class CsvItemExporter(BaseItemExporter): class PickleItemExporter(BaseItemExporter): - def __init__(self, file, protocol=2, **kwargs): + def __init__(self, file, protocol=4, **kwargs): super().__init__(**kwargs) self.file = file self.protocol = protocol diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 8546628a8..6294a9b52 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -46,9 +46,10 @@ class RFC2616Policy: def __init__(self, settings): self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE') self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') - self.ignore_response_cache_controls = [to_bytes(cc) for cc in - settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS')] self._cc_parsed = WeakKeyDictionary() + self.ignore_response_cache_controls = [ + to_bytes(cc) for cc in settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS') + ] def _parse_cachecontrol(self, r): if r not in self._cc_parsed: @@ -250,7 +251,7 @@ class DbmCacheStorage: 'headers': dict(response.headers), 'body': response.body, } - self.db['%s_data' % key] = pickle.dumps(data, protocol=2) + self.db['%s_data' % key] = pickle.dumps(data, protocol=4) self.db['%s_time' % key] = str(time()) def _read_data(self, spider, request): @@ -317,7 +318,7 @@ class FilesystemCacheStorage: with self._open(os.path.join(rpath, 'meta'), 'wb') as f: f.write(to_bytes(repr(metadata))) with self._open(os.path.join(rpath, 'pickled_meta'), 'wb') as f: - pickle.dump(metadata, f, protocol=2) + pickle.dump(metadata, f, protocol=4) with self._open(os.path.join(rpath, 'response_headers'), 'wb') as f: f.write(headers_dict_to_raw(response.headers)) with self._open(os.path.join(rpath, 'response_body'), 'wb') as f: diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 2e5ff569f..bea00596e 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -26,7 +26,7 @@ class SpiderState: def spider_closed(self, spider): if self.jobdir: with open(self.statefn, 'wb') as f: - pickle.dump(spider.state, f, protocol=2) + pickle.dump(spider.state, f, protocol=4) def spider_opened(self, spider): if self.jobdir and os.path.exists(self.statefn): diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 2f0f3820c..5614e6e55 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -5,6 +5,7 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ +import warnings from contextlib import suppress from typing import Generator from urllib.parse import urljoin @@ -14,6 +15,7 @@ from w3lib.encoding import (html_body_declared_encoding, html_to_unicode, http_content_type_encoding, resolve_encoding) from w3lib.html import strip_html5_whitespace +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import memoizemethod_noargs, to_unicode @@ -61,6 +63,9 @@ class TextResponse(Response): def body_as_unicode(self): """Return body as unicode""" + warnings.warn('Response.body_as_unicode() is deprecated, ' + 'please use Response.text instead.', + ScrapyDeprecationWarning) return self.text @property diff --git a/scrapy/item.py b/scrapy/item.py index 748368932..b75d04404 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -86,8 +86,7 @@ class DictItem(MutableMapping, BaseItem): if key in self.fields: self._values[key] = value else: - raise KeyError("%s does not support field: %s" % - (self.__class__.__name__, key)) + raise KeyError("%s does not support field: %s" % (self.__class__.__name__, key)) def __delitem__(self, key): del self._values[key] @@ -99,8 +98,7 @@ class DictItem(MutableMapping, BaseItem): def __setattr__(self, name, value): if not name.startswith('_'): - raise AttributeError("Use item[%r] = %r to set field value" % - (name, value)) + raise AttributeError("Use item[%r] = %r to set field value" % (name, value)) super(DictItem, self).__setattr__(name, value) def __len__(self): diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index ceb37c5f1..1615d44d7 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,6 +1,8 @@ """ Link extractor based on lxml.html """ +import operator +from functools import partial from urllib.parse import urljoin import lxml.etree as etree @@ -8,10 +10,10 @@ from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link +from scrapy.linkextractors import FilteringLinkExtractor from scrapy.utils.misc import arg_to_iter, rel_has_nofollow from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url -from scrapy.linkextractors import FilteringLinkExtractor # from lxml/src/lxml/html/__init__.py @@ -27,19 +29,24 @@ def _nons(tag): return tag +def _identity(x): + return x + + +def _canonicalize_link_url(link): + return canonicalize_url(link.url, keep_fragments=True) + + class LxmlParserLinkExtractor: - def __init__(self, tag="a", attr="href", process=None, unique=False, - strip=True, canonicalized=False): - self.scan_tag = tag if callable(tag) else lambda t: t == tag - self.scan_attr = attr if callable(attr) else lambda a: a == attr - self.process_attr = process if callable(process) else lambda v: v + def __init__( + self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False + ): + self.scan_tag = tag if callable(tag) else partial(operator.eq, tag) + self.scan_attr = attr if callable(attr) else partial(operator.eq, attr) + self.process_attr = process if callable(process) else _identity self.unique = unique self.strip = strip - if canonicalized: - self.link_key = lambda link: link.url - else: - self.link_key = lambda link: canonicalize_url(link.url, - keep_fragments=True) + self.link_key = operator.attrgetter("url") if canonicalized else _canonicalize_link_url def _iter_links(self, document): for el in document.iter(etree.Element): @@ -93,25 +100,44 @@ class LxmlParserLinkExtractor: class LxmlLinkExtractor(FilteringLinkExtractor): - def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), - tags=('a', 'area'), attrs=('href',), canonicalize=False, - unique=True, process_value=None, deny_extensions=None, restrict_css=(), - strip=True, restrict_text=None): + def __init__( + self, + allow=(), + deny=(), + allow_domains=(), + deny_domains=(), + restrict_xpaths=(), + tags=('a', 'area'), + attrs=('href',), + canonicalize=False, + unique=True, + process_value=None, + deny_extensions=None, + restrict_css=(), + strip=True, + restrict_text=None, + ): tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) lx = LxmlParserLinkExtractor( - tag=lambda x: x in tags, - attr=lambda x: x in attrs, + tag=partial(operator.contains, tags), + attr=partial(operator.contains, attrs), unique=unique, process=process_value, strip=strip, canonicalized=canonicalize ) - - super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny, - allow_domains=allow_domains, deny_domains=deny_domains, - restrict_xpaths=restrict_xpaths, restrict_css=restrict_css, - canonicalize=canonicalize, deny_extensions=deny_extensions, - restrict_text=restrict_text) + super(LxmlLinkExtractor, self).__init__( + link_extractor=lx, + allow=allow, + deny=deny, + allow_domains=allow_domains, + deny_domains=deny_domains, + restrict_xpaths=restrict_xpaths, + restrict_css=restrict_css, + canonicalize=canonicalize, + deny_extensions=deny_extensions, + restrict_text=restrict_text, + ) def extract_links(self, response): """Returns a list of :class:`~scrapy.link.Link` objects from the @@ -124,9 +150,11 @@ class LxmlLinkExtractor(FilteringLinkExtractor): """ base_url = get_base_url(response) if self.restrict_xpaths: - docs = [subdoc - for x in self.restrict_xpaths - for subdoc in response.xpath(x)] + docs = [ + subdoc + for x in self.restrict_xpaths + for subdoc in response.xpath(x) + ] else: docs = [response.selector] all_links = [] diff --git a/scrapy/mail.py b/scrapy/mail.py index 9d7896ef6..7d7a2c435 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -28,8 +28,10 @@ def _to_bytes_or_none(text): class MailSender: - def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost', - smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): + def __init__( + self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None, + smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False + ): self.smtphost = smtphost self.smtpport = smtpport self.smtpuser = _to_bytes_or_none(smtpuser) @@ -41,9 +43,15 @@ class MailSender: @classmethod def from_settings(cls, settings): - return cls(settings['MAIL_HOST'], settings['MAIL_FROM'], settings['MAIL_USER'], - settings['MAIL_PASS'], settings.getint('MAIL_PORT'), - settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL')) + return cls( + smtphost=settings['MAIL_HOST'], + mailfrom=settings['MAIL_FROM'], + smtpuser=settings['MAIL_USER'], + smtppass=settings['MAIL_PASS'], + smtpport=settings.getint('MAIL_PORT'), + smtptls=settings.getbool('MAIL_TLS'), + smtpssl=settings.getbool('MAIL_SSL'), + ) def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None): from twisted.internet import reactor @@ -89,9 +97,12 @@ class MailSender: return dfd = self._sendmail(rcpts, msg.as_string().encode(charset or 'utf-8')) - dfd.addCallbacks(self._sent_ok, self._sent_failed, + dfd.addCallbacks( + callback=self._sent_ok, + errback=self._sent_failed, callbackArgs=[to, cc, subject, len(attachs)], - errbackArgs=[to, cc, subject, len(attachs)]) + errbackArgs=[to, cc, subject, len(attachs)], + ) reactor.addSystemEventTrigger('before', 'shutdown', lambda: dfd) return dfd @@ -115,9 +126,10 @@ class MailSender: from twisted.mail.smtp import ESMTPSenderFactory msg = BytesIO(msg) d = defer.Deferred() - factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, - to_addrs, msg, d, heloFallback=True, requireAuthentication=False, - requireTransportSecurity=self.smtptls) + factory = ESMTPSenderFactory( + self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d, + heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls, + ) factory.noisy = False if self.smtpssl: diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index a9066986b..cd3e29057 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -83,8 +83,7 @@ class S3FilesStore: AWS_USE_SSL = None AWS_VERIFY = None - POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in - # FilesPipeline.from_settings. + POLICY = 'private' # Overriden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_settings HEADERS = { 'Cache-Control': 'max-age=172800', } diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 7c5eeac21..d207088e6 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -58,9 +58,9 @@ class ResponseTypes: def from_content_disposition(self, content_disposition): try: - filename = to_unicode(content_disposition, - encoding='latin-1', errors='replace').split(';')[1].split('=')[1] - filename = filename.strip('"\'') + filename = to_unicode( + content_disposition, encoding='latin-1', errors='replace' + ).split(';')[1].split('=')[1].strip('"\'') return self.from_filename(filename) except IndexError: return Response diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index a08955dc9..85a9bb526 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -65,9 +65,9 @@ class Selector(_ParselSelector, object_ref): selectorlist_cls = SelectorList def __init__(self, response=None, text=None, type=None, root=None, **kwargs): - if not(response is None or text is None): - raise ValueError('%s.__init__() received both response and text' - % self.__class__.__name__) + if response is not None and text is not None: + raise ValueError('%s.__init__() received both response and text' + % self.__class__.__name__) st = _st(response, type or self._default_type) diff --git a/scrapy/shell.py b/scrapy/shell.py index 08ce89481..2a3e13ddd 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -146,14 +146,13 @@ class Shell: b.append("Useful shortcuts:") if self.inthread: b.append(" fetch(url[, redirect=True]) " - "Fetch URL and update local objects " - "(by default, redirects are followed)") + "Fetch URL and update local objects (by default, redirects are followed)") b.append(" fetch(req) " "Fetch a scrapy.Request and update local objects ") b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") - return "\n".join("[s] %s" % l for l in b) + return "\n".join("[s] %s" % line for line in b) def _is_relevant(self, value): return isinstance(value, self.relevant_classes) diff --git a/scrapy/signals.py b/scrapy/signals.py index cd7ed7fb1..c61ae6ec3 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -17,6 +17,7 @@ request_reached_downloader = object() request_left_downloader = object() response_received = object() response_downloaded = object() +bytes_received = object() item_scraped = object() item_dropped = object() item_error = object() diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 3be5aaec5..db4193430 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- -from collections import defaultdict import traceback import warnings +from collections import defaultdict from zope.interface import implementer @@ -16,6 +15,7 @@ class SpiderLoader: SpiderLoader is a class which locates and loads spiders in a Scrapy project. """ + def __init__(self, settings): self.spider_modules = settings.getlist('SPIDER_MODULES') self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY') @@ -24,16 +24,21 @@ class SpiderLoader: self._load_all_spiders() def _check_name_duplicates(self): - dupes = ["\n".join(" {cls} named {name!r} (in {module})".format( - module=mod, cls=cls, name=name) - for (mod, cls) in locations) - for name, locations in self._found.items() - if len(locations) > 1] + dupes = [] + for name, locations in self._found.items(): + dupes.extend([ + " {cls} named {name!r} (in {module})".format(module=mod, cls=cls, name=name) + for mod, cls in locations + if len(locations) > 1 + ]) + if dupes: - msg = ("There are several spiders with the same name:\n\n" - "{}\n\n This can cause unexpected behavior.".format( - "\n\n".join(dupes))) - warnings.warn(msg, UserWarning) + dupes_string = "\n\n".join(dupes) + warnings.warn( + "There are several spiders with the same name:\n\n" + "{}\n\n This can cause unexpected behavior.".format(dupes_string), + category=UserWarning, + ) def _load_spiders(self, module): for spcls in iter_spider_classes(module): @@ -45,12 +50,15 @@ class SpiderLoader: try: for module in walk_modules(name): self._load_spiders(module) - except ImportError as e: + except ImportError: if self.warn_only: - msg = ("\n{tb}Could not load spiders from module '{modname}'. " - "See above traceback for details.".format( - modname=name, tb=traceback.format_exc())) - warnings.warn(msg, RuntimeWarning) + warnings.warn( + "\n{tb}Could not load spiders from module '{modname}'. " + "See above traceback for details.".format( + modname=name, tb=traceback.format_exc() + ), + category=RuntimeWarning, + ) else: raise self._check_name_duplicates() @@ -73,8 +81,10 @@ class SpiderLoader: """ Return the list of spider names that can handle the given request. """ - return [name for name, cls in self._spiders.items() - if cls.handles_request(request)] + return [ + name for name, cls in self._spiders.items() + if cls.handles_request(request) + ] def list(self): """ diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index d368c7108..c5360bfa7 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -96,5 +96,4 @@ def iterloc(it, alt=False): # Also consider alternate URLs (xhtml:link rel="alternate") if alt and 'alternate' in d: - for l in d['alternate']: - yield l + yield from d['alternate'] diff --git a/scrapy/squeues.py b/scrapy/squeues.py index d0686dac3..c7ad4d53d 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -81,12 +81,11 @@ def _scrapy_non_serialization_queue(queue_class): def _pickle_serialize(obj): try: - return pickle.dumps(obj, protocol=2) - # Python <= 3.4 raises pickle.PicklingError here while - # 3.5 <= Python < 3.6 raises AttributeError and - # Python >= 3.6 raises TypeError + return pickle.dumps(obj, protocol=4) + # Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s) + # TypeError is raised from parsel.Selector except (pickle.PicklingError, AttributeError, TypeError) as e: - raise ValueError(str(e)) + raise ValueError(str(e)) from e PickleFifoDiskQueueNonRequest = _serializable_queue( diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl index a12d08414..88a18331c 100644 --- a/scrapy/templates/project/module/items.py.tmpl +++ b/scrapy/templates/project/module/items.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Define here the models for your scraped items # # See documentation in: diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index b3e58ff94..6490f52a7 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Define here the models for your spider middleware # # See documentation in: diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index 4876526a9..ce0edd335 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index cb220eafc..a414b5fde 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - # Scrapy settings for $project_name project # # For simplicity, this file contains only settings considered important or diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index 1cfe9cc9d..e9112bc95 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import scrapy diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 878425125..356496487 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index c2e4bacfe..cbcbe9e2c 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from scrapy.spiders import CSVFeedSpider diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index 863c9772f..5aa2aa8b0 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from scrapy.spiders import XMLFeedSpider diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index c7a2ace88..133261fd7 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -28,6 +28,7 @@ def _embed_ipython_shell(namespace={}, banner=''): def _embed_bpython_shell(namespace={}, banner=''): """Start a bpython shell""" import bpython + @wraps(_embed_bpython_shell) def wrapper(namespace=namespace, banner=''): bpython.embed(locals_=namespace, banner=banner) @@ -37,6 +38,7 @@ def _embed_bpython_shell(namespace={}, banner=''): def _embed_ptpython_shell(namespace={}, banner=''): """Start a ptpython shell""" import ptpython.repl + @wraps(_embed_ptpython_shell) def wrapper(namespace=namespace, banner=''): print(banner) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 34b8d9774..a3950db75 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -88,8 +88,11 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw): """Return a Deferred built by chaining the given callbacks and errbacks""" d = defer.Deferred() for cb, eb in zip(callbacks, errbacks): - d.addCallbacks(cb, eb, callbackArgs=a, callbackKeywords=kw, - errbackArgs=a, errbackKeywords=kw) + d.addCallbacks( + callback=cb, errback=eb, + callbackArgs=a, callbackKeywords=kw, + errbackArgs=a, errbackKeywords=kw, + ) if isinstance(input, failure.Failure): d.errback(input) else: diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 5998dc33b..51d276097 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import logging import sys import warnings @@ -39,7 +37,7 @@ class TopLevelFormatter(logging.Filter): self.loggers = loggers or [] def filter(self, record): - if any(record.name.startswith(l + '.') for l in self.loggers): + if any(record.name.startswith(logger + '.') for logger in self.loggers): record.name = record.name.split('.', 1)[0] return True @@ -144,10 +142,12 @@ def _get_handler(settings): def log_scrapy_info(settings): logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) - logger.info("Versions: %(versions)s", - {'versions': ", ".join("%s %s" % (name, version) - for name, version in scrapy_components_versions() - if name != "Scrapy")}) + versions = [ + "%s %s" % (name, version) + for name, version in scrapy_components_versions() + if name != "Scrapy" + ] + logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 52cfba208..ab7cf9deb 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -137,17 +137,26 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): ``*args`` and ``**kwargs`` are forwarded to the constructors. Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``. + + Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an + extension has not been implemented correctly). """ if settings is None: if crawler is None: raise ValueError("Specify at least one of settings and crawler.") settings = crawler.settings if crawler and hasattr(objcls, 'from_crawler'): - return objcls.from_crawler(crawler, *args, **kwargs) + instance = objcls.from_crawler(crawler, *args, **kwargs) + method_name = 'from_crawler' elif hasattr(objcls, 'from_settings'): - return objcls.from_settings(settings, *args, **kwargs) + instance = objcls.from_settings(settings, *args, **kwargs) + method_name = 'from_settings' else: - return objcls(*args, **kwargs) + instance = objcls(*args, **kwargs) + method_name = '__new__' + if instance is None: + raise TypeError("%s.%s returned None" % (objcls.__qualname__, method_name)) + return instance @contextmanager diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 3d02d9478..9c1f3c2fe 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -152,6 +152,7 @@ def memoizemethod_noargs(method): weak reference to its object """ cache = weakref.WeakKeyDictionary() + @wraps(method) def new_method(self, *args, **kwargs): if self not in cache: diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 29fdaaf2c..edbc0db25 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -19,8 +19,7 @@ def get_base_url(response): """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: text = response.text[0:4096] - _baseurl_cache[response] = html.get_base_url(text, response.url, - response.encoding) + _baseurl_cache[response] = html.get_base_url(text, response.url, response.encoding) return _baseurl_cache[response] @@ -31,8 +30,8 @@ def get_meta_refresh(response, ignore_tags=('script', 'noscript')): """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] - _metaref_cache[response] = html.get_meta_refresh(text, response.url, - response.encoding, ignore_tags=ignore_tags) + _metaref_cache[response] = html.get_meta_refresh( + text, response.url, response.encoding, ignore_tags=ignore_tags) return _metaref_cache[response] diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 60c561da6..a311e9257 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -28,8 +28,7 @@ def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named): responses = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): try: - response = robustApply(receiver, signal=signal, sender=sender, - *arguments, **named) + response = robustApply(receiver, signal=signal, sender=sender, *arguments, **named) if isinstance(response, Deferred): logger.error("Cannot return deferreds from signal handler: %(receiver)s", {'receiver': receiver}, extra={'spider': spider}) @@ -63,8 +62,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): spider = named.get('spider', None) dfds = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, - *arguments, **named) + d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, *arguments, **named) d.addErrback(logerror, receiver) d.addBoth(lambda result: (receiver, result)) dfds.append(d) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 6e81b33ff..c3c5e329b 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import OpenSSL import OpenSSL._util as pyOpenSSLutil diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index c9abb12d5..955b63d4b 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -27,8 +27,7 @@ def url_is_from_any_domain(url, domains): def url_is_from_spider(url, spider): """Return True if the url belongs to the given spider""" - return url_is_from_any_domain(url, - [spider.name] + list(getattr(spider, 'allowed_domains', []))) + return url_is_from_any_domain(url, [spider.name] + list(getattr(spider, 'allowed_domains', []))) def url_has_any_extension(url, extensions): diff --git a/tests/pipelines.py b/tests/pipelines.py index cf677cc17..fed2af7d3 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -6,7 +6,7 @@ Some pipelines used for testing class ZeroDivisionErrorPipeline: def open_spider(self, spider): - a = 1 / 0 + 1 / 0 def process_item(self, item, spider): return item diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 85a24d0bc..a09dcf072 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -142,8 +142,8 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} @defer.inlineCallbacks def test_request_without_meta(self): _, _, stderr = yield self.execute(['--spider', self.spider_name, - '-c', 'parse_request_without_meta', - '--nolinks', + '-c', 'parse_request_without_meta', + '--nolinks', self.url('/html')]) self.assertIn("DEBUG: It Works!", _textmode(stderr)) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index d664b6ade..93fda2648 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -101,15 +101,13 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_local_nofile(self): filepath = 'file:///tests/sample_data/test_site/nothinghere.html' - errcode, out, err = yield self.execute([filepath, '-c', 'item'], - check_code=False) + errcode, out, err = yield self.execute([filepath, '-c', 'item'], check_code=False) self.assertEqual(errcode, 1, out or err) self.assertIn(b'No such file or directory', err) @defer.inlineCallbacks def test_dns_failures(self): url = 'www.somedomainthatdoesntexi.st' - errcode, out, err = yield self.execute([url, '-c', 'item'], - check_code=False) + errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False) self.assertEqual(errcode, 1, out or err) self.assertIn(b'DNS lookup failed', err) diff --git a/tests/test_command_version.py b/tests/test_command_version.py index 4ac7fb786..99c01c2b7 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -23,8 +23,10 @@ class VersionTest(ProcessTest, unittest.TestCase): def test_verbose_output(self): encoding = getattr(sys.stdout, 'encoding') or 'utf-8' _, out, _ = yield self.execute(['-v']) - headers = [l.partition(":")[0].strip() - for l in out.strip().decode(encoding).splitlines()] + headers = [ + line.partition(":")[0].strip() + for line in out.strip().decode(encoding).splitlines() + ] self.assertEqual(headers, ['Scrapy', 'lxml', 'libxml2', 'cssselect', 'parsel', 'w3lib', 'Twisted', 'Python', 'pyOpenSSL', diff --git a/tests/test_contracts.py b/tests/test_contracts.py index d1ce80f9d..99120b128 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -232,7 +232,8 @@ class ContractsManagerTest(unittest.TestCase): # extract contracts correctly contracts = self.conman.extract_contracts(spider.returns_request) self.assertEqual(len(contracts), 2) - self.assertEqual(frozenset(type(x) for x in contracts), + self.assertEqual( + frozenset(type(x) for x in contracts), frozenset([UrlContract, ReturnsContract])) # returns request for valid method diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 4215ca56c..84f80d103 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -104,44 +104,44 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_retry_503(self): crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_retry_conn_failed(self): crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_retry_dns_error(self): crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: # try to fetch the homepage of a non-existent domain yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): - with LogCapture('scrapy', level=logging.ERROR) as l: + with LogCapture('scrapy', level=logging.ERROR) as log: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) - self.assertEqual(len(l.records), 1) - record = l.records[0] + self.assertEqual(len(log.records), 1) + record = log.records[0] self.assertIsNotNone(record.exc_info) self.assertIs(record.exc_info[0], ZeroDivisionError) @defer.inlineCallbacks def test_start_requests_bug_yielding(self): - with LogCapture('scrapy', level=logging.ERROR) as l: + with LogCapture('scrapy', level=logging.ERROR) as log: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) - self.assertEqual(len(l.records), 1) - record = l.records[0] + self.assertEqual(len(log.records), 1) + record = log.records[0] self.assertIsNotNone(record.exc_info) self.assertIs(record.exc_info[0], ZeroDivisionError) @@ -187,25 +187,25 @@ foo body with multiples lines '''}) crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver) - self.assertEqual(str(l).count("Got response 200"), 1) + self.assertEqual(str(log).count("Got response 200"), 1) @defer.inlineCallbacks def test_retry_conn_lost(self): # connection lost after receiving data crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) @defer.inlineCallbacks def test_retry_conn_aborted(self): # connection lost before receiving data crawler = self.runner.create_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver) - self._assert_retried(l) + self._assert_retried(log) def _assert_retried(self, log): self.assertEqual(str(log).count("Retrying"), 2) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 9151278a5..ecc0cd7af 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -87,7 +87,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): class MySpider(scrapy.Spider): name = 'spider' - crawler = Crawler(MySpider, {}) + Crawler(MySpider, {}) assert get_scrapy_root_handler() is None def test_spider_custom_settings_log_level(self): @@ -240,13 +240,13 @@ class CrawlerRunnerHasSpider(unittest.TestCase): def test_crawler_runner_asyncio_enabled_true(self): if self.reactor_pytest == 'asyncio': - runner = CrawlerRunner(settings={ + CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): - runner = CrawlerRunner(settings={ + CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index a169acbe6..5d0a1d0c9 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -6,7 +6,7 @@ class ScrapyUtilsTest(unittest.TestCase): def test_required_openssl_version(self): try: module = import_module('OpenSSL') - except ImportError as ex: + except ImportError: raise unittest.SkipTest("OpenSSL is not available") if hasattr(module, '__version__'): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index f93bce8ef..c1e6f744b 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -730,6 +730,9 @@ class Http11ProxyTestCase(HttpProxyTestCase): class HttpDownloadHandlerMock: + def __init__(self, *args, **kwargs): + pass + def download_request(self, request, spider): return request @@ -853,8 +856,7 @@ class S3TestCase(unittest.TestCase): def test_request_signing4(self): # fetches the access control policy sub-resource for the 'johnsmith' bucket. date = 'Tue, 27 Mar 2007 19:44:46 +0000' - req = Request('s3://johnsmith/?acl', - method='GET', headers={'Date': date}) + req = Request('s3://johnsmith/?acl', method='GET', headers={'Date': date}) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], @@ -879,8 +881,9 @@ class S3TestCase(unittest.TestCase): with self._mocked_date(date): httpreq = self.download_request(req, self.spider) # botocore does not override Date with x-amz-date - self.assertEqual(httpreq.headers['Authorization'], - b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') + self.assertEqual( + httpreq.headers['Authorization'], + b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') def test_request_signing6(self): # uploads an object to a CNAME style virtual hosted bucket with metadata. diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index b686a14d6..d54434c8f 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -63,7 +63,7 @@ class CookiesMiddlewareTest(TestCase): mw = CookiesMiddleware.from_crawler(crawler) with LogCapture('scrapy.downloadermiddlewares.cookies', propagate=False, - level=logging.DEBUG) as l: + level=logging.DEBUG) as log: req = Request('http://scrapytest.org/') res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) @@ -71,7 +71,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request('http://scrapytest.org/sub1/') mw.process_request(req2, crawler.spider) - l.check( + log.check( ('scrapy.downloadermiddlewares.cookies', 'DEBUG', 'Received cookies from: <200 http://scrapytest.org/>\n' @@ -87,7 +87,7 @@ class CookiesMiddlewareTest(TestCase): mw = CookiesMiddleware.from_crawler(crawler) with LogCapture('scrapy.downloadermiddlewares.cookies', propagate=False, - level=logging.DEBUG) as l: + level=logging.DEBUG) as log: req = Request('http://scrapytest.org/') res = Response('http://scrapytest.org/', headers={'Set-Cookie': 'C1=value1; path=/'}) @@ -95,7 +95,7 @@ class CookiesMiddlewareTest(TestCase): req2 = Request('http://scrapytest.org/sub1/') mw.process_request(req2, crawler.spider) - l.check() + log.check() def test_do_not_break_on_non_utf8_header(self): req = Request('http://scrapytest.org/') @@ -139,10 +139,12 @@ class CookiesMiddlewareTest(TestCase): def test_complex_cookies(self): # merge some cookies into jar - cookies = [{'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'}, - {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'}, - {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'}, - {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}] + cookies = [ + {'name': 'C1', 'value': 'value1', 'path': '/foo', 'domain': 'scrapytest.org'}, + {'name': 'C2', 'value': 'value2', 'path': '/bar', 'domain': 'scrapytest.org'}, + {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'}, + {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}, + ] req = Request('http://scrapytest.org/', cookies=cookies) self.mw.process_request(req, self.spider) diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 36743b1de..9841d7a76 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -43,8 +43,11 @@ class TestHttpProxyMiddleware(TestCase): os.environ.pop('file_proxy', None) mw = HttpProxyMiddleware() - for url, proxy in [('http://e.com', http_proxy), - ('https://e.com', https_proxy), ('file://tmp/a', None)]: + for url, proxy in [ + ('http://e.com', http_proxy), + ('https://e.com', https_proxy), + ('file://tmp/a', None), + ]: req = Request(url) assert mw.process_request(req, spider) is None self.assertEqual(req.url, url) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 551e124ab..22f23d7b5 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import unittest from scrapy.downloadermiddlewares.redirect import RedirectMiddleware, MetaRefreshMiddleware @@ -72,7 +70,7 @@ class RedirectMiddlewareTest(unittest.TestCase): url = 'http://www.example.com/302' url2 = 'http://www.example.com/redirected2' req = Request(url, method='POST', body='test', - headers={'Content-Type': 'text/plain', 'Content-length': '4'}) + headers={'Content-Type': 'text/plain', 'Content-length': '4'}) rsp = Response(url, headers={'Location': url2}, status=302) req2 = self.mw.process_response(req, rsp, self.spider) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 9c989977e..29357ba94 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,8 +1,14 @@ import unittest from twisted.internet import defer -from twisted.internet.error import TimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost, TCPTimedOutError +from twisted.internet.error import ( + ConnectError, + ConnectionDone, + ConnectionLost, + ConnectionRefusedError, + DNSLookupError, + TCPTimedOutError, + TimeoutError, +) from twisted.web.client import ResponseFailed from scrapy.downloadermiddlewares.retry import RetryMiddleware @@ -75,9 +81,17 @@ class RetryTest(unittest.TestCase): assert self.crawler.stats.get_value('retry/count') == 2 def test_twistederrors(self): - exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError, - DNSLookupError, ConnectionRefusedError, ConnectionDone, - ConnectError, ConnectionLost, ResponseFailed] + exceptions = [ + ConnectError, + ConnectionDone, + ConnectionLost, + ConnectionRefusedError, + defer.TimeoutError, + DNSLookupError, + ResponseFailed, + TCPTimedOutError, + TimeoutError, + ] for exc in exceptions: req = Request('http://www.scrapytest.org/%s' % exc.__name__) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index a1645ed96..b9452a0e7 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from unittest import mock from twisted.internet import reactor, error diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 7426107c1..41a8d16bc 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -160,7 +160,7 @@ class RFPDupeFilterTest(unittest.TestCase): shutil.rmtree(path) def test_log(self): - with LogCapture() as l: + with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': False, 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(SimpleSpider, settings_dict=settings) @@ -177,15 +177,18 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.log(r2, spider) assert crawler.stats.get_value('dupefilter/filtered') == 2 - l.check_present(('scrapy.dupefilters', 'DEBUG', - ('Filtered duplicate request: ' - ' - no more duplicates will be shown' - ' (see DUPEFILTER_DEBUG to show all duplicates)'))) + log.check_present( + ( + 'scrapy.dupefilters', + 'DEBUG', + 'Filtered duplicate request: - no more duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)' + ) + ) dupefilter.close('finished') def test_log_debug(self): - with LogCapture() as l: + with LogCapture() as log: settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(SimpleSpider, settings_dict=settings) @@ -203,11 +206,19 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.log(r2, spider) assert crawler.stats.get_value('dupefilter/filtered') == 2 - l.check_present(('scrapy.dupefilters', 'DEBUG', - ('Filtered duplicate request: ' - ' (referer: None)'))) - l.check_present(('scrapy.dupefilters', 'DEBUG', - ('Filtered duplicate request: ' - ' (referer: http://scrapytest.org/INDEX.html)'))) + log.check_present( + ( + 'scrapy.dupefilters', + 'DEBUG', + 'Filtered duplicate request: (referer: None)' + ) + ) + log.check_present( + ( + 'scrapy.dupefilters', + 'DEBUG', + 'Filtered duplicate request: (referer: http://scrapytest.org/INDEX.html)' + ) + ) dupefilter.close('finished') diff --git a/tests/test_engine.py b/tests/test_engine.py index 5b7a4e676..acfe94f63 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -13,22 +13,24 @@ module with the ``runserver`` argument:: import os import re import sys +from collections import defaultdict from urllib.parse import urlparse from twisted.internet import reactor, defer -from twisted.web import server, static, util from twisted.trial import unittest +from twisted.web import server, static, util +from pydispatch import dispatcher from scrapy import signals from scrapy.core.engine import ExecutionEngine -from scrapy.utils.test import get_crawler -from pydispatch import dispatcher -from tests import tests_datadir -from scrapy.spiders import Spider +from scrapy.http import Request from scrapy.item import Item, Field from scrapy.linkextractors import LinkExtractor -from scrapy.http import Request +from scrapy.spiders import Spider from scrapy.utils.signal import disconnect_all +from scrapy.utils.test import get_crawler + +from tests import tests_datadir, get_testdata class TestItem(Item): @@ -88,6 +90,8 @@ def start_test_site(debug=False): r = static.File(root_dir) r.putChild(b"redirect", util.Redirect(b"/redirected")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) + numbers = [str(x).encode("utf8") for x in range(2**14)] + r.putChild(b"numbers", static.Data(b"".join(numbers), "text/plain")) port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") if debug: @@ -107,15 +111,20 @@ class CrawlerRun: self.reqreached = [] self.itemerror = [] self.itemresp = [] - self.signals_catched = {} + self.bytes = defaultdict(lambda: list()) + self.signals_caught = {} self.spider_class = spider_class def run(self): self.port = start_test_site() self.portno = self.port.getHost().port - start_urls = [self.geturl("/"), self.geturl("/redirect"), - self.geturl("/redirect")] # a duplicate + start_urls = [ + self.geturl("/"), + self.geturl("/redirect"), + self.geturl("/redirect"), # duplicate + self.geturl("/numbers"), + ] for name, signal in vars(signals).items(): if not name.startswith('_'): @@ -124,6 +133,7 @@ class CrawlerRun: self.crawler = get_crawler(self.spider_class) self.crawler.signals.connect(self.item_scraped, signals.item_scraped) self.crawler.signals.connect(self.item_error, signals.item_error) + self.crawler.signals.connect(self.bytes_received, signals.bytes_received) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.request_dropped, signals.request_dropped) self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader) @@ -155,6 +165,9 @@ class CrawlerRun: def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) + def bytes_received(self, data, request, spider): + self.bytes[request].append(data) + def request_scheduled(self, request, spider): self.reqplug.append((request, spider)) @@ -172,7 +185,7 @@ class CrawlerRun: signalargs = kwargs.copy() sig = signalargs.pop('signal') signalargs.pop('sender', None) - self.signals_catched[sig] = signalargs + self.signals_caught[sig] = signalargs class EngineTest(unittest.TestCase): @@ -183,16 +196,17 @@ class EngineTest(unittest.TestCase): self.run = CrawlerRun(spider) yield self.run.run() self._assert_visited_urls() - self._assert_scheduled_requests(urls_to_visit=8) + self._assert_scheduled_requests(urls_to_visit=9) self._assert_downloaded_responses() self._assert_scraped_items() - self._assert_signals_catched() + self._assert_signals_caught() + self._assert_bytes_received() @defer.inlineCallbacks def test_crawler_dupefilter(self): self.run = CrawlerRun(TestDupeFilterSpider) yield self.run.run() - self._assert_scheduled_requests(urls_to_visit=7) + self._assert_scheduled_requests(urls_to_visit=8) self._assert_dropped_requests() @defer.inlineCallbacks @@ -229,8 +243,8 @@ class EngineTest(unittest.TestCase): def _assert_downloaded_responses(self): # response tests - self.assertEqual(8, len(self.run.respplug)) - self.assertEqual(8, len(self.run.reqreached)) + self.assertEqual(9, len(self.run.respplug)) + self.assertEqual(9, len(self.run.reqreached)) for response, _ in self.run.respplug: if self.run.getpath(response.url) == '/item999.html': @@ -263,19 +277,61 @@ class EngineTest(unittest.TestCase): self.assertEqual('Item 2 name', item['name']) self.assertEqual('200', item['price']) - def _assert_signals_catched(self): - assert signals.engine_started in self.run.signals_catched - assert signals.engine_stopped in self.run.signals_catched - assert signals.spider_opened in self.run.signals_catched - assert signals.spider_idle in self.run.signals_catched - assert signals.spider_closed in self.run.signals_catched + def _assert_bytes_received(self): + self.assertEqual(9, len(self.run.bytes)) + for request, data in self.run.bytes.items(): + joined_data = b"".join(data) + if self.run.getpath(request.url) == "/": + self.assertEqual(joined_data, get_testdata("test_site", "index.html")) + elif self.run.getpath(request.url) == "/item1.html": + self.assertEqual(joined_data, get_testdata("test_site", "item1.html")) + elif self.run.getpath(request.url) == "/item2.html": + self.assertEqual(joined_data, get_testdata("test_site", "item2.html")) + elif self.run.getpath(request.url) == "/redirected": + self.assertEqual(joined_data, b"Redirected here") + elif self.run.getpath(request.url) == '/redirect': + self.assertEqual( + joined_data, + b"\n\n" + b" \n" + b" \n" + b" \n" + b" \n" + b" click here\n" + b" \n" + b"\n" + ) + elif self.run.getpath(request.url) == "/tem999.html": + self.assertEqual( + joined_data, + b"\n\n" + b" 404 - No Such Resource\n" + b" \n" + b"

No Such Resource

\n" + b"

File not found.

\n" + b" \n" + b"\n" + ) + elif self.run.getpath(request.url) == "/numbers": + # signal was fired multiple times + self.assertTrue(len(data) > 1) + # bytes were received in order + numbers = [str(x).encode("utf8") for x in range(2**14)] + self.assertEqual(joined_data, b"".join(numbers)) + + def _assert_signals_caught(self): + assert signals.engine_started in self.run.signals_caught + assert signals.engine_stopped in self.run.signals_caught + assert signals.spider_opened in self.run.signals_caught + assert signals.spider_idle in self.run.signals_caught + assert signals.spider_closed in self.run.signals_caught self.assertEqual({'spider': self.run.spider}, - self.run.signals_catched[signals.spider_opened]) + self.run.signals_caught[signals.spider_opened]) self.assertEqual({'spider': self.run.spider}, - self.run.signals_catched[signals.spider_idle]) + self.run.signals_caught[signals.spider_idle]) self.assertEqual({'spider': self.run.spider, 'reason': 'finished'}, - self.run.signals_catched[signals.spider_closed]) + self.run.signals_caught[signals.spider_closed]) @defer.inlineCallbacks def test_close_downloader(self): diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 873a97248..1e716b94a 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -11,8 +11,6 @@ class TelnetExtensionTest(unittest.TestCase): def _get_console_and_portal(self, settings=None): crawler = get_crawler(settings_dict=settings) console = TelnetConsole(crawler) - username = console.username - password = console.password # This function has some side effects we don't need for this test console._get_telnet_vars = lambda: {} diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index e02b0b840..cbc81bc35 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -715,7 +715,6 @@ class FeedExportTest(unittest.TestCase): @defer.inlineCallbacks def test_export_encoding(self): items = [dict({'foo': u'Test\xd6'})] - header = ['foo'] formats = { 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'), diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 3b6d119a9..a672963f3 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -415,8 +415,7 @@ class FormRequestTest(RequestTest): # using multiples values for a single key data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']} r3 = self.request_class("http://www.example.com", formdata=data) - self.assertQueryEqual(r3.body, - b'colours=red&colours=blue&colours=green&price=%C2%A3+100') + self.assertQueryEqual(r3.body, b'colours=red&colours=blue&colours=green&price=%C2%A3+100') def test_from_response_post(self): response = _buildresponse( @@ -426,8 +425,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') @@ -446,8 +444,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') @@ -468,8 +465,7 @@ class FormRequestTest(RequestTest): url="http://www.example.com/this/list.html", encoding='latin1', ) - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') @@ -488,8 +484,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html") - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + req = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers[b'Content-type'], b'application/x-www-form-urlencoded') @@ -502,11 +497,13 @@ class FormRequestTest(RequestTest): def test_from_response_duplicate_form_key(self): response = _buildresponse( - '
', - url='http://www.example.com') - req = self.request_class.from_response(response, - method='GET', - formdata=(('foo', 'bar'), ('foo', 'baz'))) + '
', + url='http://www.example.com') + req = self.request_class.from_response( + response=response, + method='GET', + formdata=(('foo', 'bar'), ('foo', 'baz')), + ) self.assertEqual(urlparse(req.url).hostname, 'www.example.com') self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz') @@ -530,9 +527,11 @@ class FormRequestTest(RequestTest): """) - req = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}, - headers={"Accept-Encoding": "gzip,deflate"}) + req = self.request_class.from_response( + response=response, + formdata={'one': ['two', 'three'], 'six': 'seven'}, + headers={"Accept-Encoding": "gzip,deflate"}, + ) self.assertEqual(req.method, 'POST') self.assertEqual(req.headers['Content-type'], b'application/x-www-form-urlencoded') self.assertEqual(req.headers['Accept-Encoding'], b'gzip,deflate') @@ -545,8 +544,7 @@ class FormRequestTest(RequestTest): """, url="http://www.example.com/this/list.html") - r1 = self.request_class.from_response(response, - formdata={'one': ['two', 'three'], 'six': 'seven'}) + r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}) self.assertEqual(r1.method, 'GET') self.assertEqual(urlparse(r1.url).hostname, "www.example.com") self.assertEqual(urlparse(r1.url).path, "/this/get.php") @@ -580,9 +578,9 @@ class FormRequestTest(RequestTest): def test_from_response_override_method(self): response = _buildresponse( - ''' -
- ''') + ''' +
+ ''') request = FormRequest.from_response(response) self.assertEqual(request.method, 'GET') request = FormRequest.from_response(response, method='POST') @@ -590,9 +588,9 @@ class FormRequestTest(RequestTest): def test_from_response_override_url(self): response = _buildresponse( - ''' -
- ''') + ''' +
+ ''') request = FormRequest.from_response(response) self.assertEqual(request.url, 'http://example.com/app') request = FormRequest.from_response(response, url='http://foo.bar/absolute') diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 522ec4875..f9b6b92f4 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,8 +1,9 @@ -# -*- coding: utf-8 -*- import unittest +from warnings import catch_warnings from w3lib.encoding import resolve_encoding +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import (Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers) from scrapy.selector import Selector @@ -485,8 +486,10 @@ class TextResponseTest(BaseResponseTest): response.xpath("normalize-space(//p[@class=\"content\"])").getall(), ) self.assertEqual( - response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()", - pclass="content", pcount=1).getall(), + response.xpath( + "//title[count(following::p[@class=$pclass])=$pcount]/text()", + pclass="content", pcount=1, + ).getall(), response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(), ) @@ -566,12 +569,14 @@ class TextResponseTest(BaseResponseTest): 'http://example.com', body=b'''click me''' ) - self._assert_followed_url(resp.css('a')[0], - 'http://example.com/foo', - response=resp) - self._assert_followed_url(resp.css('a::attr(href)')[0], - 'http://example.com/foo', - response=resp) + self._assert_followed_url( + resp.css('a')[0], + 'http://example.com/foo', + response=resp) + self._assert_followed_url( + resp.css('a::attr(href)')[0], + 'http://example.com/foo', + response=resp) def test_follow_encoding(self): resp1 = self.response_class( @@ -661,6 +666,13 @@ class TextResponseTest(BaseResponseTest): with self.assertRaises(ValueError): response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]') + def test_body_as_unicode_deprecation_warning(self): + with catch_warnings(record=True) as warnings: + r1 = self.response_class("http://www.example.com", body=u'Hello', encoding='utf-8') + self.assertEqual(r1.body_as_unicode(), u'Hello') + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) + class HtmlResponseTest(TextResponseTest): diff --git a/tests/test_item.py b/tests/test_item.py index 4017f6e84..687867fe0 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -162,8 +162,7 @@ class ItemTest(unittest.TestCase): item = D(save='X', load='Y') self.assertEqual(item['save'], 'X') self.assertEqual(item['load'], 'Y') - self.assertEqual(D.fields, {'load': {'default': 'A'}, - 'save': {'default': 'A'}}) + self.assertEqual(D.fields, {'load': {'default': 'A'}, 'save': {'default': 'A'}}) # D class inverted class E(C, B): @@ -171,8 +170,7 @@ class ItemTest(unittest.TestCase): self.assertEqual(E(save='X')['save'], 'X') self.assertEqual(E(load='X')['load'], 'X') - self.assertEqual(E.fields, {'load': {'default': 'C'}, - 'save': {'default': 'C'}}) + self.assertEqual(E.fields, {'load': {'default': 'C'}, 'save': {'default': 'C'}}) def test_metaclass_multiple_inheritance_diamond(self): class A(Item): @@ -193,8 +191,9 @@ class ItemTest(unittest.TestCase): self.assertEqual(D(save='X')['save'], 'X') self.assertEqual(D(load='X')['load'], 'X') - self.assertEqual(D.fields, {'save': {'default': 'C'}, - 'load': {'default': 'D'}, 'update': {'default': 'D'}}) + self.assertEqual( + D.fields, + {'save': {'default': 'C'}, 'load': {'default': 'D'}, 'update': {'default': 'D'}}) # D class inverted class E(C, B): @@ -202,8 +201,9 @@ class ItemTest(unittest.TestCase): self.assertEqual(E(save='X')['save'], 'X') self.assertEqual(E(load='X')['load'], 'X') - self.assertEqual(E.fields, {'save': {'default': 'C'}, - 'load': {'default': 'E'}, 'update': {'default': 'C'}}) + self.assertEqual( + E.fields, + {'save': {'default': 'C'}, 'load': {'default': 'E'}, 'update': {'default': 'C'}}) def test_metaclass_multiple_inheritance_without_metaclass(self): class A(Item): @@ -223,8 +223,7 @@ class ItemTest(unittest.TestCase): self.assertRaises(KeyError, D, not_allowed='value') self.assertEqual(D(save='X')['save'], 'X') - self.assertEqual(D.fields, {'save': {'default': 'A'}, - 'load': {'default': 'A'}}) + self.assertEqual(D.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}}) # D class inverted class E(C, B): @@ -232,8 +231,7 @@ class ItemTest(unittest.TestCase): self.assertRaises(KeyError, E, not_allowed='value') self.assertEqual(E(save='X')['save'], 'X') - self.assertEqual(E.fields, {'save': {'default': 'A'}, - 'load': {'default': 'A'}}) + self.assertEqual(E.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}}) def test_to_dict(self): class TestItem(Item): @@ -264,12 +262,12 @@ class ItemTest(unittest.TestCase): """Make sure the DictItem deprecation warning is not issued for Item""" with catch_warnings(record=True) as warnings: - item = Item() + Item() self.assertEqual(len(warnings), 0) class SubclassedItem(Item): pass - subclassed_item = SubclassedItem() + SubclassedItem() self.assertEqual(len(warnings), 0) @@ -321,13 +319,13 @@ class DictItemTest(unittest.TestCase): def test_deprecation_warning(self): with catch_warnings(record=True) as warnings: - dict_item = DictItem() + DictItem() self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) with catch_warnings(record=True) as warnings: class SubclassedDictItem(DictItem): pass - subclassed_dict_item = SubclassedDictItem() + SubclassedDictItem() self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].category, ScrapyDeprecationWarning) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 68e8514ba..2a2650480 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -1,3 +1,4 @@ +import pickle import re import unittest from warnings import catch_warnings @@ -171,9 +172,9 @@ class Base: self.assertEqual(lx.matches(url1), False) self.assertEqual(lx.matches(url2), True) - lx = self.extractor_cls(allow=('blah1',), deny=('blah2',), - allow_domains=('blah1.com',), - deny_domains=('blah2.com',)) + lx = self.extractor_cls(allow=['blah1'], deny=['blah2'], + allow_domains=['blah1.com'], + deny_domains=['blah2.com']) self.assertEqual(lx.matches('http://blah1.com/blah1'), True) self.assertEqual(lx.matches('http://blah1.com/blah2'), False) self.assertEqual(lx.matches('http://blah2.com/blah1'), False) @@ -462,6 +463,10 @@ class Base: Link(url='ftp://www.external.com/', text=u'An Item', fragment='', nofollow=False), ]) + def test_pickle_extractor(self): + lx = self.extractor_cls() + self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls) + class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): extractor_cls = LxmlLinkExtractor diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index cd6cb8016..7064337ad 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -34,15 +34,15 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, - "Crawled (200) (referer: None)") + self.assertEqual(logline, "Crawled (200) (referer: None)") def test_crawled_without_referer(self): req = Request("http://www.example.com", headers={'referer': 'http://example.com'}) res = Response("http://www.example.com", flags=['cached']) logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, + self.assertEqual( + logline, "Crawled (200) (referer: http://example.com) ['cached']") def test_flags_in_request(self): @@ -50,8 +50,9 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, - "Crawled (200) ['test', 'flag'] (referer: None)") + self.assertEqual( + logline, + "Crawled (200) ['test', 'flag'] (referer: None)") def test_dropped(self): item = {} @@ -140,7 +141,8 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, + self.assertEqual( + logline, "Crawled (200) (referer: None) []") def test_crawled_without_referer(self): @@ -148,7 +150,8 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, + self.assertEqual( + logline, "Crawled (200) (referer: http://example.com) ['cached']") def test_flags_in_request(self): @@ -156,7 +159,9 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, "Crawled (200) (referer: None) ['test', 'flag']") + self.assertEqual( + logline, + "Crawled (200) (referer: None) ['test', 'flag']") class SkipMessagesLogFormatter(LogFormatter): diff --git a/tests/test_mail.py b/tests/test_mail.py index f5cb81a8b..53dbc0686 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -49,7 +49,7 @@ class MailSenderTest(unittest.TestCase): mailsender = MailSender(debug=True) mailsender.send(to=['test@scrapy.org'], subject='subject', body='body', - attachs=attachs, _callback=self._catch_mail_sent) + attachs=attachs, _callback=self._catch_mail_sent) assert self.catched_msg self.assertEqual(self.catched_msg['to'], ['test@scrapy.org']) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 3af514bb0..3364d2258 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -69,11 +69,14 @@ class MiddlewareManagerTest(unittest.TestCase): def test_methods(self): mwman = TestMiddlewareManager(M1(), M2(), M3()) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']], + self.assertEqual( + [x.__self__.__class__ for x in mwman.methods['open_spider']], [M1, M2]) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']], + self.assertEqual( + [x.__self__.__class__ for x in mwman.methods['close_spider']], [M2, M1]) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']], + self.assertEqual( + [x.__self__.__class__ for x in mwman.methods['process']], [M1, M3]) def test_enabled(self): diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 962c33144..6d15aaf31 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os import shutil @@ -44,9 +43,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): name = 'redirectedmedia' def _process_url(self, url): - return add_or_replace_parameter( - self.mockserver.url('/redirect-to'), - 'goto', url) + return add_or_replace_parameter(self.mockserver.url('/redirect-to'), 'goto', url) class FileDownloadCrawlTestCase(TestCase): @@ -134,7 +131,8 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media(self): crawler = self._create_crawler(MediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_downloaded(self.items, str(log)) @@ -143,7 +141,8 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_wrong_urls(self): crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_download_failure(crawler, self.items, 404, str(log)) @@ -152,7 +151,8 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_redirected_default_failure(self): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver) @@ -166,7 +166,8 @@ class FileDownloadCrawlTestCase(TestCase): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 5018d6802..5ba03ff4c 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -15,7 +15,7 @@ from scrapy.utils.python import to_bytes skip = False try: from PIL import Image -except ImportError as e: +except ImportError: skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow' else: encoders = set(('jpeg_encoder', 'jpeg_decoder')) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 949f0dea1..19ff00350 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -63,21 +63,21 @@ class BaseMediaPipelineTestCase(unittest.TestCase): fail = Failure(Exception()) results = [(True, 1), (False, fail)] - with LogCapture() as l: + with LogCapture() as log: new_item = self.pipe.item_completed(results, item, self.info) assert new_item is item - assert len(l.records) == 1 - record = l.records[0] + assert len(log.records) == 1 + record = log.records[0] assert record.levelname == 'ERROR' self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail)) # disable failure logging and check again self.pipe.LOG_FAILED_RESULTS = False - with LogCapture() as l: + with LogCapture() as log: new_item = self.pipe.item_completed(results, item, self.info) assert new_item is item - assert len(l.records) == 0 + assert len(log.records) == 0 @inlineCallbacks def test_default_process_item(self): @@ -214,9 +214,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = dict(requests=req) new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item['results'], [(True, rsp)]) - self.assertEqual(self.pipe._mockcalled, - ['get_media_requests', 'media_to_download', - 'media_downloaded', 'request_callback', 'item_completed']) + self.assertEqual( + self.pipe._mockcalled, + ['get_media_requests', 'media_to_download', 'media_downloaded', 'request_callback', 'item_completed']) @inlineCallbacks def test_result_failure(self): @@ -227,9 +227,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = dict(requests=req) new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item['results'], [(False, fail)]) - self.assertEqual(self.pipe._mockcalled, - ['get_media_requests', 'media_to_download', - 'media_failed', 'request_errback', 'item_completed']) + self.assertEqual( + self.pipe._mockcalled, + ['get_media_requests', 'media_to_download', 'media_failed', 'request_errback', 'item_completed']) @inlineCallbacks def test_mix_of_success_and_failure(self): diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 188ec68dd..4763a5417 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -76,35 +76,35 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_connect_tunnel(self): crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) + self._assert_got_response_code(200, log) @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6) @defer.inlineCallbacks def test_https_connect_tunnel_error(self): crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl("https://localhost:99999/status?n=200") - self._assert_got_tunnel_error(l) + self._assert_got_tunnel_error(log) @defer.inlineCallbacks def test_https_tunnel_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) crawler = get_crawler(SimpleSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) # The proxy returns a 407 error code but it does not reach the client; # he just sees a TunnelError. - self._assert_got_tunnel_error(l) + self._assert_got_tunnel_error(log) @defer.inlineCallbacks def test_https_tunnel_without_leak_proxy_authorization_header(self): request = Request(self.mockserver.url("/echo", is_secure=True)) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as l: + with LogCapture() as log: yield crawler.crawl(seed=request) - self._assert_got_response_code(200, l) + self._assert_got_response_code(200, log) echo = json.loads(crawler.spider.meta['responses'][0].text) self.assertTrue('Proxy-Authorization' not in echo['headers']) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index 8cdf7a176..9e63ac924 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import unittest from scrapy.responsetypes import responsetypes diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index fda44653a..2da6aa4b5 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -314,13 +314,17 @@ class BaseSettingsTest(unittest.TestCase): 'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'), 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) - self.assertDictEqual(s.copy_to_dict(), - {'HASNOBASE': {3: 3000}, - 'TEST': {1: 10, 3: 30}, - 'TEST_BASE': {1: 1, 2: 2}, - 'TEST_BOOLEAN': False, - 'TEST_LIST': [1, 2], - 'TEST_STRING': 'a string'}) + self.assertDictEqual( + s.copy_to_dict(), + { + 'HASNOBASE': {3: 3000}, + 'TEST': {1: 10, 3: 30}, + 'TEST_BASE': {1: 1, 2: 2}, + 'TEST_LIST': [1, 2], + 'TEST_BOOLEAN': False, + 'TEST_STRING': 'a string', + } + ) def test_freeze(self): self.settings.freeze() diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index d8be6e277..265970b43 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -40,7 +40,8 @@ class SpiderLoaderTest(unittest.TestCase): verifyObject(ISpiderLoader, self.spider_loader) def test_list(self): - self.assertEqual(set(self.spider_loader.list()), + self.assertEqual( + set(self.spider_loader.list()), set(['spider1', 'spider2', 'spider3', 'spider4'])) def test_load(self): @@ -48,17 +49,23 @@ class SpiderLoaderTest(unittest.TestCase): self.assertEqual(spider1.__name__, 'Spider1') def test_find_by_request(self): - self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy1.org/test')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://scrapy1.org/test')), ['spider1']) - self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy2.org/test')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://scrapy2.org/test')), ['spider2']) - self.assertEqual(set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))), + self.assertEqual( + set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))), set(['spider1', 'spider2'])) - self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy999.org/test')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://scrapy999.org/test')), []) - self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://spider3.com')), []) - self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')), + self.assertEqual( + self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')), ['spider3']) def test_load_spider_module(self): @@ -137,6 +144,11 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): msg = str(w[0].message) self.assertIn("several spiders with the same name", msg) self.assertIn("'spider3'", msg) + self.assertTrue(msg.count("'spider3'") == 2) + + self.assertNotIn("'spider1'", msg) + self.assertNotIn("'spider2'", msg) + self.assertNotIn("'spider4'", msg) spiders = set(spider_loader.list()) self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) @@ -156,7 +168,13 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase): msg = str(w[0].message) self.assertIn("several spiders with the same name", msg) self.assertIn("'spider1'", msg) + self.assertTrue(msg.count("'spider1'") == 2) + self.assertIn("'spider2'", msg) + self.assertTrue(msg.count("'spider2'") == 2) + + self.assertNotIn("'spider3'", msg) + self.assertNotIn("'spider4'", msg) spiders = set(spider_loader.list()) self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4'])) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 6b61df56f..e032b247c 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -21,10 +21,10 @@ class _HttpErrorSpider(MockServerSpider): def __init__(self, *args, **kwargs): super(_HttpErrorSpider, self).__init__(*args, **kwargs) self.start_urls = [ - self.mockserver.url("/status?n=200"), - self.mockserver.url("/status?n=404"), - self.mockserver.url("/status?n=402"), - self.mockserver.url("/status?n=500"), + self.mockserver.url("/status?n=200"), + self.mockserver.url("/status?n=404"), + self.mockserver.url("/status?n=402"), + self.mockserver.url("/status?n=500"), ] self.failed = set() self.skipped = set() @@ -68,29 +68,23 @@ class TestHttpErrorMiddleware(TestCase): self.res200, self.res404 = _responses(self.req, [200, 404]) def test_process_spider_input(self): - self.assertEqual(None, - self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, self.res404, self.spider) + self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) def test_process_spider_exception(self): - self.assertEqual([], - self.mw.process_spider_exception(self.res404, - HttpError(self.res404), self.spider)) - self.assertEqual(None, - self.mw.process_spider_exception(self.res404, - Exception(), self.spider)) + self.assertEqual( + [], + self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider)) + self.assertIsNone(self.mw.process_spider_exception(self.res404, Exception(), self.spider)) def test_handle_httpstatus_list(self): res = self.res404.copy() res.request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) - self.assertEqual(None, - self.mw.process_spider_input(res, self.spider)) + self.assertIsNone(self.mw.process_spider_input(res, self.spider)) self.spider.handle_httpstatus_list = [404] - self.assertEqual(None, - self.mw.process_spider_input(self.res404, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) class TestHttpErrorMiddlewareSettings(TestCase): @@ -103,12 +97,9 @@ class TestHttpErrorMiddlewareSettings(TestCase): self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertEqual(None, - self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, self.res404, self.spider) - self.assertEqual(None, - self.mw.process_spider_input(self.res402, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) + self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) @@ -117,17 +108,13 @@ class TestHttpErrorMiddlewareSettings(TestCase): res402 = self.res402.copy() res402.request = request - self.assertEqual(None, - self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, res402, self.spider) + self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] - self.assertEqual(None, - self.mw.process_spider_input(self.res404, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, self.res402, self.spider) + self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider) class TestHttpErrorMiddlewareHandleAll(TestCase): @@ -139,10 +126,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertEqual(None, - self.mw.process_spider_input(self.res200, self.spider)) - self.assertEqual(None, - self.mw.process_spider_input(self.res404, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) def test_meta_overrides_settings(self): request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) @@ -151,10 +136,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402 = self.res402.copy() res402.request = request - self.assertEqual(None, - self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, res402, self.spider) + self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) class TestHttpErrorMiddlewareIntegrational(TrialTestCase): diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index b96807bc2..0f4b98a07 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -22,20 +22,24 @@ class TestOffsiteMiddleware(TestCase): def test_process_spider_output(self): res = Response('http://scrapytest.org') - onsite_reqs = [Request('http://scrapytest.org/1'), - Request('http://scrapy.org/1'), - Request('http://sub.scrapy.org/1'), - Request('http://offsite.tld/letmepass', dont_filter=True), - Request('http://scrapy.test.org/'), - Request('http://scrapy.test.org:8000/')] - offsite_reqs = [Request('http://scrapy2.org'), - Request('http://offsite.tld/'), - Request('http://offsite.tld/scrapytest.org'), - Request('http://offsite.tld/rogue.scrapytest.org'), - Request('http://rogue.scrapytest.org.haha.com'), - Request('http://roguescrapytest.org'), - Request('http://test.org/'), - Request('http://notscrapy.test.org/')] + onsite_reqs = [ + Request('http://scrapytest.org/1'), + Request('http://scrapy.org/1'), + Request('http://sub.scrapy.org/1'), + Request('http://offsite.tld/letmepass', dont_filter=True), + Request('http://scrapy.test.org/'), + Request('http://scrapy.test.org:8000/'), + ] + offsite_reqs = [ + Request('http://scrapy2.org'), + Request('http://offsite.tld/'), + Request('http://offsite.tld/scrapytest.org'), + Request('http://offsite.tld/rogue.scrapytest.org'), + Request('http://rogue.scrapytest.org.haha.com'), + Request('http://roguescrapytest.org'), + Request('http://test.org/'), + Request('http://notscrapy.test.org/'), + ] reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider)) diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 41589177a..ca765518b 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -459,7 +459,6 @@ class TestRequestMetaSettingFallback(TestCase): target = 'http://www.example.com' for settings, response_headers, request_meta, policy_class, check_warning in self.params[3:]: - spider = Spider('foo') mw = RefererMiddleware(Settings(settings)) response = Response(origin, headers=response_headers) @@ -511,7 +510,7 @@ class TestSettingsPolicyByName(TestCase): def test_invalid_name(self): settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'}) with self.assertRaises(RuntimeError): - mw = RefererMiddleware(settings) + RefererMiddleware(settings) class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 5ad8035f7..becacce62 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,4 +1,5 @@ import pickle +import sys from queuelib.tests import test_queue as t from scrapy.squeues import ( @@ -28,31 +29,13 @@ class TestLoader(ItemLoader): def nonserializable_object_test(self): q = self.queue() - try: - pickle.dumps(lambda x: x) - except Exception: - # Trigger Twisted bug #7989 - import twisted.persisted.styles # NOQA - self.assertRaises(ValueError, q.push, lambda x: x) - else: - # Use a different unpickleable object - class A: - pass - - a = A() - a.__reduce__ = a.__reduce_ex__ = None - self.assertRaises(ValueError, q.push, a) + self.assertRaises(ValueError, q.push, lambda x: x) # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) sel = Selector(text='

some text

') self.assertRaises(ValueError, q.push, sel) -class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): - - chunksize = 100000 - - def queue(self): - return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize) +class FifoDiskQueueTestMixin: def test_serialize(self): q = self.queue() @@ -66,6 +49,13 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): test_nonserializable_object = nonserializable_object_test +class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): + chunksize = 100000 + + def queue(self): + return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize) + + class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 @@ -82,7 +72,7 @@ class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 4 -class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest): +class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): chunksize = 100000 @@ -99,12 +89,12 @@ class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest): def test_serialize_loader(self): q = self.queue() - l = TestLoader() - q.push(l) - l2 = q.pop() - assert isinstance(l2, TestLoader) - assert l2.default_item_class is TestItem - self.assertEqual(l2.name_out('x'), 'xx') + loader = TestLoader() + q.push(loader) + loader2 = q.pop() + assert isinstance(loader2, TestLoader) + assert loader2.default_item_class is TestItem + self.assertEqual(loader2.name_out('x'), 'xx') def test_serialize_request_recursive(self): q = self.queue() @@ -116,6 +106,21 @@ class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest): self.assertEqual(r.url, r2.url) assert r2.meta['request'] is r2 + def test_non_pickable_object(self): + q = self.queue() + try: + q.push(lambda x: x) + except ValueError as exc: + if hasattr(sys, "pypy_version_info"): + self.assertIsInstance(exc.__context__, pickle.PicklingError) + else: + self.assertIsInstance(exc.__context__, AttributeError) + sel = Selector(text='

some text

') + try: + q.push(sel) + except ValueError as exc: + self.assertIsInstance(exc.__context__, TypeError) + class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 1 @@ -133,10 +138,7 @@ class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 4 -class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): - - def queue(self): - return MarshalLifoDiskQueue(self.qpath) +class LifoDiskQueueTestMixin: def test_serialize(self): q = self.queue() @@ -150,7 +152,13 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): test_nonserializable_object = nonserializable_object_test -class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest): +class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): + + def queue(self): + return MarshalLifoDiskQueue(self.qpath) + + +class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): def queue(self): return PickleLifoDiskQueue(self.qpath) @@ -165,12 +173,12 @@ class PickleLifoDiskQueueTest(MarshalLifoDiskQueueTest): def test_serialize_loader(self): q = self.queue() - l = TestLoader() - q.push(l) - l2 = q.pop() - assert isinstance(l2, TestLoader) - assert l2.default_item_class is TestItem - self.assertEqual(l2.name_out('x'), 'xx') + loader = TestLoader() + q.push(loader) + loader2 = q.pop() + assert isinstance(loader2, TestLoader) + assert loader2.default_item_class is TestItem + self.assertEqual(loader2.name_out('x'), 'xx') def test_serialize_request_recursive(self): q = self.queue() diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 332120021..e5d3ef582 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -93,7 +93,8 @@ class BuildComponentListTest(unittest.TestCase): class UtilsConfTestCase(unittest.TestCase): def test_arglist_to_dict(self): - self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']), + self.assertEqual( + arglist_to_dict(['arg1=val1', 'arg2=val2']), {'arg1': 'val1', 'arg2': 'val2'}) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index a3b6e64f1..2d4b88121 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -64,7 +64,7 @@ class DeferUtilsTest(unittest.TestCase): gotexc = False try: yield process_chain([cb1, cb_fail, cb3], 'res', 'v1', 'v2') - except TypeError as e: + except TypeError: gotexc = True self.assertTrue(gotexc) @@ -104,7 +104,7 @@ class IterErrbackTest(unittest.TestCase): def iterbad(): for x in range(10): if x == 5: - a = 1 / 0 + 1 / 0 yield x errors = [] diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index b17e17f2f..35d35b45d 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import inspect import unittest from unittest import mock @@ -26,7 +25,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): def test_no_warning_on_definition(self): with warnings.catch_warnings(record=True) as w: - Deprecated = create_deprecated_class('Deprecated', NewName) + create_deprecated_class('Deprecated', NewName) w = self._mywarnings(w) self.assertEqual(w, []) @@ -218,7 +217,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): def test_deprecate_a_class_with_custom_metaclass(self): Meta1 = type('Meta1', (type,), {}) New = Meta1('New', (), {}) - Deprecated = create_deprecated_class('Deprecated', New) + create_deprecated_class('Deprecated', New) def test_deprecate_subclass_of_deprecated_class(self): with warnings.catch_warnings(record=True) as w: diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 46aaaecbc..824170d71 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os from twisted.trial import unittest @@ -47,8 +46,7 @@ class XmliterTestCase(unittest.TestCase): """ response = XmlResponse(url="http://example.com", body=body) - nodenames = [e.xpath('name()').getall() - for e in self.xmliter(response, 'matchme...')] + nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')] self.assertEqual(nodenames, [['matchme...']]) def test_xmliter_unicode(self): @@ -359,15 +357,23 @@ class UtilsCsvTestCase(unittest.TestCase): response = TextResponse(url="http://example.com/", body=body1, encoding='latin1') csv = csviter(response) - self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'latin1', u'value': u'test'}, - {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}]) + self.assertEqual( + list(csv), + [ + {u'id': u'1', u'name': u'latin1', u'value': u'test'}, + {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}, + ] + ) response = TextResponse(url="http://example.com/", body=body2, encoding='cp852') csv = csviter(response) - self.assertEqual([row for row in csv], - [{u'id': u'1', u'name': u'cp852', u'value': u'test'}, - {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}]) + self.assertEqual( + list(csv), + [ + {u'id': u'1', u'name': u'cp852', u'value': u'test'}, + {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}, + ] + ) class TestHelper(unittest.TestCase): diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 21100aeb8..535f56691 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import sys import logging import unittest @@ -35,31 +34,27 @@ class TopLevelFormatterTest(unittest.TestCase): def test_top_level_logger(self): logger = logging.getLogger('test') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('test', 'WARNING', 'test log msg')) + log.check(('test', 'WARNING', 'test log msg')) def test_children_logger(self): logger = logging.getLogger('test.test1') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('test', 'WARNING', 'test log msg')) + log.check(('test', 'WARNING', 'test log msg')) def test_overlapping_name_logger(self): logger = logging.getLogger('test2') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('test2', 'WARNING', 'test log msg')) + log.check(('test2', 'WARNING', 'test log msg')) def test_different_name_logger(self): logger = logging.getLogger('different') - with self.handler as l: + with self.handler as log: logger.warning('test log msg') - - l.check(('different', 'WARNING', 'test log msg')) + log.check(('different', 'WARNING', 'test log msg')) class LogCounterHandlerTest(unittest.TestCase): @@ -108,6 +103,6 @@ class StreamLoggerTest(unittest.TestCase): sys.stdout = self.stdout def test_redirect(self): - with LogCapture() as l: + with LogCapture() as log: print('test log msg') - l.check(('test', 'ERROR', 'test log msg')) + log.check(('test', 'ERROR', 'test log msg')) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 6f945cd01..28205e0d9 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -67,12 +67,12 @@ class UtilsMiscTestCase(unittest.TestCase): assert hasattr(arg_to_iter(100), '__iter__') assert hasattr(arg_to_iter('lala'), '__iter__') assert hasattr(arg_to_iter([1, 2, 3]), '__iter__') - assert hasattr(arg_to_iter(l for l in 'abcd'), '__iter__') + assert hasattr(arg_to_iter(c for c in 'abcd'), '__iter__') self.assertEqual(list(arg_to_iter(None)), []) self.assertEqual(list(arg_to_iter('lala')), ['lala']) self.assertEqual(list(arg_to_iter(100)), [100]) - self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c']) + self.assertEqual(list(arg_to_iter(c for c in 'abc')), ['a', 'b', 'c']) self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3]) self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}]) self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")]) @@ -114,8 +114,12 @@ class UtilsMiscTestCase(unittest.TestCase): # 2. with from_settings() constructor # 3. with from_crawler() constructor # 4. with from_settings() and from_crawler() constructor - spec_sets = ([], ['from_settings'], ['from_crawler'], - ['from_settings', 'from_crawler']) + spec_sets = ( + ['__qualname__'], + ['__qualname__', 'from_settings'], + ['__qualname__', 'from_crawler'], + ['__qualname__', 'from_settings', 'from_crawler'], + ) for specs in spec_sets: m = mock.MagicMock(spec_set=specs) _test_with_settings(m, settings) @@ -123,7 +127,7 @@ class UtilsMiscTestCase(unittest.TestCase): _test_with_crawler(m, settings, crawler) # Check adoption of crawler settings - m = mock.MagicMock(spec_set=['from_settings']) + m = mock.MagicMock(spec_set=['__qualname__', 'from_settings']) create_instance(m, None, crawler, *args, **kwargs) m.from_settings.assert_called_once_with(crawler.settings, *args, **kwargs) @@ -131,6 +135,10 @@ class UtilsMiscTestCase(unittest.TestCase): with self.assertRaises(ValueError): create_instance(m, None, None) + m.from_settings.return_value = None + with self.assertRaises(TypeError): + create_instance(m, settings, None) + def test_set_environ(self): assert os.environ.get('some_test_environ') is None with set_environ(some_test_environ='test_value'): diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 50b026d1c..450e4bdca 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -15,7 +15,8 @@ class RequestSerializationTest(unittest.TestCase): self._assert_serializes_ok(r) def test_all_attributes(self): - r = Request("http://www.example.com", + r = Request( + url="http://www.example.com", callback=self.spider.parse_item, errback=self.spider.handle_error, method="POST", diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 45f0f59e4..50efb63ca 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -36,8 +36,9 @@ class UtilsRequestTest(unittest.TestCase): self.assertEqual(request_fingerprint(r1), request_fingerprint(r1, include_headers=['Accept-Language'])) - self.assertNotEqual(request_fingerprint(r1), - request_fingerprint(r2, include_headers=['Accept-Language'])) + self.assertNotEqual( + request_fingerprint(r1), + request_fingerprint(r2, include_headers=['Accept-Language'])) self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']), request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language'])) diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index bb211dc60..b66588efb 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -20,7 +20,7 @@ class SendCatchLogTest(unittest.TestCase): dispatcher.connect(self.error_handler, signal=test_signal) dispatcher.connect(self.ok_handler, signal=test_signal) - with LogCapture() as l: + with LogCapture() as log: result = yield defer.maybeDeferred( self._get_result, test_signal, arg='test', handlers_called=handlers_called @@ -28,8 +28,8 @@ class SendCatchLogTest(unittest.TestCase): assert self.error_handler in handlers_called assert self.ok_handler in handlers_called - self.assertEqual(len(l.records), 1) - record = l.records[0] + self.assertEqual(len(log.records), 1) + record = log.records[0] self.assertIn('error_handler', record.getMessage()) self.assertEqual(record.levelname, 'ERROR') self.assertEqual(result[0][0], self.error_handler) @@ -44,7 +44,7 @@ class SendCatchLogTest(unittest.TestCase): def error_handler(self, arg, handlers_called): handlers_called.add(self.error_handler) - a = 1 / 0 + 1 / 0 def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) @@ -95,8 +95,8 @@ class SendCatchLogTest2(unittest.TestCase): test_signal = object() dispatcher.connect(test_handler, test_signal) - with LogCapture() as l: + with LogCapture() as log: send_catch_log(test_signal) - self.assertEqual(len(l.records), 1) - self.assertIn("Cannot return deferreds from signal handler", str(l)) + self.assertEqual(len(log.records), 1) + self.assertIn("Cannot return deferreds from signal handler", str(log)) dispatcher.disconnect(test_handler, test_signal) diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index 08b215434..d77978ff1 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -22,8 +22,13 @@ class SitemapTest(unittest.TestCase): """) assert s.type == 'urlset' - self.assertEqual(list(s), - [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}]) + self.assertEqual( + list(s), + [ + {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, + {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}, + ] + ) def test_sitemap_index(self): s = Sitemap(b""" diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 1f8388957..16e7449c9 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import unittest from scrapy.spiders import Spider diff --git a/tests/test_webclient.py b/tests/test_webclient.py index b657c7ab6..188e54602 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -18,6 +18,14 @@ except ImportError: from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory from twisted.internet.defer import inlineCallbacks +from twisted.web.test.test_webclient import ( + ForeverTakingResource, + ErrorResource, + NoLengthResource, + HostHeaderResource, + PayloadResource, + BrokenDownloadResource, +) from scrapy.core.downloader import webclient as client from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory @@ -39,8 +47,9 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): return f from twisted.web.client import _makeGetterFactory - return _makeGetterFactory(to_bytes(url), _clientfactory, - contextFactory=contextFactory, *args, **kwargs).deferred + return _makeGetterFactory( + to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs + ).deferred class ParseUrlTestCase(unittest.TestCase): @@ -97,7 +106,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): 'Content-Length': '12981', 'Useful': 'value'})) - self._test(factory, + self._test( + factory, b"GET /bar HTTP/1.0\r\n" b"Content-Length: 9\r\n" b"Useful: value\r\n" @@ -110,7 +120,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): # test minimal sent headers factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar')) - self._test(factory, + self._test( + factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"\r\n") @@ -122,7 +133,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): body='name=value', headers={'Content-Type': 'application/x-www-form-urlencoded'})) - self._test(factory, + self._test( + factory, b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Connection: close\r\n" @@ -137,7 +149,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): url='http://foo/bar' )) - self._test(factory, + self._test( + factory, b"POST /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"Content-Length: 0\r\n" @@ -152,7 +165,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): }, )) - self._test(factory, + self._test( + factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"X-Meta-Multivalued: value1\r\n" @@ -169,7 +183,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): }), )) - self._test(factory, + self._test( + factory, b"GET /bar HTTP/1.0\r\n" b"Host: foo\r\n" b"X-Meta-Multivalued: value1\r\n" @@ -198,13 +213,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): protocol.dataReceived(b"Hello: World\n") protocol.dataReceived(b"Foo: Bar\n") protocol.dataReceived(b"\n") - self.assertEqual(protocol.headers, - Headers({'Hello': ['World'], 'Foo': ['Bar']})) - - -from twisted.web.test.test_webclient import ForeverTakingResource, \ - ErrorResource, NoLengthResource, HostHeaderResource, \ - PayloadResource, BrokenDownloadResource + self.assertEqual(protocol.headers, Headers({'Hello': ['World'], 'Foo': ['Bar']})) class EncodingResource(resource.Resource): @@ -337,10 +346,11 @@ class WebClientTestCase(unittest.TestCase): return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) def _cbRedirect(self, pageData): - self.assertEqual(pageData, - b'\n\n \n \n' - b' \n \n ' - b'click here\n \n\n') + self.assertEqual( + pageData, + b'\n\n \n \n' + b' \n \n ' + b'click here\n \n\n') def test_encoding(self): """ Test that non-standart body encoding matches @@ -400,8 +410,9 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase): s = "0123456789" * 10 settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': self.custom_ciphers}) client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None) - return getPage(self.getURL("payload"), body=s, - contextFactory=client_context_factory).addCallback(self.assertEqual, to_bytes(s)) + return getPage( + self.getURL("payload"), body=s, contextFactory=client_context_factory + ).addCallback(self.assertEqual, to_bytes(s)) def testPayloadDefaultCiphers(self): s = "0123456789" * 10 diff --git a/tox.ini b/tox.ini index 2102fc602..69b1bdfdd 100644 --- a/tox.ini +++ b/tox.ini @@ -37,6 +37,19 @@ deps = pytest-flake8 commands = py.test --flake8 {posargs:docs scrapy tests} + +[testenv:pylint] +basepython = python3 +deps = + {[testenv]deps} + # Optional dependencies + boto + reppy + robotexclusionrulesparser + # Test dependencies + pylint +commands = + pylint conftest.py docs extras scrapy setup.py tests [testenv:pypy3] basepython = pypy3