diff --git a/conftest.py b/conftest.py index 55294feca..b39d644a5 100644 --- a/conftest.py +++ b/conftest.py @@ -12,7 +12,10 @@ collect_ignore = [ "scrapy/utils/testsite.py", # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), + # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess *_py_files("tests/CrawlerRunner"), + # Py36-only parts of respective tests + *_py_files("tests/py36"), ] for line in open('tests/ignores.txt'): diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 178be723c..49968437c 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -81,35 +81,18 @@ Python packages can be installed either globally (a.k.a system wide), or in user-space. We do not recommend installing Scrapy system wide. Instead, we recommend that you install Scrapy within a so-called -"virtual environment" (`virtualenv`_). -Virtualenvs allow you to not conflict with already-installed Python +"virtual environment" (:mod:`venv`). +Virtual environments allow you to not conflict with already-installed Python system packages (which could break some of your system tools and scripts), and still install packages normally with ``pip`` (without ``sudo`` and the likes). -To get started with virtual environments, see `virtualenv installation instructions`_. -To install it globally (having it globally installed actually helps here), -it should be a matter of running:: +See :ref:`tut-venv` on how to create your virtual environment. - $ [sudo] pip install virtualenv - -Check this `user guide`_ on how to create your virtualenv. - -.. note:: - If you use Linux or OS X, `virtualenvwrapper`_ is a handy tool to create virtualenvs. - -Once you have created a virtualenv, you can install Scrapy inside it with ``pip``, +Once you have created a virtual environment, you can install Scrapy inside it with ``pip``, just like any other Python package. (See :ref:`platform-specific guides ` below for non-Python dependencies that you may need to install beforehand). -Python virtualenvs can be created to use Python 2 by default, or Python 3 by default. As Scrapy -only supports Python 3, make sure you created a Python 3 virtualenv. - -.. _virtualenv: https://virtualenv.pypa.io -.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/ -.. _virtualenvwrapper: https://virtualenvwrapper.readthedocs.io/en/latest/install.html -.. _user guide: https://virtualenv.pypa.io/en/stable/userguide/ - .. _intro-install-platform-notes: @@ -205,15 +188,12 @@ solutions: brew update; brew upgrade python -* *(Optional)* Install Scrapy inside an isolated python environment. +* *(Optional)* :ref:`Install Scrapy inside a Python virtual environment + `. This method is a workaround for the above OS X issue, but it's an overall good practice for managing dependencies and can complement the first method. - `virtualenv`_ is a tool you can use to create virtual environments in python. - We recommended reading a tutorial like - http://docs.python-guide.org/en/latest/dev/virtualenvs/ to get started. - After any of these workarounds you should be able to install Scrapy:: pip install Scrapy diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index ee10048b5..798fe4a7a 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -212,7 +212,7 @@ using the :ref:`Scrapy shell `. Run:: .. note:: Remember to always enclose urls in quotes when running Scrapy shell from - command-line, otherwise urls containing arguments (ie. ``&`` character) + command-line, otherwise urls containing arguments (i.e. ``&`` character) will not work. On Windows, use double quotes instead:: diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 3ec6e0c17..a83cedcfd 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -259,8 +259,8 @@ COOKIES_DEBUG Default: ``False`` -If enabled, Scrapy will log all cookies sent in requests (ie. ``Cookie`` -header) and all cookies received in responses (ie. ``Set-Cookie`` header). +If enabled, Scrapy will log all cookies sent in requests (i.e. ``Cookie`` +header) and all cookies received in responses (i.e. ``Set-Cookie`` header). Here's an example of a log with :setting:`COOKIES_DEBUG` enabled:: diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 0a7455ec9..dc057f6b6 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -63,7 +63,7 @@ but disabled unless the :setting:`HTTPCACHE_ENABLED` setting is set. Disabling an extension ====================== -In order to disable an extension that comes enabled by default (ie. those +In order to disable an extension that comes enabled by default (i.e. those included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to ``None``. For example:: @@ -345,7 +345,7 @@ signal is received. The information dumped is the following: After the stack trace and engine status is dumped, the Scrapy process continues running normally. -This extension only works on POSIX-compliant platforms (ie. not Windows), +This extension only works on POSIX-compliant platforms (i.e. not Windows), because the `SIGQUIT`_ and `SIGUSR2`_ signals are not available on Windows. There are at least two ways to send Scrapy the `SIGQUIT`_ signal: @@ -370,7 +370,7 @@ running normally. For more info see `Debugging in Python`_. -This extension only works on POSIX-compliant platforms (ie. not Windows). +This extension only works on POSIX-compliant platforms (i.e. not Windows). .. _Python debugger: https://docs.python.org/2/library/pdb.html .. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/ diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7481b1a99..1d94807a4 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -301,7 +301,7 @@ FEED_STORE_EMPTY Default: ``False`` -Whether to export empty feeds (ie. feeds with no items). +Whether to export empty feeds (i.e. feeds with no items). .. setting:: FEED_STORAGES diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 8816a028c..c34ba336b 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -22,7 +22,7 @@ Job directory To enable persistence support you just need to define a *job directory* through the ``JOBDIR`` setting. This directory will be for storing all required data to -keep the state of a single job (ie. a spider run). It's important to note that +keep the state of a single job (i.e. a spider run). It's important to note that this directory must not be shared by different spiders, or even different jobs/runs of the same spider, as it's meant to be used for storing the state of a *single* job. diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 2119cb8f8..8c8019438 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -49,7 +49,7 @@ LxmlLinkExtractor :type allow: a regular expression (or list of) :param deny: a single regular expression (or list of regular expressions) - that the (absolute) urls must match in order to be excluded (ie. not + that the (absolute) urls must match in order to be excluded (i.e. not extracted). It has precedence over the ``allow`` parameter. If not given (or empty) it won't exclude any links. :type deny: a regular expression (or list of) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 8f2504a33..b5f5aff83 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -609,7 +609,10 @@ Response objects :param request: the initial value of the :attr:`Response.request` attribute. This represents the :class:`Request` that generated this response. - :type request: :class:`Request` object + :type request: scrapy.http.Request + + :param certificate: an object representing the server's SSL certificate. + :type certificate: twisted.internet.ssl.Certificate :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` @@ -667,7 +670,7 @@ Response objects .. attribute:: Response.meta A shortcut to the :attr:`Request.meta` attribute of the - :attr:`Response.request` object (ie. ``self.request.meta``). + :attr:`Response.request` object (i.e. ``self.request.meta``). Unlike the :attr:`Response.request` attribute, the :attr:`Response.meta` attribute is propagated along redirects and retries, so you will get @@ -675,6 +678,18 @@ Response objects .. seealso:: :attr:`Request.meta` attribute + .. attribute:: Response.cb_kwargs + + A shortcut to the :attr:`Request.cb_kwargs` attribute of the + :attr:`Response.request` object (i.e. ``self.request.cb_kwargs``). + + Unlike the :attr:`Response.request` attribute, the + :attr:`Response.cb_kwargs` attribute is propagated along redirects and + retries, so you will get the original :attr:`Request.cb_kwargs` sent + from your spider. + + .. seealso:: :attr:`Request.cb_kwargs` attribute + .. attribute:: Response.flags A list that contains flags for this response. Flags are labels used for @@ -682,6 +697,13 @@ Response objects they're shown on the string representation of the Response (`__str__` method) which is used by the engine for logging. + .. attribute:: Response.certificate + + A :class:`twisted.internet.ssl.Certificate` object representing + the server's SSL certificate. + + Only populated for ``https`` responses, ``None`` otherwise. + .. attribute:: Response.ip_address The IP address of the server from which the Response originated. @@ -771,7 +793,7 @@ TextResponse objects 1. the encoding passed in the ``__init__`` method ``encoding`` argument 2. the encoding declared in the Content-Type HTTP header. If this - encoding is not valid (ie. unknown), it is ignored and the next + encoding is not valid (i.e. unknown), it is ignored and the next resolution mechanism is tried. 3. the encoding declared in the response body. The TextResponse class diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 8ec758b0e..c3d431e2a 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -986,7 +986,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this:: sel = Selector(html_response) 1. Select all ``

`` elements from an HTML response body, returning a list of - :class:`Selector` objects (ie. a :class:`SelectorList` object):: + :class:`Selector` objects (i.e. a :class:`SelectorList` object):: sel.xpath("//h1") @@ -1013,7 +1013,7 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object:: sel = Selector(xml_response) 1. Select all ```` elements from an XML response body, returning a list - of :class:`Selector` objects (ie. a :class:`SelectorList` object):: + of :class:`Selector` objects (i.e. a :class:`SelectorList` object):: sel.xpath("//product") diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index fa63a5807..5394147da 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -248,7 +248,7 @@ CONCURRENT_REQUESTS Default: ``16`` -The maximum number of concurrent (ie. simultaneous) requests that will be +The maximum number of concurrent (i.e. simultaneous) requests that will be performed by the Scrapy downloader. .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN @@ -258,7 +258,7 @@ CONCURRENT_REQUESTS_PER_DOMAIN Default: ``8`` -The maximum number of concurrent (ie. simultaneous) requests that will be +The maximum number of concurrent (i.e. simultaneous) requests that will be performed to any single domain. See also: :ref:`topics-autothrottle` and its @@ -272,7 +272,7 @@ CONCURRENT_REQUESTS_PER_IP Default: ``0`` -The maximum number of concurrent (ie. simultaneous) requests that will be +The maximum number of concurrent (i.e. simultaneous) requests that will be performed to any single IP. If non-zero, the :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is ignored, and this one is used instead. In other words, concurrency limits will be applied per IP, not diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 886d1b866..d3cfb0307 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -141,7 +141,7 @@ item_error .. signal:: item_error .. function:: item_error(item, response, spider, failure) - Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises + Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises an exception), except :exc:`~scrapy.exceptions.DropItem` exception. This signal supports returning deferreds from their handlers. @@ -232,7 +232,7 @@ spider_error .. signal:: spider_error .. function:: spider_error(failure, response, spider) - Sent when a spider callback generates an error (ie. raises an exception). + Sent when a spider callback generates an error (i.e. raises an exception). This signal does not support returning deferreds from their handlers. diff --git a/pytest.ini b/pytest.ini index 552829d4e..141a13a4f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -37,7 +37,7 @@ flake8-ignore = scrapy/commands/edit.py E501 scrapy/commands/fetch.py E401 E501 E128 E731 scrapy/commands/genspider.py E128 E501 E502 - scrapy/commands/parse.py E128 E501 E731 E226 + scrapy/commands/parse.py E128 E501 E731 scrapy/commands/runspider.py E501 scrapy/commands/settings.py E128 scrapy/commands/shell.py E128 E501 E502 @@ -47,22 +47,22 @@ flake8-ignore = scrapy/contracts/__init__.py E501 W504 scrapy/contracts/default.py E128 # scrapy/core - scrapy/core/engine.py E501 E128 E127 E306 E502 + scrapy/core/engine.py E501 E128 E127 E502 scrapy/core/scheduler.py E501 - scrapy/core/scraper.py E501 E306 E128 W504 - scrapy/core/spidermw.py E501 E731 E126 E226 + scrapy/core/scraper.py E501 E128 W504 + scrapy/core/spidermw.py E501 E731 E126 scrapy/core/downloader/__init__.py E501 scrapy/core/downloader/contextfactory.py E501 E128 E126 scrapy/core/downloader/middleware.py E501 E502 - scrapy/core/downloader/tls.py E501 E305 E241 - scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226 + scrapy/core/downloader/tls.py E501 E241 + scrapy/core/downloader/webclient.py E731 E501 E128 E126 scrapy/core/downloader/handlers/__init__.py E501 - scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127 + scrapy/core/downloader/handlers/ftp.py E501 E128 E127 scrapy/core/downloader/handlers/http10.py E501 scrapy/core/downloader/handlers/http11.py E501 scrapy/core/downloader/handlers/s3.py E501 E128 E126 # scrapy/downloadermiddlewares - scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226 + scrapy/downloadermiddlewares/ajaxcrawl.py E501 scrapy/downloadermiddlewares/decompression.py E501 scrapy/downloadermiddlewares/defaultheaders.py E501 scrapy/downloadermiddlewares/httpcache.py E501 E126 @@ -76,7 +76,7 @@ flake8-ignore = scrapy/extensions/closespider.py E501 E128 E123 scrapy/extensions/corestats.py E501 scrapy/extensions/feedexport.py E128 E501 - scrapy/extensions/httpcache.py E128 E501 E303 + scrapy/extensions/httpcache.py E128 E501 scrapy/extensions/memdebug.py E501 scrapy/extensions/spiderstate.py E501 scrapy/extensions/telnet.py E501 W504 @@ -91,7 +91,7 @@ flake8-ignore = scrapy/http/response/text.py E501 E128 E124 # scrapy/linkextractors scrapy/linkextractors/__init__.py E731 E501 E402 W504 - scrapy/linkextractors/lxmlhtml.py E501 E731 E226 + scrapy/linkextractors/lxmlhtml.py E501 E731 # scrapy/loader scrapy/loader/__init__.py E501 E128 scrapy/loader/processors.py E501 @@ -105,7 +105,7 @@ flake8-ignore = scrapy/selector/unified.py E501 E111 # scrapy/settings scrapy/settings/__init__.py E501 - scrapy/settings/default_settings.py E501 E114 E116 E226 + scrapy/settings/default_settings.py E501 E114 E116 scrapy/settings/deprecated.py E501 # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 @@ -121,28 +121,27 @@ flake8-ignore = scrapy/utils/asyncio.py E501 scrapy/utils/benchserver.py E501 scrapy/utils/conf.py E402 E501 - scrapy/utils/console.py E306 E305 - scrapy/utils/datatypes.py E501 E226 + scrapy/utils/datatypes.py E501 scrapy/utils/decorators.py E501 scrapy/utils/defer.py E501 E128 scrapy/utils/deprecate.py E128 E501 E127 E502 - scrapy/utils/gz.py E305 E501 W504 - scrapy/utils/http.py F403 E226 + scrapy/utils/gz.py E501 W504 + scrapy/utils/http.py F403 scrapy/utils/httpobj.py E501 - scrapy/utils/iterators.py E501 E701 + scrapy/utils/iterators.py E501 scrapy/utils/log.py E128 E501 scrapy/utils/markup.py F403 - scrapy/utils/misc.py E501 E226 + scrapy/utils/misc.py E501 scrapy/utils/multipart.py F403 scrapy/utils/project.py E501 scrapy/utils/python.py E501 - scrapy/utils/reactor.py E226 E501 + scrapy/utils/reactor.py E501 scrapy/utils/reqser.py E501 scrapy/utils/request.py E127 E501 scrapy/utils/response.py E501 E128 scrapy/utils/signal.py E501 E128 scrapy/utils/sitemap.py E501 - scrapy/utils/spider.py E271 E501 + scrapy/utils/spider.py E501 scrapy/utils/ssl.py E501 scrapy/utils/test.py E501 scrapy/utils/url.py E501 F403 E128 F405 @@ -152,7 +151,7 @@ flake8-ignore = scrapy/crawler.py E501 scrapy/dupefilters.py E501 E202 scrapy/exceptions.py E501 - scrapy/exporters.py E501 E226 + scrapy/exporters.py E501 scrapy/interfaces.py E501 scrapy/item.py E501 E128 scrapy/link.py E501 @@ -161,93 +160,91 @@ flake8-ignore = scrapy/middleware.py E128 E501 scrapy/pqueues.py E501 scrapy/resolver.py E501 - scrapy/responsetypes.py E128 E501 E305 + scrapy/responsetypes.py E128 E501 scrapy/robotstxt.py E501 scrapy/shell.py E501 scrapy/signalmanager.py E501 - scrapy/spiderloader.py E225 F841 E501 E126 + scrapy/spiderloader.py F841 E501 E126 scrapy/squeues.py E128 scrapy/statscollectors.py E501 # tests tests/__init__.py E402 E501 tests/mockserver.py E401 E501 E126 E123 - tests/pipelines.py F841 E226 + tests/pipelines.py F841 tests/spiders.py E501 E127 tests/test_closespider.py E501 E127 tests/test_command_fetch.py E501 - tests/test_command_parse.py E501 E128 E303 E226 + tests/test_command_parse.py E501 E128 tests/test_command_shell.py E501 E128 tests/test_commands.py E128 E501 tests/test_contracts.py E501 E128 tests/test_crawl.py E501 E741 E265 - tests/test_crawler.py F841 E306 E501 - tests/test_dependencies.py F841 E501 E305 - tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123 + tests/test_crawler.py F841 E501 + tests/test_dependencies.py F841 E501 + tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 - tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126 + tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126 tests/test_downloadermiddleware_decompression.py E127 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 - tests/test_downloadermiddleware_httpcache.py E501 E305 - tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123 + tests/test_downloadermiddleware_httpcache.py E501 + tests/test_downloadermiddleware_httpcompression.py E501 E126 E123 tests/test_downloadermiddleware_httpproxy.py E501 E128 - tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305 - tests/test_downloadermiddleware_retry.py E501 E128 E251 E303 E126 + tests/test_downloadermiddleware_redirect.py E501 E128 E127 + tests/test_downloadermiddleware_retry.py E501 E128 E126 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E221 E501 E741 E128 E124 + tests/test_dupefilters.py E501 E741 E128 E124 tests/test_engine.py E401 E501 E128 - tests/test_exporters.py E501 E731 E306 E128 E124 + tests/test_exporters.py E501 E731 E128 E124 tests/test_extension_telnet.py F841 tests/test_feedexport.py E501 F841 E241 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123 - tests/test_http_response.py E501 E301 E128 E265 - tests/test_item.py E701 E128 F841 E306 + tests/test_http_response.py E501 E128 E265 + tests/test_item.py E128 F841 tests/test_link.py E501 tests/test_linkextractors.py E501 E128 E124 - tests/test_loader.py E501 E731 E303 E741 E128 E117 E241 + tests/test_loader.py E501 E731 E741 E128 E117 E241 tests/test_logformatter.py E128 E501 E122 - tests/test_mail.py E128 E501 E305 + tests/test_mail.py E128 E501 tests/test_middleware.py E501 E128 - tests/test_pipeline_crawl.py E131 E501 E128 E126 - tests/test_pipeline_files.py E501 E303 E272 E226 - tests/test_pipeline_images.py F841 E501 E303 - tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502 + tests/test_pipeline_crawl.py E501 E128 E126 + tests/test_pipeline_files.py E501 + tests/test_pipeline_images.py F841 E501 + tests/test_pipeline_media.py E501 E741 E731 E128 E502 tests/test_proxy_connect.py E501 E741 tests/test_request_cb_kwargs.py E501 - tests/test_responsetypes.py E501 E305 + tests/test_responsetypes.py E501 tests/test_robotstxt_interface.py E501 E501 tests/test_scheduler.py E501 E126 E123 tests/test_selector.py E501 E127 tests/test_spider.py E501 - tests/test_spidermiddleware.py E501 E226 + tests/test_spidermiddleware.py E501 tests/test_spidermiddleware_httperror.py E128 E501 E127 E121 tests/test_spidermiddleware_offsite.py E501 E128 E111 - tests/test_spidermiddleware_output_chain.py E501 E226 + tests/test_spidermiddleware_output_chain.py E501 tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121 - tests/test_squeues.py E501 E701 E741 + tests/test_squeues.py E501 E741 tests/test_utils_asyncio.py E501 - tests/test_utils_conf.py E501 E303 E128 + tests/test_utils_conf.py E501 E128 tests/test_utils_curl.py E501 - tests/test_utils_datatypes.py E402 E501 E305 - tests/test_utils_defer.py E306 E501 F841 E226 - tests/test_utils_deprecate.py F841 E306 E501 + tests/test_utils_datatypes.py E402 E501 + tests/test_utils_defer.py E501 F841 + tests/test_utils_deprecate.py F841 E501 tests/test_utils_http.py E501 E128 W504 - tests/test_utils_iterators.py E501 E128 E129 E303 E241 - tests/test_utils_log.py E741 E226 - tests/test_utils_python.py E501 E303 E731 E701 E305 + tests/test_utils_iterators.py E501 E128 E129 E241 + tests/test_utils_log.py E741 + tests/test_utils_python.py E501 E731 tests/test_utils_reqser.py E501 E128 - tests/test_utils_request.py E501 E128 E305 + tests/test_utils_request.py E501 E128 tests/test_utils_response.py E501 - tests/test_utils_signal.py E741 F841 E731 E226 + tests/test_utils_signal.py E741 F841 E731 tests/test_utils_sitemap.py E128 E501 E124 - tests/test_utils_spider.py E305 - tests/test_utils_template.py E305 - tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123 - tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126 + tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123 + tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126 tests/test_cmdline/__init__.py E501 tests/test_settings/__init__.py E501 E128 tests/test_spiderloader/__init__.py E128 E501 diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ff6f1d8cd..3ef8ddcb3 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -80,7 +80,7 @@ class Command(ScrapyCommand): else: items = self.items.get(lvl, []) - print("# Scraped Items ", "-"*60) + print("# Scraped Items ", "-" * 60) display.pprint([dict(x) for x in items], colorize=colour) def print_requests(self, lvl=None, colour=True): @@ -92,14 +92,14 @@ class Command(ScrapyCommand): else: requests = self.requests.get(lvl, []) - print("# Requests ", "-"*65) + print("# Requests ", "-" * 65) display.pprint(requests, colorize=colour) def print_results(self, opts): colour = not opts.nocolour if opts.verbose: - for level in range(1, self.max_level+1): + for level in range(1, self.max_level + 1): print('\n>>> DEPTH LEVEL: %s <<<' % level) if not opts.noitems: self.print_items(level, colour) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index b690f439f..e72275021 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -3,12 +3,13 @@ import logging import re import warnings +from contextlib import suppress from io import BytesIO from ipaddress import ip_address from time import time from urllib.parse import urldefrag -from twisted.internet import defer, protocol, reactor +from twisted.internet import defer, protocol, reactor, ssl from twisted.internet.endpoints import TCP4ClientEndpoint from twisted.internet.error import TimeoutError from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI @@ -419,11 +420,12 @@ class ScrapyAgent(object): return d def _cb_bodydone(self, result, request, url): - txresponse, body, flags, ip_address = result + txresponse, body, flags, certificate, ip_address = result status = int(txresponse.code) headers = Headers(txresponse.headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url, body=body) - return respcls(url=url, status=status, headers=headers, body=body, flags=flags, ip_address=ip_address) + return respcls(url=url, status=status, headers=headers, body=body, + flags=flags, certificate=certificate, ip_address=ip_address) @implementer(IBodyProducer) @@ -457,9 +459,14 @@ class _ResponseReader(protocol.Protocol): self._fail_on_dataloss_warned = False self._reached_warnsize = False self._bytes_received = 0 + self._certificate = None self._ip_address = None def connectionMade(self): + if self._certificate is None: + with suppress(AttributeError): + self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate()) + if self._ip_address is None: self._ip_address = ip_address(self.transport._producer.getPeer().host) @@ -494,16 +501,22 @@ class _ResponseReader(protocol.Protocol): body = self._bodybuf.getvalue() if reason.check(ResponseDone): - self._finished.callback((self._txresponse, body, None, self._ip_address)) + self._finished.callback( + (self._txresponse, body, None, self._certificate, self._ip_address) + ) return if reason.check(PotentialDataLoss): - self._finished.callback((self._txresponse, body, ['partial'], self._ip_address)) + self._finished.callback( + (self._txresponse, body, ['partial'], self._certificate, self._ip_address) + ) return if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons): if not self._fail_on_dataloss: - self._finished.callback((self._txresponse, body, ['dataloss'], self._ip_address)) + self._finished.callback( + (self._txresponse, body, ['dataloss'], self._certificate, self._ip_address) + ) return elif not self._fail_on_dataloss_warned: diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 4ed482058..a1c881d5e 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -89,4 +89,5 @@ class ScrapyClientTLSOptions(ClientTLSOptions): 'from host "{}" (exception: {})'.format( self._hostnameASCII, repr(e))) + DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT') diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index fc796e8bb..a71dc5fb3 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -140,7 +140,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): self.headers['Content-Length'] = 0 def _build_response(self, body, request): - request.meta['download_latency'] = self.headers_time-self.start_time + request.meta['download_latency'] = self.headers_time - self.start_time status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 829e69993..6ab8cde6b 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -230,6 +230,7 @@ class ExecutionEngine(object): def _download(self, request, spider): slot = self.slot slot.add_request(request) + def _on_success(response): assert isinstance(response, (Response, Request)) if isinstance(response, Response): diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 975aede0c..e184ed50e 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -119,7 +119,7 @@ class Scheduler(object): if self.dqs is None: return try: - self.dqs.push(request, -request.priority) + self.dqs.push(request) except ValueError as e: # non serializable request if self.logunser: msg = ("Unable to serialize request: %(request)s - reason:" @@ -135,35 +135,29 @@ class Scheduler(object): return True def _mqpush(self, request): - self.mqs.push(request, -request.priority) + self.mqs.push(request) def _dqpop(self): if self.dqs: return self.dqs.pop() - def _newmq(self, priority): - """ Factory for creating memory queues. """ - return self.mqclass() - - def _newdq(self, priority): - """ Factory for creating disk queues. """ - path = join(self.dqdir, 'p%s' % (priority, )) - return self.dqclass(path) - def _mq(self): """ Create a new priority queue instance, with in-memory storage """ - return create_instance(self.pqclass, None, self.crawler, self._newmq, - serialize=False) + return create_instance(self.pqclass, + settings=None, + crawler=self.crawler, + downstream_queue_cls=self.mqclass, + key='') def _dq(self): """ Create a new priority queue instance, with disk storage """ state = self._read_dqs_state(self.dqdir) q = create_instance(self.pqclass, - None, - self.crawler, - self._newdq, - state, - serialize=True) + settings=None, + crawler=self.crawler, + downstream_queue_cls=self.dqclass, + key=self.dqdir, + startprios=state) if q: logger.info("Resuming crawl (%(queuesize)d requests scheduled)", {'queuesize': len(q)}, extra={'spider': self.spider}) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index facbd8b73..41f015017 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -16,7 +16,6 @@ from scrapy import signals from scrapy.http import Request, Response from scrapy.item import BaseItem from scrapy.core.spidermw import SpiderMiddlewareManager -from scrapy.utils.request import referer_str logger = logging.getLogger(__name__) @@ -158,9 +157,9 @@ class Scraper(object): if isinstance(exc, CloseSpider): self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') return - logger.error( - "Spider error processing %(request)s (referer: %(referer)s)", - {'request': request, 'referer': referer_str(request)}, + logkws = self.logformatter.spider_error(_failure, request, response, spider) + logger.log( + *logformatter_adapter(logkws), exc_info=failure_to_exc_info(_failure), extra={'spider': spider} ) @@ -208,16 +207,21 @@ class Scraper(object): """ if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest): if download_failure.frames: - logger.error('Error downloading %(request)s', - {'request': request}, - exc_info=failure_to_exc_info(download_failure), - extra={'spider': spider}) + logkws = self.logformatter.download_error(download_failure, request, spider) + logger.log( + *logformatter_adapter(logkws), + extra={'spider': spider}, + exc_info=failure_to_exc_info(download_failure), + ) else: errmsg = download_failure.getErrorMessage() if errmsg: - logger.error('Error downloading %(request)s: %(errmsg)s', - {'request': request, 'errmsg': errmsg}, - extra={'spider': spider}) + logkws = self.logformatter.download_error( + download_failure, request, spider, errmsg) + logger.log( + *logformatter_adapter(logkws), + extra={'spider': spider}, + ) if spider_failure is not download_failure: return spider_failure @@ -236,7 +240,7 @@ class Scraper(object): signal=signals.item_dropped, item=item, response=response, spider=spider, exception=output.value) else: - logkws = self.logformatter.error(item, ex, response, spider) + logkws = self.logformatter.item_error(item, ex, response, spider) logger.log(*logformatter_adapter(logkws), extra={'spider': spider}, exc_info=failure_to_exc_info(output)) return self.signals.send_catch_log_deferred( diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index dd9b3c376..87d08cab7 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -82,7 +82,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - return process_spider_output(result, method_index+1) + return process_spider_output(result, method_index + 1) elif result is None: continue else: @@ -103,12 +103,12 @@ class SpiderMiddlewareManager(MiddlewareManager): # might fail directly if the output value is not a generator result = method(response=response, result=result, spider=spider) except Exception as ex: - exception_result = process_spider_exception(Failure(ex), method_index+1) + exception_result = process_spider_exception(Failure(ex), method_index + 1) if isinstance(exception_result, Failure): raise return exception_result if _isiterable(result): - result = _evaluate_iterable(result, method_index+1, recovered) + result = _evaluate_iterable(result, method_index + 1, recovered) else: msg = "Middleware {} must return an iterable, got {}" raise _InvalidOutput(msg.format(_fname(method), type(result))) diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 7a140fcad..16b046e99 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -47,7 +47,7 @@ class AjaxCrawlMiddleware(object): return response # scrapy already handles #! links properly - ajax_crawl_request = request.replace(url=request.url+'#!') + ajax_crawl_request = request.replace(url=request.url + '#!') logger.debug("Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s", {'ajax_crawl_request': ajax_crawl_request, 'request': request}, extra={'spider': spider}) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 1a3c9345f..96416f075 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -173,12 +173,12 @@ class XmlItemExporter(BaseItemExporter): if hasattr(serialized_value, 'items'): self._beautify_newline() for subname, value in serialized_value.items(): - self._export_xml_field(subname, value, depth=depth+1) + self._export_xml_field(subname, value, depth=depth + 1) self._beautify_indent(depth=depth) elif is_listlike(serialized_value): self._beautify_newline() for value in serialized_value: - self._export_xml_field('value', value, depth=depth+1) + self._export_xml_field('value', value, depth=depth + 1) self._beautify_indent(depth=depth) elif isinstance(serialized_value, str): self.xg.characters(serialized_value) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index ca5ecc02c..1095cb87e 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -17,15 +17,27 @@ from scrapy.utils.trackref import object_ref class Response(object_ref): - def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, ip_address=None): + def __init__(self, url, status=200, headers=None, body=b'', flags=None, + request=None, certificate=None, ip_address=None): self.headers = Headers(headers or {}) self.status = int(status) self._set_body(body) self._set_url(url) self.request = request self.flags = [] if flags is None else list(flags) + self.certificate = certificate self.ip_address = ip_address + @property + def cb_kwargs(self): + try: + return self.request.cb_kwargs + except AttributeError: + raise AttributeError( + "Response.cb_kwargs not available, this response " + "is not tied to any request" + ) + @property def meta(self): try: @@ -77,7 +89,8 @@ class Response(object_ref): """Create a new Response with the same attributes except for those given new values. """ - for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'ip_address']: + for x in ['url', 'status', 'headers', 'body', + 'request', 'flags', 'certificate', 'ip_address']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) @@ -108,7 +121,7 @@ class Response(object_ref): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, cb_kwargs=None): + dont_filter=False, errback=None, cb_kwargs=None, flags=None): # type: (...) -> Request """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -125,6 +138,7 @@ class Response(object_ref): elif url is None: raise ValueError("url can't be None") url = self.urljoin(url) + return Request( url=url, callback=callback, @@ -138,11 +152,12 @@ class Response(object_ref): dont_filter=dont_filter, errback=errback, cb_kwargs=cb_kwargs, + flags=flags, ) def follow_all(self, urls, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, cb_kwargs=None): + dont_filter=False, errback=None, cb_kwargs=None, flags=None): # type: (...) -> Generator[Request, None, None] """ Return an iterable of :class:`~.Request` instances to follow all links @@ -170,6 +185,7 @@ class Response(object_ref): dont_filter=dont_filter, errback=errback, cb_kwargs=cb_kwargs, + flags=flags, ) for url in urls ) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 09049c157..33a485328 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -121,7 +121,7 @@ class TextResponse(Response): def follow(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding=None, priority=0, - dont_filter=False, errback=None, cb_kwargs=None): + dont_filter=False, errback=None, cb_kwargs=None, flags=None): # type: (...) -> Request """ Return a :class:`~.Request` instance to follow a link ``url``. @@ -157,11 +157,12 @@ class TextResponse(Response): dont_filter=dont_filter, errback=errback, cb_kwargs=cb_kwargs, + flags=flags, ) def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding=None, priority=0, - dont_filter=False, errback=None, cb_kwargs=None, + dont_filter=False, errback=None, cb_kwargs=None, flags=None, css=None, xpath=None): # type: (...) -> Generator[Request, None, None] """ @@ -214,6 +215,7 @@ class TextResponse(Response): dont_filter=dont_filter, errback=errback, cb_kwargs=cb_kwargs, + flags=flags, ) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index fdfa92370..ab82e1915 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -5,11 +5,11 @@ from urllib.parse import urljoin import lxml.etree as etree from w3lib.html import strip_html5_whitespace -from w3lib.url import canonicalize_url +from w3lib.url import canonicalize_url, safe_url_string from scrapy.link import Link from scrapy.utils.misc import arg_to_iter, rel_has_nofollow -from scrapy.utils.python import unique as unique_list, to_unicode +from scrapy.utils.python import unique as unique_list from scrapy.utils.response import get_base_url from scrapy.linkextractors import FilteringLinkExtractor @@ -22,7 +22,7 @@ _collect_string_content = etree.XPath("string()") def _nons(tag): if isinstance(tag, str): - if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE: + if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE: return tag.split('}')[-1] return tag @@ -66,7 +66,7 @@ class LxmlParserLinkExtractor(object): url = self.process_attr(attr_val) if url is None: continue - url = to_unicode(url, encoding=response_encoding) + url = safe_url_string(url, encoding=response_encoding) # to fix relative links after process_value url = urljoin(response_url, url) link = Link(url, _collect_string_content(el) or u'', diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 4e5963e99..194013642 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -5,10 +5,13 @@ from twisted.python.failure import Failure from scrapy.utils.request import referer_str -SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s" -DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s" -CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s" -ERRORMSG = u"'Error processing %(item)s'" +SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s" +DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s" +CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s" +ITEMERRORMSG = "Error processing %(item)s" +SPIDERERRORMSG = "Spider error processing %(request)s (referer: %(referer)s)" +DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s" +DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s" class LogFormatter(object): @@ -93,16 +96,41 @@ class LogFormatter(object): } } - def error(self, item, exception, response, spider): + def item_error(self, item, exception, response, spider): """Logs a message when an item causes an error while it is passing through the item pipeline.""" return { 'level': logging.ERROR, - 'msg': ERRORMSG, + 'msg': ITEMERRORMSG, 'args': { 'item': item, } } + def spider_error(self, failure, request, response, spider): + """Logs an error message from a spider.""" + return { + 'level': logging.ERROR, + 'msg': SPIDERERRORMSG, + 'args': { + 'request': request, + 'referer': referer_str(request), + } + } + + def download_error(self, failure, request, spider, errmsg=None): + """Logs a download error message from a spider (typically coming from the engine).""" + args = {'request': request} + if errmsg: + msg = DOWNLOADERRORMSG_LONG + args['errmsg'] = errmsg + else: + msg = DOWNLOADERRORMSG_SHORT + return { + 'level': logging.ERROR, + 'msg': msg, + 'args': args, + } + @classmethod def from_crawler(cls, crawler): return cls() diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 717ed4d27..1afe58dab 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,11 +1,7 @@ import hashlib import logging -from collections import namedtuple - -from queuelib import PriorityQueue - -from scrapy.utils.reqser import request_to_dict, request_from_dict +from scrapy.utils.misc import create_instance logger = logging.getLogger(__name__) @@ -29,88 +25,89 @@ def _path_safe(text): return '-'.join([pathable_slot, unique_slot]) -class _Priority(namedtuple("_Priority", ["priority", "slot"])): - """ Slot-specific priority. It is a hack - ``(priority, slot)`` tuple - which can be used instead of int priorities in queues: +class ScrapyPriorityQueue: + """A priority queue implemented using multiple internal queues (typically, + FIFO queues). It uses one internal queue for each priority value. The internal + queue must implement the following methods: + + * push(obj) + * pop() + * close() + * __len__() + + ``__init__`` method of ScrapyPriorityQueue receives a downstream_queue_cls + argument, which is a class used to instantiate a new (internal) queue when + a new priority is allocated. + + Only integer priorities should be used. Lower numbers are higher + priorities. + + startprios is a sequence of priorities to start with. If the queue was + previously closed leaving some priority buckets non-empty, those priorities + should be passed in startprios. - * they are ordered in the same way - order is still by priority value, - min(prios) works; - * str(p) representation is guaranteed to be different when slots - are different - this is important because str(p) is used to create - queue files on disk; - * they have readable str(p) representation which is safe - to use as a file name. """ - __slots__ = () - def __str__(self): - return '%s_%s' % (self.priority, _path_safe(str(self.slot))) + @classmethod + def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()): + return cls(crawler, downstream_queue_cls, key, startprios) + def __init__(self, crawler, downstream_queue_cls, key, startprios=()): + self.crawler = crawler + self.downstream_queue_cls = downstream_queue_cls + self.key = key + self.queues = {} + self.curprio = None + self.init_prios(startprios) -class _SlotPriorityQueues(object): - """ Container for multiple priority queues. """ - def __init__(self, pqfactory, slot_startprios=None): - """ - ``pqfactory`` is a factory for creating new PriorityQueues. - It must be a function which accepts a single optional ``startprios`` - argument, with a list of priorities to create queues for. + def init_prios(self, startprios): + if not startprios: + return - ``slot_startprios`` is a ``{slot: startprios}`` dict. - """ - self.pqfactory = pqfactory - self.pqueues = {} # slot -> priority queue - for slot, startprios in (slot_startprios or {}).items(): - self.pqueues[slot] = self.pqfactory(startprios) + for priority in startprios: + self.queues[priority] = self.qfactory(priority) - def pop_slot(self, slot): - """ Pop an object from a priority queue for this slot """ - queue = self.pqueues[slot] - request = queue.pop() - if len(queue) == 0: - del self.pqueues[slot] - return request + self.curprio = min(startprios) - def push_slot(self, slot, obj, priority): - """ Push an object to a priority queue for this slot """ - if slot not in self.pqueues: - self.pqueues[slot] = self.pqfactory() - queue = self.pqueues[slot] - queue.push(obj, priority) + def qfactory(self, key): + return create_instance(self.downstream_queue_cls, + None, + self.crawler, + self.key + '/' + str(key)) + + def priority(self, request): + return -request.priority + + def push(self, request): + priority = self.priority(request) + if priority not in self.queues: + self.queues[priority] = self.qfactory(priority) + q = self.queues[priority] + q.push(request) # this may fail (eg. serialization error) + if self.curprio is None or priority < self.curprio: + self.curprio = priority + + def pop(self): + if self.curprio is None: + return + q = self.queues[self.curprio] + m = q.pop() + if not q: + del self.queues[self.curprio] + q.close() + prios = [p for p, q in self.queues.items() if q] + self.curprio = min(prios) if prios else None + return m def close(self): - active = {slot: queue.close() - for slot, queue in self.pqueues.items()} - self.pqueues.clear() + active = [] + for p, q in self.queues.items(): + active.append(p) + q.close() return active def __len__(self): - return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 - - -class ScrapyPriorityQueue(PriorityQueue): - """ - PriorityQueue which works with scrapy.Request instances and - can optionally convert them to/from dicts before/after putting to a queue. - """ - def __init__(self, crawler, qfactory, startprios=(), serialize=False): - super(ScrapyPriorityQueue, self).__init__(qfactory, startprios) - self.serialize = serialize - self.spider = crawler.spider - - @classmethod - def from_crawler(cls, crawler, qfactory, startprios=(), serialize=False): - return cls(crawler, qfactory, startprios, serialize) - - def push(self, request, priority=0): - if self.serialize: - request = request_to_dict(request, self.spider) - super(ScrapyPriorityQueue, self).push(request, priority) - - def pop(self): - request = super(ScrapyPriorityQueue, self).pop() - if request and self.serialize: - request = request_from_dict(request, self.spider) - return request + return sum(len(x) for x in self.queues.values()) if self.queues else 0 class DownloaderInterface(object): @@ -133,16 +130,16 @@ class DownloaderInterface(object): class DownloaderAwarePriorityQueue(object): - """ PriorityQueue which takes Downlaoder activity in account: + """ PriorityQueue which takes Downloader activity in account: domains (slots) with the least amount of active downloads are dequeued first. """ @classmethod - def from_crawler(cls, crawler, qfactory, slot_startprios=None, serialize=False): - return cls(crawler, qfactory, slot_startprios, serialize) + def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()): + return cls(crawler, downstream_queue_cls, key, startprios) - def __init__(self, crawler, qfactory, slot_startprios=None, serialize=False): + def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()): if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0: raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP' % (self.__class__,)) @@ -156,35 +153,49 @@ class DownloaderAwarePriorityQueue(object): "queue class can be resumed." % slot_startprios.__class__) - slot_startprios = { - slot: [_Priority(p, slot) for p in startprios] - for slot, startprios in (slot_startprios or {}).items()} - - def pqfactory(startprios=()): - return ScrapyPriorityQueue(crawler, qfactory, startprios, serialize) - self._slot_pqueues = _SlotPriorityQueues(pqfactory, slot_startprios) - self.serialize = serialize self._downloader_interface = DownloaderInterface(crawler) + self.downstream_queue_cls = downstream_queue_cls + self.key = key + self.crawler = crawler + + self.pqueues = {} # slot -> priority queue + for slot, startprios in (slot_startprios or {}).items(): + self.pqueues[slot] = self.pqfactory(slot, startprios) + + def pqfactory(self, slot, startprios=()): + return ScrapyPriorityQueue(self.crawler, + self.downstream_queue_cls, + self.key + '/' + _path_safe(slot), + startprios) def pop(self): - stats = self._downloader_interface.stats(self._slot_pqueues.pqueues) + stats = self._downloader_interface.stats(self.pqueues) if not stats: return slot = min(stats)[1] - request = self._slot_pqueues.pop_slot(slot) + queue = self.pqueues[slot] + request = queue.pop() + if len(queue) == 0: + del self.pqueues[slot] return request - def push(self, request, priority): + def push(self, request): slot = self._downloader_interface.get_slot_key(request) - priority_slot = _Priority(priority=priority, slot=slot) - self._slot_pqueues.push_slot(slot, request, priority_slot) + if slot not in self.pqueues: + self.pqueues[slot] = self.pqfactory(slot) + queue = self.pqueues[slot] + queue.push(request) def close(self): - active = self._slot_pqueues.close() - return {slot: [p.priority for p in startprios] - for slot, startprios in active.items()} + active = {slot: queue.close() + for slot, queue in self.pqueues.items()} + self.pqueues.clear() + return active def __len__(self): - return len(self._slot_pqueues) + return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 + + def __contains__(self, slot): + return slot in self.pqueues diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 91d309147..64bf93e86 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -116,4 +116,5 @@ class ResponseTypes(object): cls = self.from_body(body) return cls + responsetypes = ResponseTypes() diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index fc7b62e78..f8a0457ce 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -75,8 +75,8 @@ DOWNLOAD_HANDLERS_BASE = { DOWNLOAD_TIMEOUT = 180 # 3mins -DOWNLOAD_MAXSIZE = 1024*1024*1024 # 1024m -DOWNLOAD_WARNSIZE = 32*1024*1024 # 32m +DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m +DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m DOWNLOAD_FAIL_ON_DATALOSS = True diff --git a/scrapy/shell.py b/scrapy/shell.py index a23b04df9..a5e140484 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -5,14 +5,13 @@ See documentation in docs/topics/shell.rst """ import os import signal -import warnings from twisted.internet import threads, defer from twisted.python import threadable from w3lib.url import any_to_uri from scrapy.crawler import Crawler -from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning +from scrapy.exceptions import IgnoreRequest from scrapy.http import Request, Response from scrapy.item import BaseItem from scrapy.settings import Settings @@ -126,7 +125,6 @@ class Shell(object): self.vars['spider'] = spider self.vars['request'] = request self.vars['response'] = response - self.vars['sel'] = _SelectorProxy(response) if self.inthread: self.vars['fetch'] = self.fetch self.vars['view'] = open_in_browser @@ -173,7 +171,7 @@ def _request_deferred(request): This returns a Deferred whose first pair of callbacks are the request callback and errback. The Deferred also triggers when the request - callback/errback is executed (ie. when the request is downloaded) + callback/errback is executed (i.e. when the request is downloaded) WARNING: Do not call request.replace() until after the deferred is called. """ @@ -192,15 +190,3 @@ def _request_deferred(request): request.callback, request.errback = d.callback, d.errback return d - - -class _SelectorProxy(object): - - def __init__(self, response): - self._proxiedresponse = response - - def __getattr__(self, name): - warnings.warn('"sel" shortcut is deprecated. Use "response.xpath()", ' - '"response.css()" or "response.selector" instead', - category=ScrapyDeprecationWarning, stacklevel=2) - return getattr(self._proxiedresponse.selector, name) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 3beca4060..048e84e4f 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -28,7 +28,7 @@ class SpiderLoader(object): module=mod, cls=cls, name=name) for (mod, cls) in locations) for name, locations in self._found.items() - if len(locations)>1] + if len(locations) > 1] if dupes: msg = ("There are several spiders with the same name:\n\n" "{}\n\n This can cause unexpected behavior.".format( diff --git a/scrapy/squeues.py b/scrapy/squeues.py index d5d3be67e..d0686dac3 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -3,10 +3,27 @@ Scheduler queues """ import marshal +import os import pickle from queuelib import queue +from scrapy.utils.reqser import request_to_dict, request_from_dict + + +def _with_mkdir(queue_class): + + class DirectoriesCreated(queue_class): + + def __init__(self, path, *args, **kwargs): + dirname = os.path.dirname(path) + if not os.path.exists(dirname): + os.makedirs(dirname, exist_ok=True) + + super(DirectoriesCreated, self).__init__(path, *args, **kwargs) + + return DirectoriesCreated + def _serializable_queue(queue_class, serialize, deserialize): @@ -24,6 +41,44 @@ def _serializable_queue(queue_class, serialize, deserialize): return SerializableQueue +def _scrapy_serialization_queue(queue_class): + + class ScrapyRequestQueue(queue_class): + + def __init__(self, crawler, key): + self.spider = crawler.spider + super(ScrapyRequestQueue, self).__init__(key) + + @classmethod + def from_crawler(cls, crawler, key, *args, **kwargs): + return cls(crawler, key) + + def push(self, request): + request = request_to_dict(request, self.spider) + return super(ScrapyRequestQueue, self).push(request) + + def pop(self): + request = super(ScrapyRequestQueue, self).pop() + + if not request: + return None + + request = request_from_dict(request, self.spider) + return request + + return ScrapyRequestQueue + + +def _scrapy_non_serialization_queue(queue_class): + + class ScrapyRequestQueue(queue_class): + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + return cls() + + return ScrapyRequestQueue + + def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) @@ -34,13 +89,38 @@ def _pickle_serialize(obj): raise ValueError(str(e)) -PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, - _pickle_serialize, pickle.loads) -PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, - _pickle_serialize, pickle.loads) -MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, - marshal.dumps, marshal.loads) -MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, - marshal.dumps, marshal.loads) -FifoMemoryQueue = queue.FifoMemoryQueue -LifoMemoryQueue = queue.LifoMemoryQueue +PickleFifoDiskQueueNonRequest = _serializable_queue( + _with_mkdir(queue.FifoDiskQueue), + _pickle_serialize, + pickle.loads +) +PickleLifoDiskQueueNonRequest = _serializable_queue( + _with_mkdir(queue.LifoDiskQueue), + _pickle_serialize, + pickle.loads +) +MarshalFifoDiskQueueNonRequest = _serializable_queue( + _with_mkdir(queue.FifoDiskQueue), + marshal.dumps, + marshal.loads +) +MarshalLifoDiskQueueNonRequest = _serializable_queue( + _with_mkdir(queue.LifoDiskQueue), + marshal.dumps, + marshal.loads +) + +PickleFifoDiskQueue = _scrapy_serialization_queue( + PickleFifoDiskQueueNonRequest +) +PickleLifoDiskQueue = _scrapy_serialization_queue( + PickleLifoDiskQueueNonRequest +) +MarshalFifoDiskQueue = _scrapy_serialization_queue( + MarshalFifoDiskQueueNonRequest +) +MarshalLifoDiskQueue = _scrapy_serialization_queue( + MarshalLifoDiskQueueNonRequest +) +FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) +LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 7eb40f0ce..c7a2ace88 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -54,6 +54,7 @@ def _embed_standard_shell(namespace={}, banner=''): else: import rlcompleter # noqa: F401 readline.parse_and_bind("tab:complete") + @wraps(_embed_standard_shell) def wrapper(namespace=namespace, banner=''): code.interact(banner=banner, local=namespace) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index a52bbc70e..b07f995cf 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -175,12 +175,12 @@ class SiteNode(object): node.parent = self def to_string(self, level=0): - s = "%s%s\n" % (' '*level, self.url) + s = "%s%s\n" % (' ' * level, self.url) if self.itemnames: for n in self.itemnames: - s += "%sScraped: %s\n" % (' '*(level+1), n) + s += "%sScraped: %s\n" % (' ' * (level + 1), n) for node in self.children: - s += node.to_string(level+1) + s += node.to_string(level + 1) return s diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 6a21a4903..34b8d9774 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -153,3 +153,20 @@ def deferred_f_from_coro_f(coro_f): def f(*coro_args, **coro_kwargs): return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) return f + + +def maybeDeferred_coro(f, *args, **kw): + """ Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """ + try: + result = f(*args, **kw) + except: # noqa: E722 + return defer.fail(failure.Failure(captureVars=defer.Deferred.debug)) + + if isinstance(result, defer.Deferred): + return result + elif _isfuture(result) or inspect.isawaitable(result): + return deferred_from_coro(result) + elif isinstance(result, failure.Failure): + return defer.fail(result) + else: + return defer.succeed(result) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 9672e28da..c291ae237 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -42,6 +42,7 @@ def gunzip(data): raise return b''.join(output_list) + _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py index bab262393..ceb3f0509 100644 --- a/scrapy/utils/http.py +++ b/scrapy/utils/http.py @@ -32,5 +32,5 @@ def decode_chunked_transfer(chunked_body): break size = int(h, 16) body += t[:size] - t = t[size+2:] + t = t[size + 2:] return body diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 3c0cb68c3..7849174fb 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -101,8 +101,10 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): lines = StringIO(_body_or_str(obj, unicode=True)) kwargs = {} - if delimiter: kwargs["delimiter"] = delimiter - if quotechar: kwargs["quotechar"] = quotechar + if delimiter: + kwargs["delimiter"] = delimiter + if quotechar: + kwargs["quotechar"] = quotechar csv_r = csv.reader(lines, **kwargs) if not headers: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index cb0ee5af3..52cfba208 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -37,8 +37,8 @@ def arg_to_iter(arg): def load_object(path): """Load an object given its absolute object path, and return it. - object can be a class, function, variable or an instance. - path ie: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware' + object can be the import path of a class, function, variable or an + instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware' """ try: @@ -46,7 +46,7 @@ def load_object(path): except ValueError: raise ValueError("Error loading object '%s': not a full path" % path) - module, name = path[:dot], path[dot+1:] + module, name = path[:dot], path[dot + 1:] mod = import_module(module) try: diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index f28c2eaa1..d9a03ff63 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -68,7 +68,6 @@ def get_project_settings(): if settings_module_path: settings.setmodule(settings_module_path, priority='project') - # XXX: remove this hack pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE") if pickled_settings: warnings.warn("Use of environment variable " @@ -76,10 +75,9 @@ def get_project_settings(): "is deprecated.", ScrapyDeprecationWarning) settings.setdict(pickle.loads(pickled_settings), priority='project') - # XXX: deprecate and remove this functionality env_overrides = {k[7:]: v for k, v in os.environ.items() if k.startswith('SCRAPY_')} if env_overrides: + warnings.warn("Use of 'SCRAPY_'-prefixed environment variables to override settings is deprecated.", ScrapyDeprecationWarning) settings.setdict(env_overrides, priority='project') - return settings diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py new file mode 100644 index 000000000..c8c24076e --- /dev/null +++ b/scrapy/utils/py36.py @@ -0,0 +1,10 @@ +""" +Helpers using Python 3.6+ syntax (ignore SyntaxError on import). +""" + + +async def collect_asyncgen(result): + results = [] + async for x in result: + results.append(x) + return results diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index e5582cc18..e95a4648e 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -4,7 +4,6 @@ This module contains essential stuff that should've come with Python itself ;) import errno import gc import inspect -import os import re import sys import weakref @@ -165,14 +164,6 @@ _BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\ _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS} -@deprecated("scrapy.utils.python.binary_is_text") -def isbinarytext(text): - """ This function is deprecated. - Please use scrapy.utils.python.binary_is_text, which was created to be more - clear about the functions behavior: it is behaving inverted to this one. """ - return not binary_is_text(text) - - def binary_is_text(data): """ Returns ``True`` if the given ``data`` argument (a ``bytes`` object) does not contain unprintable control characters. @@ -293,41 +284,6 @@ class WeakKeyCache(object): return self._weakdict[key] -@deprecated -def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): - """Return a (new) dict with unicode keys (and values when "keys_only" is - False) of the given dict converted to strings. ``dct_or_tuples`` can be a - dict or a list of tuples, like any dict ``__init__`` method supports. - """ - d = {} - for k, v in dict(dct_or_tuples).items(): - k = k.encode(encoding) if isinstance(k, str) else k - if not keys_only: - v = v.encode(encoding) if isinstance(v, str) else v - d[k] = v - return d - - -@deprecated -def is_writable(path): - """Return True if the given path can be written (if it exists) or created - (if it doesn't exist) - """ - if os.path.exists(path): - return os.access(path, os.W_OK) - else: - return os.access(os.path.dirname(path), os.W_OK) - - -@deprecated -def setattr_default(obj, name, value): - """Set attribute value, but only if it's not already set. Similar to - setdefault() for dicts. - """ - if not hasattr(obj, name): - setattr(obj, name, value) - - def retry_on_eintr(function, *args, **kw): """Run a function and retry it while getting EINTR errors""" while True: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 80f52a4ef..6513e06c9 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -16,7 +16,7 @@ def listen_tcp(portrange, host, factory): return reactor.listenTCP(portrange, factory, interface=host) if len(portrange) == 1: return reactor.listenTCP(portrange[0], factory, interface=host) - for x in range(portrange[0], portrange[1]+1): + for x in range(portrange[0], portrange[1] + 1): try: return reactor.listenTCP(x, factory, interface=host) except error.CannotListenError: diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 356753ab5..b8c140a7e 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -28,7 +28,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False): http://www.example.com/query?cat=222&id=111 Even though those are two different URLs both point to the same resource - and are equivalent (ie. they should return the same response). + and are equivalent (i.e. they should return the same response). Another example are cookies used to store session ids. Suppose the following page is only accessible to authenticated users: diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index de00bac49..60c561da6 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -2,12 +2,14 @@ import logging -from twisted.internet.defer import maybeDeferred, DeferredList, Deferred +from twisted.internet.defer import DeferredList, Deferred from twisted.python.failure import Failure from pydispatch.dispatcher import Any, Anonymous, liveReceivers, \ getAllReceivers, disconnect from pydispatch.robustapply import robustApply + +from scrapy.utils.defer import maybeDeferred_coro from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) @@ -61,7 +63,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named): spider = named.get('spider', None) dfds = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - d = maybeDeferred(robustApply, receiver, signal=signal, sender=sender, + d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender, *arguments, **named) d.addErrback(logerror, receiver) d.addBoth(lambda result: (receiver, result)) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 72775df5c..1b8a82829 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -4,18 +4,26 @@ import inspect from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter +try: + from scrapy.utils.py36 import collect_asyncgen +except SyntaxError: + collect_asyncgen = None logger = logging.getLogger(__name__) def iterate_spider_output(result): + if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result): + d = deferred_from_coro(collect_asyncgen(result)) + d.addCallback(iterate_spider_output) + return d return arg_to_iter(deferred_from_coro(result)) def iter_spider_classes(module): """Return an iterator over all spider classes defined in the given module - that can be instantiated (ie. which have name) + that can be instantiated (i.e. which have name) """ # this needs to be imported here until get rid of the spider manager # singleton in scrapy.spider.spiders diff --git a/sep/sep-003.rst b/sep/sep-003.rst index 184839525..e6357313d 100644 --- a/sep/sep-003.rst +++ b/sep/sep-003.rst @@ -18,7 +18,7 @@ Prerequisites This API proposal relies on the following API: -1. instantiating a item with an item instance as its first argument (ie. +1. instantiating a item with an item instance as its first argument (i.e. ``item2 = MyItem(item1)``) must return a **copy** of the first item instance) 2. items can be instantiated using this syntax: ``item = Item(attr1=value1, @@ -78,7 +78,7 @@ Defining an item containing ItemField's variants2 = ListField(ItemField(Variant), default=[]) It's important to note here that the (perhaps most intuitive) way of defining a -Product-Variant relationship (ie. defining a recursive !ItemField) doesn't +Product-Variant relationship (i.e. defining a recursive !ItemField) doesn't work. For example, this fails to compile: :: diff --git a/sep/sep-013.rst b/sep/sep-013.rst index 5b18b7501..4bc9abd30 100644 --- a/sep/sep-013.rst +++ b/sep/sep-013.rst @@ -59,7 +59,7 @@ Global changes to all middlewares To be discussed: -1. should we support returning deferreds (ie. ``maybeDeferred``) in middleware +1. should we support returning deferreds (i.e. ``maybeDeferred``) in middleware methods? 2. should we pass Twisted Failures instead of exceptions to error methods? diff --git a/sep/sep-021.rst b/sep/sep-021.rst index 628a95dd2..372429791 100644 --- a/sep/sep-021.rst +++ b/sep/sep-021.rst @@ -38,7 +38,7 @@ Goals: * simple to manage: adding or removing extensions should be just a matter of adding or removing lines in a ``scrapy.cfg`` file -* backward compatibility with enabling extension the "old way" (ie. modifying +* backward compatibility with enabling extension the "old way" (i.e. modifying settings directly) Non-goals: diff --git a/tests/pipelines.py b/tests/pipelines.py index d7d3b5259..de4894c32 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -6,7 +6,7 @@ Some pipelines used for testing class ZeroDivisionErrorPipeline(object): def open_spider(self, spider): - a = 1/0 + a = 1 / 0 def process_item(self, item, spider): return item @@ -15,4 +15,4 @@ class ZeroDivisionErrorPipeline(object): class ProcessWithZeroDivisionErrorPipiline(object): def process_item(self, item, spider): - 1/0 + 1 / 0 diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py new file mode 100644 index 000000000..162a53760 --- /dev/null +++ b/tests/py36/_test_crawl.py @@ -0,0 +1,57 @@ +import asyncio + +from scrapy import Request +from tests.spiders import SimpleSpider + + +class AsyncDefAsyncioGenSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen' + + async def parse(self, response): + await asyncio.sleep(0.2) + yield {'foo': 42} + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioGenLoopSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen_loop' + + async def parse(self, response): + for i in range(10): + await asyncio.sleep(0.1) + yield {'foo': i} + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioGenComplexSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen_complex' + initial_reqs = 4 + following_reqs = 3 + depth = 2 + + def _get_req(self, index, cb=None): + return Request(self.mockserver.url("/status?n=200&request=%d" % index), + meta={'index': index}, + dont_filter=True, + callback=cb) + + def start_requests(self): + for i in range(1, self.initial_reqs + 1): + yield self._get_req(i) + + async def parse(self, response): + index = response.meta['index'] + yield {'index': index} + if index < 10 ** self.depth: + for new_index in range(10 * index, 10 * index + self.following_reqs): + yield self._get_req(new_index) + yield self._get_req(index, cb=self.parse2) + await asyncio.sleep(0.1) + yield {'index': index + 5} + + async def parse2(self, response): + await asyncio.sleep(0.1) + yield {'index2': response.meta['index']} diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index b7035fdff..5bf92b71a 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -147,7 +147,6 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} self.url('/html')]) self.assertIn("DEBUG: It Works!", _textmode(stderr)) - @defer.inlineCallbacks def test_pipelines(self): _, _, stderr = yield self.execute(['--spider', self.spider_name, @@ -183,7 +182,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} def test_crawlspider_matching_rule_callback_set(self): """If a rule matches the URL, use it's defined callback.""" status, out, stderr = yield self.execute( - ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')] + ['--spider', 'goodcrawl' + self.spider_name, '-r', self.url('/html')] ) self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out)) @@ -191,7 +190,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} def test_crawlspider_matching_rule_default_callback(self): """If a rule match but it has no callback set, use the 'parse' callback.""" status, out, stderr = yield self.execute( - ['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')] + ['--spider', 'goodcrawl' + self.spider_name, '-r', self.url('/text')] ) self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out)) @@ -207,7 +206,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} @defer.inlineCallbacks def test_crawlspider_missing_callback(self): status, out, stderr = yield self.execute( - ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')] + ['--spider', 'badcrawl' + self.spider_name, '-r', self.url('/html')] ) self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") @@ -215,7 +214,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} def test_crawlspider_no_matching_rule(self): """The requested URL has no matching rule, so no items should be scraped""" status, out, stderr = yield self.execute( - ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')] + ['--spider', 'badcrawl' + self.spider_name, '-r', self.url('/enc-gb18030')] ) self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3116eb88a..3a9b00ab3 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,5 +1,6 @@ import json import logging +import sys from ipaddress import IPv4Address from socket import gethostbyname from urllib.parse import urlparse @@ -7,6 +8,7 @@ from urllib.parse import urlparse from pytest import mark from testfixtures import LogCapture from twisted.internet import defer +from twisted.internet.ssl import Certificate from twisted.trial.unittest import TestCase from scrapy import signals @@ -348,6 +350,59 @@ with multiples lines self.assertIn({'id': 1}, items) self.assertIn({'id': 2}, items) + @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_asyncgen_parse(self): + from tests.py36._test_crawl import AsyncDefAsyncioGenSpider + crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + self.assertIn("Got response 200", str(log)) + itemcount = crawler.stats.get_value('item_scraped_count') + self.assertEqual(itemcount, 1) + + @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_asyncgen_parse_loop(self): + items = [] + + def _on_item_scraped(item): + items.append(item) + + from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider + crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + with LogCapture() as log: + yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + self.assertIn("Got response 200", str(log)) + itemcount = crawler.stats.get_value('item_scraped_count') + self.assertEqual(itemcount, 10) + for i in range(10): + self.assertIn({'foo': i}, items) + + @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") + @mark.only_asyncio() + @defer.inlineCallbacks + def test_async_def_asyncgen_parse_complex(self): + items = [] + + def _on_item_scraped(item): + items.append(item) + + from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider + crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + yield crawler.crawl(mockserver=self.mockserver) + itemcount = crawler.stats.get_value('item_scraped_count') + self.assertEqual(itemcount, 156) + # some random items + for i in [1, 4, 21, 22, 207, 311]: + self.assertIn({'index': i}, items) + for i in [10, 30, 122]: + self.assertIn({'index2': i}, items) + @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncio_parse_reqs_list(self): @@ -357,6 +412,34 @@ with multiples lines for req_id in range(3): self.assertIn("Got response 200, req_id %d" % req_id, str(log)) + @defer.inlineCallbacks + def test_response_ssl_certificate_none(self): + crawler = self.runner.create_crawler(SingleRequestSpider) + url = self.mockserver.url("/echo?body=test", is_secure=False) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + self.assertIsNone(crawler.spider.meta['responses'][0].certificate) + + @defer.inlineCallbacks + def test_response_ssl_certificate(self): + crawler = self.runner.create_crawler(SingleRequestSpider) + url = self.mockserver.url("/echo?body=test", is_secure=True) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + cert = crawler.spider.meta['responses'][0].certificate + self.assertIsInstance(cert, Certificate) + self.assertEqual(cert.getSubject().commonName, b"localhost") + self.assertEqual(cert.getIssuer().commonName, b"localhost") + + @mark.xfail(reason="Responses with no body return early and contain no certificate") + @defer.inlineCallbacks + def test_response_ssl_certificate_empty_response(self): + crawler = self.runner.create_crawler(SingleRequestSpider) + url = self.mockserver.url("/status?n=200", is_secure=True) + yield crawler.crawl(seed=url, mockserver=self.mockserver) + cert = crawler.spider.meta['responses'][0].certificate + self.assertIsInstance(cert, Certificate) + self.assertEqual(cert.getSubject().commonName, b"localhost") + self.assertEqual(cert.getIssuer().commonName, b"localhost") + @defer.inlineCallbacks def test_dns_server_ip_address(self): crawler = self.runner.create_crawler(SingleRequestSpider) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index e31ccd9b5..a169acbe6 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -13,5 +13,6 @@ class ScrapyUtilsTest(unittest.TestCase): installed_version = [int(x) for x in module.__version__.split('.')[:2]] assert installed_version >= [0, 6], "OpenSSL >= 0.6 required" + if __name__ == "__main__": unittest.main() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 8d95d7cac..29d06bab4 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -348,7 +348,7 @@ class HttpTestCase(unittest.TestCase): return self.download_request(request, Spider('foo')).addCallback(_test) def test_payload(self): - body = b'1'*100 # PayloadResource requires body length to be 100 + body = b'1' * 100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) @@ -812,7 +812,7 @@ class S3TestCase(unittest.TestCase): def test_request_signing1(self): # gets an object from the johnsmith bucket. - date ='Tue, 27 Mar 2007 19:36:42 +0000' + date = 'Tue, 27 Mar 2007 19:36:42 +0000' req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date}) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 04884fb78..051f66680 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -145,7 +145,6 @@ class CookiesMiddlewareTest(TestCase): {'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'}, {'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}] - req = Request('http://scrapytest.org/', cookies=cookies) self.mw.process_request(req, self.spider) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 9401dd66d..9b77c97a8 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -501,5 +501,6 @@ class RFC2616PolicyTest(DefaultStorageTest): self.assertEqualResponse(res1, res2) assert 'cached' in res2.flags + if __name__ == '__main__': unittest.main() diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 64488841a..106ca3360 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -245,7 +245,7 @@ class HttpCompressionTest(TestCase): response.headers['Content-Type'] = 'application/gzip' request = response.request request.method = 'HEAD' - response = response.replace(body = None) + response = response.replace(body=None) newresponse = self.mw.process_response(request, response, self.spider) self.assertIs(newresponse, response) self.assertEqual(response.body, b'') diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index e0f145d0e..053e26fc3 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -68,7 +68,6 @@ class RedirectMiddlewareTest(unittest.TestCase): assert isinstance(r, Response) assert r is rsp - def test_redirect_302(self): url = 'http://www.example.com/302' url2 = 'http://www.example.com/redirected2' @@ -122,7 +121,6 @@ class RedirectMiddlewareTest(unittest.TestCase): del rsp.headers['Location'] assert self.mw.process_response(req, rsp, self.spider) is rsp - def test_max_redirect_times(self): self.mw.max_redirect_times = 1 req = Request('http://scrapytest.org/302') @@ -178,6 +176,7 @@ class RedirectMiddlewareTest(unittest.TestCase): def test_request_meta_handling(self): url = 'http://www.example.com/301' url2 = 'http://www.example.com/redirected' + def _test_passthrough(req): rsp = Response(url, headers={'Location': url2}, status=301, request=req) r = self.mw.process_response(req, rsp, self.spider) @@ -316,5 +315,6 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): response = mw.process_response(req, rsp, self.spider) assert isinstance(response, Response) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 88ce9627f..9e24d86dd 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -43,7 +43,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_crawler_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) @@ -51,14 +51,14 @@ class RFPDupeFilterTest(unittest.TestCase): def test_df_from_settings_scheduler(self): settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'} + 'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, 'from_settings') def test_df_direct_scheduler(self): - settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'} + settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertEqual(scheduler.df.method, 'n/a') @@ -162,7 +162,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log(self): with LogCapture() as l: settings = {'DUPEFILTER_DEBUG': False, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(SimpleSpider, settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) spider = SimpleSpider.from_crawler(crawler) @@ -187,7 +187,7 @@ class RFPDupeFilterTest(unittest.TestCase): def test_log_debug(self): with LogCapture() as l: settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(SimpleSpider, settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) spider = SimpleSpider.from_crawler(crawler) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 5d1f5c182..6e2507508 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -312,6 +312,7 @@ class XmlItemExporterTest(BaseItemExporterTest): for child in children] else: return [(elem.tag, [(elem.text, ())])] + def xmlsplit(xmlcontent): doc = lxml.etree.fromstring(xmlcontent) return xmltuple(doc) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 4c1b2afc3..be17dfd6b 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -72,6 +72,22 @@ class BaseResponseTest(unittest.TestCase): r1 = self.response_class("http://www.example.com", body=b"Some body", request=req) assert r1.meta is req.meta + def test_copy_cb_kwargs(self): + req = Request("http://www.example.com") + req.cb_kwargs['foo'] = 'bar' + r1 = self.response_class("http://www.example.com", body=b"Some body", request=req) + assert r1.cb_kwargs is req.cb_kwargs + + def test_unavailable_meta(self): + r1 = self.response_class("http://www.example.com", body=b"Some body") + with self.assertRaisesRegex(AttributeError, r'Response\.meta not available'): + r1.meta + + def test_unavailable_cb_kwargs(self): + r1 = self.response_class("http://www.example.com", body=b"Some body") + with self.assertRaisesRegex(AttributeError, r'Response\.cb_kwargs not available'): + r1.cb_kwargs + def test_copy_inherited_classes(self): """Test Response children copies preserve their class""" @@ -167,6 +183,11 @@ class BaseResponseTest(unittest.TestCase): self._assert_followed_url(Link('http://example.com/foo '), 'http://example.com/foo%20') + def test_follow_flags(self): + res = self.response_class('http://example.com/') + fol = res.follow('http://example.com/', flags=['cached', 'allowed']) + self.assertEqual(fol.flags, ['cached', 'allowed']) + # Response.follow_all def test_follow_all_absolute(self): @@ -232,6 +253,17 @@ class BaseResponseTest(unittest.TestCase): expected = [u.replace(' ', '%20') for u in absolute] self._assert_followed_all_urls(links, expected) + def test_follow_all_flags(self): + re = self.response_class('http://www.example.com/') + urls = [ + 'http://www.example.com/', + 'http://www.example.com/2', + 'http://www.example.com/foo', + ] + fol = re.follow_all(urls, flags=['cached', 'allowed']) + for req in fol: + self.assertEqual(req.flags, ['cached', 'allowed']) + def _assert_followed_url(self, follow_obj, target_url, response=None): if response is None: response = self._links_response() @@ -562,6 +594,22 @@ class TextResponseTest(BaseResponseTest): ) self.assertEqual(req.encoding, 'cp1251') + def test_follow_flags(self): + res = self.response_class('http://example.com/') + fol = res.follow('http://example.com/', flags=['cached', 'allowed']) + self.assertEqual(fol.flags, ['cached', 'allowed']) + + def test_follow_all_flags(self): + re = self.response_class('http://www.example.com/') + urls = [ + 'http://www.example.com/', + 'http://www.example.com/2', + 'http://www.example.com/foo', + ] + fol = re.follow_all(urls, flags=['cached', 'allowed']) + for req in fol: + self.assertEqual(req.flags, ['cached', 'allowed']) + def test_follow_all_css(self): expected = [ 'http://example.com/sample3.html', diff --git a/tests/test_item.py b/tests/test_item.py index 30463a0f5..f70632d57 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -149,13 +149,15 @@ class ItemTest(unittest.TestCase): fields = {'load': Field(default='A')} save = Field(default='A') - class B(A): pass + class B(A): + pass class C(Item): fields = {'load': Field(default='C')} save = Field(default='C') - class D(B, C): pass + class D(B, C): + pass item = D(save='X', load='Y') self.assertEqual(item['save'], 'X') @@ -164,7 +166,8 @@ class ItemTest(unittest.TestCase): 'save': {'default': 'A'}}) # D class inverted - class E(C, B): pass + class E(C, B): + pass self.assertEqual(E(save='X')['save'], 'X') self.assertEqual(E(load='X')['load'], 'X') @@ -177,7 +180,8 @@ class ItemTest(unittest.TestCase): save = Field(default='A') load = Field(default='A') - class B(A): pass + class B(A): + pass class C(A): fields = {'update': Field(default='C')} @@ -206,14 +210,16 @@ class ItemTest(unittest.TestCase): fields = {'load': Field(default='A')} save = Field(default='A') - class B(A): pass + class B(A): + pass class C(object): fields = {'load': Field(default='C')} not_allowed = Field(default='not_allowed') save = Field(default='C') - class D(B, C): pass + class D(B, C): + pass self.assertRaises(KeyError, D, not_allowed='value') self.assertEqual(D(save='X')['save'], 'X') @@ -221,7 +227,8 @@ class ItemTest(unittest.TestCase): 'load': {'default': 'A'}}) # D class inverted - class E(C, B): pass + class E(C, B): + pass self.assertRaises(KeyError, E, not_allowed='value') self.assertEqual(E(save='X')['save'], 'X') @@ -259,6 +266,7 @@ class ItemTest(unittest.TestCase): with catch_warnings(record=True) as warnings: item = Item() self.assertEqual(len(warnings), 0) + class SubclassedItem(Item): pass subclassed_item = SubclassedItem() diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 38fb8fb4a..53968e60e 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -2,8 +2,6 @@ import re import unittest from warnings import catch_warnings -import pytest - from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link @@ -16,7 +14,6 @@ from tests import get_testdata class Base: class LinkExtractorTestCase(unittest.TestCase): extractor_cls = None - escapes_whitespace = False def setUp(self): body = get_testdata('link_extractor', 'linkextractor.html') @@ -30,10 +27,7 @@ class Base: def test_extract_all_links(self): lx = self.extractor_cls() - if self.escapes_whitespace: - page4_url = 'http://example.com/page%204.html' - else: - page4_url = 'http://example.com/page 4.html' + page4_url = 'http://example.com/page%204.html' self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), @@ -214,7 +208,7 @@ class Base: response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15') links = self.extractor_cls(restrict_xpaths='//p').extract_links(response) self.assertEqual(links, - [Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')]) + [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')]) def test_restrict_xpaths_concat_in_handle_data(self): """html entities cause SGMLParser to call handle_data hook twice""" @@ -310,10 +304,7 @@ class Base: def test_attrs(self): lx = self.extractor_cls(attrs="href") - if self.escapes_whitespace: - page4_url = 'http://example.com/page%204.html' - else: - page4_url = 'http://example.com/page 4.html' + page4_url = 'http://example.com/page%204.html' self.assertEqual(lx.extract_links(self.response), [ Link(url='http://example.com/sample1.html', text=u''), @@ -506,7 +497,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False), ]) - @pytest.mark.xfail def test_restrict_xpaths_with_html_entities(self): super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 7d8c6ec7f..bf9fbe5e4 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -2,6 +2,7 @@ import unittest from testfixtures import LogCapture from twisted.internet import defer +from twisted.python.failure import Failure from twisted.trial.unittest import TestCase as TwistedTestCase from scrapy.crawler import CrawlerRunner @@ -62,15 +63,46 @@ class LogFormatterTestCase(unittest.TestCase): assert all(isinstance(x, str) for x in lines) self.assertEqual(lines, [u"Dropped: \u2018", '{}']) - def test_error(self): + def test_item_error(self): # In practice, the complete traceback is shown by passing the # 'exc_info' argument to the logging function item = {'key': 'value'} exception = Exception() response = Response("http://www.example.com") - logkws = self.formatter.error(item, exception, response, self.spider) + logkws = self.formatter.item_error(item, exception, response, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, u"'Error processing {'key': 'value'}'") + self.assertEqual(logline, u"Error processing {'key': 'value'}") + + def test_spider_error(self): + # In practice, the complete traceback is shown by passing the + # 'exc_info' argument to the logging function + failure = Failure(Exception()) + request = Request("http://www.example.com", headers={'Referer': 'http://example.org'}) + response = Response("http://www.example.com", request=request) + logkws = self.formatter.spider_error(failure, request, response, self.spider) + logline = logkws['msg'] % logkws['args'] + self.assertEqual( + logline, + "Spider error processing (referer: http://example.org)" + ) + + def test_download_error_short(self): + # In practice, the complete traceback is shown by passing the + # 'exc_info' argument to the logging function + failure = Failure(Exception()) + request = Request("http://www.example.com") + logkws = self.formatter.download_error(failure, request, self.spider) + logline = logkws['msg'] % logkws['args'] + self.assertEqual(logline, "Error downloading ") + + def test_download_error_long(self): + # In practice, the complete traceback is shown by passing the + # 'exc_info' argument to the logging function + failure = Failure(Exception()) + request = Request("http://www.example.com") + logkws = self.formatter.download_error(failure, request, self.spider, "Some message") + logline = logkws['msg'] % logkws['args'] + self.assertEqual(logline, "Error downloading : Some message") def test_scraped(self): item = CustomItem() diff --git a/tests/test_mail.py b/tests/test_mail.py index ddb0f1e70..f5cb81a8b 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -121,5 +121,6 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(text.get_charset(), Charset('utf-8')) self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index fb72c9d6d..962c33144 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -26,10 +26,9 @@ class MediaDownloadSpider(SimpleSpider): self.media_key: [], self.media_urls_key: [ self._process_url(response.urljoin(href)) - for href in response.xpath(''' - //table[thead/tr/th="Filename"] - /tbody//a/@href - ''').getall()], + for href in response.xpath( + '//table[thead/tr/th="Filename"]/tbody//a/@href' + ).getall()], } yield item @@ -99,8 +98,9 @@ class FileDownloadCrawlTestCase(TestCase): if self.expected_checksums is not None: checksums = set( i['checksum'] - for item in items - for i in item[self.media_key]) + for item in items + for i in item[self.media_key] + ) self.assertEqual(checksums, self.expected_checksums) # check that the image files where actually written to the media store diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e5bad2ed0..f155db4ce 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -272,7 +272,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): prefix = pipeline_cls.__name__.upper() settings = self._generate_fake_settings(prefix=prefix) user_pipeline = pipeline_cls.from_settings(Settings(settings)) - for pipe_cls_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: + for pipe_cls_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map: custom_value = settings.get(prefix + "_" + settings_attr) self.assertNotEqual(custom_value, self.default_cls_settings[pipe_cls_attr]) self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value) @@ -286,7 +286,6 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase): self.assertEqual(pipeline.files_result_field, "this") self.assertEqual(pipeline.files_urls_field, "that") - def test_user_defined_subclass_default_key_names(self): """Test situation when user defines subclass of FilesPipeline, but uses attribute names for default pipeline (without prefixing @@ -360,7 +359,7 @@ class TestGCSFilesStore(unittest.TestCase): self.assertIn('checksum', s) self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==') u = urlparse(uri) - content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path) + content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:] + path) self.assertEqual(content, data) self.assertEqual(blob.metadata, {'foo': 'bar'}) self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 7f1cb4a11..5018d6802 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -177,7 +177,6 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase): IMAGES_RESULT_FIELD='images' ) - def setUp(self): self.tempdir = mkdtemp() diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 1fcc5799e..d369e147d 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -304,6 +304,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): return response rsp1 = Response('http://url') + def rsp1_func(): dfd = Deferred().addCallback(_check_downloading) reactor.callLater(.1, dfd.callback, rsp1) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index d5a3371ab..8cdf7a176 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -90,5 +90,6 @@ class ResponseTypesTest(unittest.TestCase): # check that mime.types files shipped with scrapy are loaded self.assertEqual(responsetypes.mimetypes.guess_type('x.scrapytest')[0], 'x-scrapy/test') + if __name__ == "__main__": unittest.main() diff --git a/tests/test_signals.py b/tests/test_signals.py new file mode 100644 index 000000000..d6ae526be --- /dev/null +++ b/tests/test_signals.py @@ -0,0 +1,44 @@ +from pytest import mark +from twisted.internet import defer +from twisted.trial import unittest + +from scrapy import signals, Request, Spider +from scrapy.utils.test import get_crawler, get_from_asyncio_queue + +from tests.mockserver import MockServer + + +class ItemSpider(Spider): + name = 'itemspider' + + def start_requests(self): + for index in range(10): + yield Request(self.mockserver.url('/status?n=200&id=%d' % index), + meta={'index': index}) + + def parse(self, response): + return {'index': response.meta['index']} + + +class AsyncSignalTestCase(unittest.TestCase): + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self.items = [] + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + async def _on_item_scraped(self, item): + item = await get_from_asyncio_queue(item) + self.items.append(item) + + @mark.only_asyncio() + @defer.inlineCallbacks + def test_simple_pipeline(self): + crawler = get_crawler(ItemSpider) + crawler.signals.connect(self._on_item_scraped, signals.item_scraped) + yield crawler.crawl(mockserver=self.mockserver) + self.assertEqual(len(self.items), 10) + for index in range(10): + self.assertIn({'index': index}, self.items) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 55d665e79..78e926adc 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -94,7 +94,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): class RaiseExceptionProcessSpiderOutputMiddleware: def process_spider_output(self, response, result, spider): - 1/0 + 1 / 0 self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index b19a74609..b26353d6c 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -125,7 +125,7 @@ class NotGeneratorCallbackSpider(Spider): yield Request(self.mockserver.url('/status?n=200')) def parse(self, response): - return [{'test': 1}, {'test': 1/0}] + return [{'test': 1}, {'test': 1 / 0}] # ================================================================================ diff --git a/tests/test_squeues.py b/tests/test_squeues.py index d5fcf2f7f..f0f3dd4c6 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,7 +1,12 @@ import pickle from queuelib.tests import test_queue as t -from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue +from scrapy.squeues import ( + MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue, + MarshalLifoDiskQueueNonRequest as MarshalLifoDiskQueue, + PickleFifoDiskQueueNonRequest as PickleFifoDiskQueue, + PickleLifoDiskQueueNonRequest as PickleLifoDiskQueue +) from scrapy.item import Item, Field from scrapy.http import Request from scrapy.loader import ItemLoader @@ -31,7 +36,9 @@ def nonserializable_object_test(self): self.assertRaises(ValueError, q.push, lambda x: x) else: # Use a different unpickleable object - class A(object): pass + class A(object): + pass + a = A() a.__reduce__ = a.__reduce_ex__ = None self.assertRaises(ValueError, q.push, a) diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 02d8ba51e..61e110845 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -83,7 +83,6 @@ class BuildComponentListTest(unittest.TestCase): self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x) - class UtilsConfTestCase(unittest.TestCase): def test_arglist_to_dict(self): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index dfbe71ae2..a3b6e64f1 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -9,6 +9,7 @@ from scrapy.utils.defer import mustbe_deferred, process_chain, \ class MustbeDeferredTest(unittest.TestCase): def test_success_function(self): steps = [] + def _append(v): steps.append(v) return steps @@ -20,6 +21,7 @@ class MustbeDeferredTest(unittest.TestCase): def test_unfired_deferred(self): steps = [] + def _append(v): steps.append(v) dfd = defer.Deferred() @@ -102,7 +104,7 @@ class IterErrbackTest(unittest.TestCase): def iterbad(): for x in range(10): if x == 5: - a = 1/0 + a = 1 / 0 yield x errors = [] diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 159ef8f25..b3a90d314 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -110,6 +110,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): # ignore subclassing warnings with warnings.catch_warnings(): warnings.simplefilter('ignore', ScrapyDeprecationWarning) + class UserClass(Deprecated): pass @@ -233,6 +234,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): with warnings.catch_warnings(record=True) as w: AlsoDeprecated() + class UserClass(AlsoDeprecated): pass @@ -247,6 +249,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): with mock.patch('inspect.stack', side_effect=IndexError): with warnings.catch_warnings(record=True) as w: DeprecatedName = create_deprecated_class('DeprecatedName', NewName) + class SubClass(DeprecatedName): pass diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 9776dfb2a..33fc4d570 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -387,7 +387,6 @@ class TestHelper(unittest.TestCase): self.assertTrue(type(r1) is type(r2)) self.assertTrue(type(r1) is not type(r3)) - def _assert_type_and_value(self, a, b, obj): self.assertTrue(type(a) is type(b), 'Got {}, expected {} for {!r}'.format(type(a), type(b), obj)) diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 2c23f3616..21100aeb8 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -16,7 +16,7 @@ class FailureToExcInfoTest(unittest.TestCase): def test_failure(self): try: - 0/0 + 0 / 0 except ZeroDivisionError: exc_info = sys.exc_info() failure = Failure() diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index b79e0ac1c..ec5b4c596 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -104,7 +104,6 @@ class BinaryIsTextTest(unittest.TestCase): assert not binary_is_text(b"\x02\xa3") - class UtilsPythonTestCase(unittest.TestCase): def test_equal_attributes(self): @@ -154,7 +153,9 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) def test_weakkeycache(self): - class _Weakme(object): pass + class _Weakme(object): + pass + _values = count() wk = WeakKeyCache(lambda k: next(_values)) k = _Weakme() @@ -215,7 +216,6 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertEqual( get_func_args(operator.itemgetter(2), stripself=True), ['obj']) - def test_without_none_values(self): self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4]) self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4)) @@ -223,5 +223,6 @@ class UtilsPythonTestCase(unittest.TestCase): without_none_values({'one': 1, 'none': None, 'three': 3, 'four': 4}), {'one': 1, 'three': 3, 'four': 4}) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 3e664fc74..45f0f59e4 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -83,5 +83,6 @@ class UtilsRequestTest(unittest.TestCase): request_httprepr(Request("file:///tmp/foo.txt")) request_httprepr(Request("ftp://localhost/tmp/foo.txt")) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 16b7c5c68..9f6da09ed 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -1,3 +1,6 @@ +import asyncio + +from pytest import mark from testfixtures import LogCapture from twisted.trial import unittest from twisted.python.failure import Failure @@ -5,6 +8,7 @@ from twisted.internet import defer, reactor from pydispatch import dispatcher from scrapy.utils.signal import send_catch_log, send_catch_log_deferred +from scrapy.utils.test import get_from_asyncio_queue class SendCatchLogTest(unittest.TestCase): @@ -40,7 +44,7 @@ class SendCatchLogTest(unittest.TestCase): def error_handler(self, arg, handlers_called): handlers_called.add(self.error_handler) - a = 1/0 + a = 1 / 0 def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) @@ -54,7 +58,7 @@ class SendCatchLogDeferredTest(SendCatchLogTest): return send_catch_log_deferred(signal, *a, **kw) -class SendCatchLogDeferredTest2(SendCatchLogTest): +class SendCatchLogDeferredTest2(SendCatchLogDeferredTest): def ok_handler(self, arg, handlers_called): handlers_called.add(self.ok_handler) @@ -63,8 +67,24 @@ class SendCatchLogDeferredTest2(SendCatchLogTest): reactor.callLater(0, d.callback, "OK") return d - def _get_result(self, signal, *a, **kw): - return send_catch_log_deferred(signal, *a, **kw) + +class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): + + async def ok_handler(self, arg, handlers_called): + handlers_called.add(self.ok_handler) + assert arg == 'test' + await defer.succeed(42) + return "OK" + + +@mark.only_asyncio() +class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): + + async def ok_handler(self, arg, handlers_called): + handlers_called.add(self.ok_handler) + assert arg == 'test' + await asyncio.sleep(0.2) + return await get_from_asyncio_queue("OK") class SendCatchLogTest2(unittest.TestCase): diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 40b733233..5a52dd695 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -38,5 +38,6 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase): os.remove(render_path) assert not os.path.exists(render_path) # Failure of test iself + if '__main__' == __name__: unittest.main() diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 21e9a056a..7abff8281 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -30,7 +30,7 @@ class UrlUtilsTest(unittest.TestCase): url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29' self.assertFalse(url_is_from_any_domain(url, ['testdomain.com'])) - self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com'])) + self.assertFalse(url_is_from_any_domain(url + '.testdomain.com', ['testdomain.com'])) def test_url_is_from_spider(self): spider = Spider(name='example.com') @@ -201,7 +201,8 @@ def create_skipped_scheme_t(args): assert url.startswith(args[1]) return do_expected -for k, args in enumerate ([ + +for k, args in enumerate([ ('/index', 'file://'), ('/index.html', 'file://'), ('./index.html', 'file://'), @@ -229,7 +230,7 @@ for k, args in enumerate ([ ], start=1): t_method = create_guess_scheme_t(args) t_method.__name__ = 'test_uri_%03d' % k - setattr (GuessSchemeTest, t_method.__name__, t_method) + setattr(GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation for k, args in enumerate([ @@ -238,7 +239,7 @@ for k, args in enumerate([ ], start=1): t_method = create_skipped_scheme_t(args) t_method.__name__ = 'test_uri_skipped_%03d' % k - setattr (GuessSchemeTest, t_method.__name__, t_method) + setattr(GuessSchemeTest, t_method.__name__, t_method) class StripUrl(unittest.TestCase): diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 746367b41..99a998a46 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -51,22 +51,22 @@ class ParseUrlTestCase(unittest.TestCase): ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')), - ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip+':100', lip, 100, '/?c=v&c2=v2')), - ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip+':100', lip, 100, '/?c=v&c2=v2')), - ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip+':100', lip, 100, '/foo?c=v&c2=v2')), + ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), + ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')), + ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')), ("http://127.0.0.1", ('http', lip, lip, 80, '/')), ("http://127.0.0.1/", ('http', lip, lip, 80, '/')), ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')), ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')), ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')), - ("http://127.0.0.1:12345/foo", ('http', lip+':12345', lip, 12345, '/foo')), + ("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')), ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')), ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')), ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')), ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')), - ("https://127.0.0.1:12345/", ('https', lip+':12345', lip, 12345, '/')), + ("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')), ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')), ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')), @@ -294,6 +294,7 @@ class WebClientTestCase(unittest.TestCase): finished = self.assertFailure( getPage(self.getURL("wait"), timeout=0.000001), defer.TimeoutError) + def cleanup(passthrough): # Clean up the server which is hanging around not doing # anything.