diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 34819f227..06da46ca1 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -38,6 +38,9 @@ jobs: - python-version: pypy3.10 env: TOXENV: pypy3 + - python-version: pypy3.11 + env: + TOXENV: pypy3 # pinned deps - python-version: "3.9.21" @@ -59,7 +62,7 @@ jobs: - python-version: "3.13" env: TOXENV: extra-deps - - python-version: pypy3.10 + - python-version: pypy3.11 env: TOXENV: pypy3-extra-deps - python-version: "3.13" diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 7700ae7bd..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,22 +0,0 @@ -include CODE_OF_CONDUCT.md -include CONTRIBUTING.md -include INSTALL.md -include NEWS -include SECURITY.md - -include scrapy/VERSION -include scrapy/mime.types -include scrapy/py.typed - -include codecov.yml -include conftest.py -include tox.ini - -recursive-include scrapy/templates * -recursive-include docs * -prune docs/build - -recursive-include extras * -recursive-include tests * - -global-exclude __pycache__ *.py[cod] diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 000000000..6ec565e24 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,23 @@ +{% extends "!layout.html" %} + +{# Overriden to include a link to scrapy.org, not just to the docs root #} +{%- block sidebartitle %} + +{# the logo helper function was removed in Sphinx 6 and deprecated since Sphinx 4 #} +{# the master_doc variable was renamed to root_doc in Sphinx 4 (master_doc still exists in later Sphinx versions) #} +{%- set _logo_url = logo_url|default(pathto('_static/' + (logo or ""), 1)) %} +{%- set _root_doc = root_doc|default(master_doc) %} +scrapy.org / docs + +{%- if READTHEDOCS or DEBUG %} + {%- if theme_version_selector or theme_language_selector %} +
+
+
+
+ {%- endif %} +{%- endif %} + +{%- include "searchbox.html" %} + +{%- endblock %} diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 1a84f893c..fd040fdba 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -62,18 +62,103 @@ In addition to native coroutine APIs Scrapy has some APIs that return a :class:`~twisted.internet.defer.Deferred` object or take a user-supplied function that returns a :class:`~twisted.internet.defer.Deferred` object. These APIs are also asynchronous but don't yet support native ``async def`` syntax. -For example: +In the future we plan to add support for the ``async def`` syntax to these APIs +or replace them with other APIs where changing the existing ones is +possible. -- The :meth:`ExecutionEngine.download` method returns a - :class:`~twisted.internet.defer.Deferred` object. -- A custom download handler needs to define a ``download_request()`` method that - returns a :class:`~twisted.internet.defer.Deferred` object. +The following Scrapy methods return :class:`~twisted.internet.defer.Deferred` +objects (this list is not complete as it only includes methods that we think +may be useful for user code): + +- :class:`scrapy.crawler.Crawler`: + + - :meth:`~scrapy.crawler.Crawler.crawl` + + - :meth:`~scrapy.crawler.Crawler.stop` + +- :class:`scrapy.crawler.CrawlerRunner` (also inherited by + :class:`scrapy.crawler.CrawlerProcess`): + + - :meth:`~scrapy.crawler.CrawlerRunner.crawl` + + - :meth:`~scrapy.crawler.CrawlerRunner.stop` + + - :meth:`~scrapy.crawler.CrawlerRunner.join` + +- :class:`scrapy.core.engine.ExecutionEngine`: + + - :meth:`~scrapy.core.engine.ExecutionEngine.download` + +- :class:`scrapy.signalmanager.SignalManager`: + + - :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_deferred` + +- :class:`~scrapy.mail.MailSender` + + - :meth:`~scrapy.mail.MailSender.send` + +The following user-supplied methods can return +:class:`~twisted.internet.defer.Deferred` objects (the methods that can also +return coroutines are listed in :ref:`coroutine-support`): + +- Custom download handlers (see :setting:`DOWNLOAD_HANDLERS`): + + - ``download_request()`` + + - ``close()`` + +- Custom downloader implementations (see :setting:`DOWNLOADER`): + + - ``fetch()`` + +- Custom scheduler implementations (see :setting:`SCHEDULER`): + + - :meth:`~scrapy.core.scheduler.BaseScheduler.open` + + - :meth:`~scrapy.core.scheduler.BaseScheduler.close` + +- Custom dupefilters (see :setting:`DUPEFILTER_CLASS`): + + - ``open()`` + + - ``close()`` + +- Custom feed storages (see :setting:`FEED_STORAGES`): + + - ``store()`` + +- Subclasses of :class:`scrapy.pipelines.media.MediaPipeline`: + + - ``media_to_download()`` + + - ``item_completed()`` + +- Custom storages used by subclasses of + :class:`scrapy.pipelines.files.FilesPipeline`: + + - ``persist_file()`` + + - ``stat_file()`` In most cases you can use these APIs in code that otherwise uses coroutines, by wrapping a :class:`~twisted.internet.defer.Deferred` object into a :class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for more information about this. +For example: + +- The :meth:`ExecutionEngine.download() + ` method returns a + :class:`~twisted.internet.defer.Deferred` object that fires with the + downloaded response. You can use this object directly in Deferred-based + code or convert it into a :class:`~asyncio.Future` object with + :func:`~scrapy.utils.defer.maybe_deferred_to_future`. +- A custom download handler needs to define a ``download_request()`` method + that returns a :class:`~twisted.internet.defer.Deferred` object. You can + write a method that works with Deferreds and returns one directly, or you + can write a coroutine and convert it into a function that returns a + Deferred with :func:`~scrapy.utils.defer.deferred_f_from_coro_f`. + General usage ============= diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7f401f0c7..2184f2d0e 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -539,18 +539,18 @@ as a fallback value if that key is not provided for a specific feed definition: FEED_EXPORT_ENCODING -------------------- -Default: ``None`` +Default: ``"utf-8"`` (:ref:`fallback `: ``None``) The encoding to be used for the feed. -If unset or set to ``None`` (default) it uses UTF-8 for everything except JSON output, -which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons. +If set to ``None``, it uses UTF-8 for everything except JSON output, which uses +safe numeric encoding (``\uXXXX`` sequences) for historic reasons. -Use ``utf-8`` if you want UTF-8 for JSON too. +Use ``"utf-8"`` if you want UTF-8 for JSON too. .. versionchanged:: 2.8 The :command:`startproject` command now sets this setting to - ``utf-8`` in the generated ``settings.py`` file. + ``"utf-8"`` in the generated ``settings.py`` file. .. setting:: FEED_EXPORT_FIELDS diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index fec82f8e3..e68fc4f0b 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -162,8 +162,17 @@ Those command-specific default settings are specified in the 6. Default global settings -------------------------- -The global defaults are located in the ``scrapy.settings.default_settings`` -module and documented in the :ref:`topics-settings-ref` section. +The ``scrapy.settings.default_settings`` module defines global default values +for some :ref:`built-in settings `. + +.. note:: :command:`startproject` generates a ``settings.py`` file that sets + some settings to different values. + + The reference documentation of settings indicates the default value if one + exists. If :command:`startproject` sets a value, that value is documented + as default, and the value from ``scrapy.settings.default_settings`` is + documented as “fallback”. + Compatibility with pickle ========================= @@ -461,7 +470,7 @@ Note that the event loop class must inherit from :class:`asyncio.AbstractEventLo BOT_NAME -------- -Default: ``'scrapybot'`` +Default: ```` (:ref:`fallback `: ``'scrapybot'``) The name of the bot implemented by this Scrapy project (also known as the project name). This name will be used for the logging too. @@ -1564,7 +1573,7 @@ email notifying about it. If zero, no warning will be produced. NEWSPIDER_MODULE ---------------- -Default: ``''`` +Default: ``".spiders"`` (:ref:`fallback `: ``""``) Module where to create new spiders using the :command:`genspider` command. @@ -1623,9 +1632,7 @@ Adjust redirect request priority relative to original request: ROBOTSTXT_OBEY -------------- -Default: ``False`` - -Scope: ``scrapy.downloadermiddlewares.robotstxt`` +Default: ``True`` (:ref:`fallback `: ``False``) If enabled, Scrapy will respect robots.txt policies. For more information see :ref:`topics-dlmw-robots`. @@ -1839,7 +1846,7 @@ the spider. For more info see :ref:`topics-spider-middleware-setting`. SPIDER_MODULES -------------- -Default: ``[]`` +Default: ``[".spiders"]`` (:ref:`fallback `: ``[]``) A list of modules where Scrapy will look for spiders. diff --git a/pyproject.toml b/pyproject.toml index 84bf41a94..e14efdd17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] -requires = ["setuptools >= 61.0"] -build-backend = "setuptools.build_meta" +requires = ["hatchling>=1.27.0"] +build-backend = "hatchling.build" [project] name = "Scrapy" @@ -10,29 +10,28 @@ dependencies = [ "Twisted>=21.7.0", "cryptography>=37.0.0", "cssselect>=0.9.1", + "defusedxml>=0.7.1", + "itemadapter>=0.1.0", "itemloaders>=1.0.1", + "lxml>=4.6.0", + "packaging", "parsel>=1.5.0", + "protego>=0.1.15", "pyOpenSSL>=22.0.0", "queuelib>=1.4.2", "service_identity>=18.1.0", + "tldextract", "w3lib>=1.17.0", "zope.interface>=5.1.0", - "protego>=0.1.15", - "itemadapter>=0.1.0", - "packaging", - "tldextract", - "lxml>=4.6.0", - "defusedxml>=0.7.1", # Platform-specific dependencies 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"', 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"', ] classifiers = [ - "Framework :: Scrapy", "Development Status :: 5 - Production/Stable", "Environment :: Console", + "Framework :: Scrapy", "Intended Audience :: Developers", - "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", @@ -47,6 +46,8 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Application Frameworks", "Topic :: Software Development :: Libraries :: Python Modules", ] +license = "BSD-3-Clause" +license-files = ["LICENSE", "AUTHORS"] readme = "README.rst" requires-python = ">=3.9" authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }] @@ -63,12 +64,26 @@ releasenotes = "https://docs.scrapy.org/en/latest/news.html" [project.scripts] scrapy = "scrapy.cmdline:execute" -[tool.setuptools.packages.find] -where = ["."] -include = ["scrapy", "scrapy.*",] +[tool.hatch.build.targets.sdist] +include = [ + "/docs", + "/extras", + "/scrapy", + "/tests", + "/tests_typing", + "/CODE_OF_CONDUCT.md", + "/CONTRIBUTING.md", + "/INSTALL.md", + "/NEWS", + "/SECURITY.md", + "/codecov.yml", + "/conftest.py", + "/tox.ini", +] -[tool.setuptools.dynamic] -version = {file = "./scrapy/VERSION"} +[tool.hatch.version] +path = "scrapy/VERSION" +pattern = "^(?P.+)$" [tool.mypy] ignore_missing_imports = true diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b7a73700b..653e5e05c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -265,7 +265,7 @@ class ExecutionEngine: self.crawl(result) return None - d = self.scraper.enqueue_scrape(result, request, self.spider) + d = self.scraper.enqueue_scrape(result, request) d.addErrback( lambda f: logger.error( "Error while enqueuing downloader output", @@ -290,14 +290,14 @@ class ExecutionEngine: """Inject the request into the spider <-> downloader pipeline""" if self.spider is None: raise RuntimeError(f"No open spider to crawl: {request}") - self._schedule_request(request, self.spider) + self._schedule_request(request) self.slot.nextcall.schedule() # type: ignore[union-attr] - def _schedule_request(self, request: Request, spider: Spider) -> None: + def _schedule_request(self, request: Request) -> None: request_scheduled_result = self.signals.send_catch_log( signals.request_scheduled, request=request, - spider=spider, + spider=self.spider, dont_log=IgnoreRequest, ) for handler, result in request_scheduled_result: @@ -305,7 +305,7 @@ class ExecutionEngine: return if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr] self.signals.send_catch_log( - signals.request_dropped, request=request, spider=spider + signals.request_dropped, request=request, spider=self.spider ) def download(self, request: Request) -> Deferred[Response]: @@ -438,7 +438,7 @@ class ExecutionEngine: dfd.addBoth(lambda _: self.downloader.close()) dfd.addErrback(log_failure("Downloader close failure")) - dfd.addBoth(lambda _: self.scraper.close_spider(spider)) + dfd.addBoth(lambda _: self.scraper.close_spider()) dfd.addErrback(log_failure("Scraper close failure")) if hasattr(self.slot.scheduler, "close"): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 496adb500..6f69d668e 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -4,6 +4,7 @@ extracts information from them""" from __future__ import annotations import logging +import warnings from collections import deque from collections.abc import AsyncIterable, Iterator from typing import TYPE_CHECKING, Any, TypeVar, Union, cast @@ -13,7 +14,12 @@ from twisted.python.failure import Failure from scrapy import Spider, signals from scrapy.core.spidermw import SpiderMiddlewareManager -from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest +from scrapy.exceptions import ( + CloseSpider, + DropItem, + IgnoreRequest, + ScrapyDeprecationWarning, +) from scrapy.http import Request, Response from scrapy.utils.defer import ( aiter_errback, @@ -110,27 +116,43 @@ class Scraper: self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE")) yield self.itemproc.open_spider(spider) - def close_spider(self, spider: Spider) -> Deferred[Spider]: + def close_spider(self, spider: Spider | None = None) -> Deferred[Spider]: """Close a spider being scraped and release its resources""" + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.close_spider() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self.slot is None: raise RuntimeError("Scraper slot not assigned") self.slot.closing = Deferred() self.slot.closing.addCallback(self.itemproc.close_spider) - self._check_if_closing(spider) + self._check_if_closing() return self.slot.closing def is_idle(self) -> bool: """Return True if there isn't any more spiders to process""" return not self.slot - def _check_if_closing(self, spider: Spider) -> None: + def _check_if_closing(self) -> None: assert self.slot is not None # typing + assert self.crawler.spider if self.slot.closing and self.slot.is_idle(): - self.slot.closing.callback(spider) + assert self.crawler.spider + self.slot.closing.callback(self.crawler.spider) def enqueue_scrape( - self, result: Response | Failure, request: Request, spider: Spider + self, result: Response | Failure, request: Request, spider: Spider | None = None ) -> _HandleOutputDeferred: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.enqueue_scrape() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self.slot is None: raise RuntimeError("Scraper slot not assigned") dfd = self.slot.add_response_request(result, request) @@ -138,8 +160,8 @@ class Scraper: def finish_scraping(_: _T) -> _T: assert self.slot is not None self.slot.finish_response(result, request) - self._check_if_closing(spider) - self._scrape_next(spider) + self._check_if_closing() + self._scrape_next() return _ dfd.addBoth(finish_scraping) @@ -148,20 +170,20 @@ class Scraper: "Scraper bug processing %(request)s", {"request": request}, exc_info=failure_to_exc_info(f), - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) ) - self._scrape_next(spider) + self._scrape_next() return dfd - def _scrape_next(self, spider: Spider) -> None: + def _scrape_next(self) -> None: assert self.slot is not None # typing while self.slot.queue: response, request, deferred = self.slot.next_response_request_deferred() - self._scrape(response, request, spider).chainDeferred(deferred) + self._scrape(response, request).chainDeferred(deferred) def _scrape( - self, result: Response | Failure, request: Request, spider: Spider + self, result: Response | Failure, request: Request ) -> _HandleOutputDeferred: """ Handle the downloaded response or failure through the spider callback/errback @@ -171,40 +193,49 @@ class Scraper: f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}" ) dfd: Deferred[Iterable[Any] | AsyncIterable[Any]] = self._scrape2( - result, request, spider + result, request ) # returns spider's processed output - dfd.addErrback(self.handle_spider_error, request, result, spider) + dfd.addErrback(self.handle_spider_error, request, result) dfd2: _HandleOutputDeferred = dfd.addCallback( - self.handle_spider_output, request, cast(Response, result), spider + self.handle_spider_output, request, cast(Response, result) ) return dfd2 def _scrape2( - self, result: Response | Failure, request: Request, spider: Spider + self, result: Response | Failure, request: Request ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: """ Handle the different cases of request's result been a Response or a Failure """ if isinstance(result, Response): # Deferreds are invariant so Mutable*Chain isn't matched to *Iterable + assert self.crawler.spider return self.spidermw.scrape_response( # type: ignore[return-value] - self.call_spider, result, request, spider + self.call_spider, result, request, self.crawler.spider ) # else result is a Failure - dfd = self.call_spider(result, request, spider) - dfd.addErrback(self._log_download_errors, result, request, spider) + dfd = self.call_spider(result, request) + dfd.addErrback(self._log_download_errors, result, request) return dfd def call_spider( - self, result: Response | Failure, request: Request, spider: Spider + self, result: Response | Failure, request: Request, spider: Spider | None = None ) -> Deferred[Iterable[Any] | AsyncIterable[Any]]: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.call_spider() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + assert self.crawler.spider dfd: Deferred[Any] if isinstance(result, Response): if getattr(result, "request", None) is None: result.request = request assert result.request - callback = result.request.callback or spider._parse - warn_on_generator_with_return_value(spider, callback) + callback = result.request.callback or self.crawler.spider._parse + warn_on_generator_with_return_value(self.crawler.spider, callback) dfd = defer_succeed(result) dfd.addCallbacks( callback=callback, callbackKeywords=result.request.cb_kwargs @@ -214,7 +245,9 @@ class Scraper: result.request = request # type: ignore[attr-defined] dfd = defer_fail(result) if request.errback: - warn_on_generator_with_return_value(spider, request.errback) + warn_on_generator_with_return_value( + self.crawler.spider, request.errback + ) dfd.addErrback(request.errback) dfd2: Deferred[Iterable[Any] | AsyncIterable[Any]] = dfd.addCallback( iterate_spider_output @@ -226,29 +259,44 @@ class Scraper: _failure: Failure, request: Request, response: Response | Failure, - spider: Spider, + spider: Spider | None = None, ) -> None: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.handle_spider_error() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + assert self.crawler.spider exc = _failure.value if isinstance(exc, CloseSpider): assert self.crawler.engine is not None # typing - self.crawler.engine.close_spider(spider, exc.reason or "cancelled") + self.crawler.engine.close_spider( + self.crawler.spider, exc.reason or "cancelled" + ) return - logkws = self.logformatter.spider_error(_failure, request, response, spider) + logkws = self.logformatter.spider_error( + _failure, request, response, self.crawler.spider + ) logger.log( *logformatter_adapter(logkws), exc_info=failure_to_exc_info(_failure), - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) self.signals.send_catch_log( signal=signals.spider_error, failure=_failure, response=response, - spider=spider, + spider=self.crawler.spider, ) assert self.crawler.stats - self.crawler.stats.inc_value("spider_exceptions/count", spider=spider) self.crawler.stats.inc_value( - f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider + "spider_exceptions/count", spider=self.crawler.spider + ) + self.crawler.stats.inc_value( + f"spider_exceptions/{_failure.value.__class__.__name__}", + spider=self.crawler.spider, ) def handle_spider_output( @@ -256,41 +304,40 @@ class Scraper: result: Iterable[_T] | AsyncIterable[_T], request: Request, response: Response, - spider: Spider, + spider: Spider | None = None, ) -> _HandleOutputDeferred: + if spider is not None: + warnings.warn( + "Passing a 'spider' argument to Scraper.handle_spider_output() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if not result: return defer_succeed(None) it: Iterable[_T] | AsyncIterable[_T] dfd: Deferred[_ParallelResult] if isinstance(result, AsyncIterable): - it = aiter_errback( - result, self.handle_spider_error, request, response, spider - ) + it = aiter_errback(result, self.handle_spider_error, request, response) dfd = parallel_async( it, self.concurrent_items, self._process_spidermw_output, - request, response, - spider, ) else: - it = iter_errback( - result, self.handle_spider_error, request, response, spider - ) + it = iter_errback(result, self.handle_spider_error, request, response) dfd = parallel( it, self.concurrent_items, self._process_spidermw_output, - request, response, - spider, ) # returning Deferred[_ParallelResult] instead of Deferred[Union[_ParallelResult, None]] return dfd # type: ignore[return-value] def _process_spidermw_output( - self, output: Any, request: Request, response: Response, spider: Spider + self, output: Any, response: Response ) -> Deferred[Any] | None: """Process each Request/Item (given in the output parameter) returned from the given spider @@ -314,7 +361,7 @@ class Scraper: assert self.crawler.spider is not None # typing self.slot.itemproc_size += 1 dfd = self.itemproc.process_item(item, self.crawler.spider) - dfd.addBoth(self._itemproc_finished, item, response, self.crawler.spider) + dfd.addBoth(self._itemproc_finished, item, response) return dfd def _log_download_errors( @@ -322,76 +369,81 @@ class Scraper: spider_failure: Failure, download_failure: Failure, request: Request, - spider: Spider, ) -> Failure | None: """Log and silence errors that come from the engine (typically download errors that got propagated thru here). spider_failure: the value passed into the errback of self.call_spider() + (likely raised in the request errback) + download_failure: the value passed into _scrape2() from ExecutionEngine._handle_downloader_output() as "result" + (likely raised in the download handler or a downloader middleware) """ if not download_failure.check(IgnoreRequest): - if download_failure.frames: - logkws = self.logformatter.download_error( - download_failure, request, spider - ) - logger.log( - *logformatter_adapter(logkws), - extra={"spider": spider}, - exc_info=failure_to_exc_info(download_failure), - ) - else: - errmsg = download_failure.getErrorMessage() - if errmsg: - logkws = self.logformatter.download_error( - download_failure, request, spider, errmsg - ) - logger.log( - *logformatter_adapter(logkws), - extra={"spider": spider}, - ) - + assert self.crawler.spider + logkws = self.logformatter.download_error( + download_failure, request, self.crawler.spider + ) + logger.log( + *logformatter_adapter(logkws), + extra={"spider": self.crawler.spider}, + exc_info=failure_to_exc_info(download_failure), + ) if spider_failure is not download_failure: + # a request errback raised a different exception, it needs to be handled later return spider_failure return None def _itemproc_finished( - self, output: Any, item: Any, response: Response | None, spider: Spider + self, output: Any, item: Any, response: Response | None ) -> Deferred[Any]: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing + assert self.crawler.spider self.slot.itemproc_size -= 1 if isinstance(output, Failure): ex = output.value if isinstance(ex, DropItem): - logkws = self.logformatter.dropped(item, ex, response, spider) + logkws = self.logformatter.dropped( + item, ex, response, self.crawler.spider + ) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) + logger.log( + *logformatter_adapter(logkws), + extra={"spider": self.crawler.spider}, + ) return self.signals.send_catch_log_deferred( signal=signals.item_dropped, item=item, response=response, - spider=spider, + spider=self.crawler.spider, exception=output.value, ) assert ex - logkws = self.logformatter.item_error(item, ex, response, spider) + logkws = self.logformatter.item_error( + item, ex, response, self.crawler.spider + ) logger.log( *logformatter_adapter(logkws), - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, exc_info=failure_to_exc_info(output), ) return self.signals.send_catch_log_deferred( signal=signals.item_error, item=item, response=response, - spider=spider, + spider=self.crawler.spider, failure=output, ) - logkws = self.logformatter.scraped(output, response, spider) + logkws = self.logformatter.scraped(output, response, self.crawler.spider) if logkws is not None: - logger.log(*logformatter_adapter(logkws), extra={"spider": spider}) + logger.log( + *logformatter_adapter(logkws), extra={"spider": self.crawler.spider} + ) return self.signals.send_catch_log_deferred( - signal=signals.item_scraped, item=output, response=response, spider=spider + signal=signals.item_scraped, + item=output, + response=response, + spider=self.crawler.spider, ) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 85a3b5895..b8b0aec44 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -40,7 +40,7 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") ScrapeFunc = Callable[ - [Union[Response, Failure], Request, Spider], Union[Iterable[_T], AsyncIterable[_T]] + [Union[Response, Failure], Request], Union[Iterable[_T], AsyncIterable[_T]] ] @@ -86,8 +86,8 @@ class SpiderMiddlewareManager(MiddlewareManager): except _InvalidOutput: raise except Exception: - return scrape_func(Failure(), request, spider) - return scrape_func(response, request, spider) + return scrape_func(Failure(), request) + return scrape_func(response, request) def _evaluate_iterable( self, diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py index 45479ea4f..cb8f869e1 100644 --- a/tests/CrawlerProcess/sleeping.py +++ b/tests/CrawlerProcess/sleeping.py @@ -1,3 +1,5 @@ +import sys + from twisted.internet.defer import Deferred import scrapy @@ -14,7 +16,7 @@ class SleepingSpider(scrapy.Spider): from twisted.internet import reactor d = Deferred() - reactor.callLater(int(self.sleep), d.callback, None) + reactor.callLater(int(sys.argv[1]), d.callback, None) await maybe_deferred_to_future(d) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f49deac1f..b7a8a9628 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import logging import unittest @@ -14,7 +16,7 @@ from twisted.trial.unittest import TestCase from scrapy import signals from scrapy.crawler import CrawlerRunner -from scrapy.exceptions import StopDownload +from scrapy.exceptions import CloseSpider, StopDownload from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import to_unicode @@ -186,11 +188,18 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_requests_items(self): + items = [] + + def _on_item_scraped(item): + items.append(item) + with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(StartRequestsItemSpider) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) assert len(log.records) == 0 + assert items == [{"name": "test item"}] @defer.inlineCallbacks def test_start_requests_unsupported_output(self): @@ -199,11 +208,19 @@ class TestCrawl(TestCase): things fail when ItemAdapter is actually used on the corresponding non-item object.""" + items = [] + + def _on_item_scraped(item): + items.append(item) + with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(StartRequestsGoodAndBadOutput) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) assert len(log.records) == 0 + assert len(items) == 3 + assert not any(isinstance(item, Request) for item in items) @defer.inlineCallbacks def test_start_requests_laziness(self): @@ -723,3 +740,100 @@ class TestCrawlSpider(TestCase): assert crawler.spider.meta[ "failure" ].value.response.headers == crawler.spider.meta.get("headers_received") + + @defer.inlineCallbacks + def test_spider_errback(self): + failures = [] + + def eb(failure: Failure) -> Failure: + failures.append(failure) + return failure + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert len(failures) == 1 + assert "HTTP status code is not handled or not allowed" in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_spider_errback_silence(self): + failures = [] + + def eb(failure: Failure) -> None: + failures.append(failure) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert len(failures) == 1 + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_spider_errback_exception(self): + def eb(failure: Failure) -> None: + raise ValueError("foo") + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "Spider error processing" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_downloader_error(self): + failures = [] + + def eb(failure: Failure) -> Failure: + failures.append(failure) + return failure + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert len(failures) == 1 + assert "Error downloading" in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_spider_errback_exception_downloader_error(self): + def eb(failure: Failure) -> None: + raise ValueError("foo") + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "Error downloading" in str(log) + assert "Spider error processing" in str(log) + + @defer.inlineCallbacks + def test_raise_closespider(self): + def cb(response): + raise CloseSpider + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) + assert "Closing spider (cancelled)" in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_raise_closespider_reason(self): + def cb(response): + raise CloseSpider("my_reason") + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) + assert "Closing spider (my_reason)" in str(log) + assert "Spider error processing" not in str(log) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6c465f000..efb346dde 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -890,7 +890,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_shutdown_graceful(self): sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py", "-a", "sleep=3") + args = self.get_script_args("sleeping.py", "3") p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") @@ -904,7 +904,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): from twisted.internet import reactor sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK - args = self.get_script_args("sleeping.py", "-a", "sleep=10") + args = self.get_script_args("sleeping.py", "10") p = PopenSpawn(args, timeout=5) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 8e718ad5b..408160ccb 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -1,17 +1,18 @@ +from __future__ import annotations + import asyncio from gzip import BadGzipFile from unittest import mock import pytest -from twisted.internet import defer -from twisted.internet.defer import Deferred -from twisted.python.failure import Failure +from twisted.internet.defer import Deferred, succeed from twisted.trial.unittest import TestCase from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue @@ -29,38 +30,36 @@ class TestManagerBase(TestCase): def tearDown(self): return self.crawler.engine.close_spider(self.spider) - def _download(self, request, response=None): + async def _download( + self, request: Request, response: Response | None = None + ) -> Response | Request: """Executes downloader mw manager's download method and returns - the result (Request or Response) or raise exception in case of + the result (Request or Response) or raises exception in case of failure. """ if not response: response = Response(request.url) - def download_func(request, spider): - return response + def download_func(request: Request, spider: Spider) -> Deferred[Response]: + return succeed(response) - dfd = self.mwman.download(download_func, request, self.spider) - # catch deferred result and return the value - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - ret = results[0] - if isinstance(ret, Failure): - ret.raiseException() - return ret + return await maybe_deferred_to_future( + self.mwman.download(download_func, request, self.spider) + ) class TestDefaults(TestManagerBase): """Tests default behavior with default settings""" - def test_request_response(self): + @deferred_f_from_coro_f + async def test_request_response(self): req = Request("http://example.com/index.html") resp = Response(req.url, status=200) - ret = self._download(req, resp) + ret = await self._download(req, resp) assert isinstance(ret, Response), "Non-response returned" - def test_3xx_and_invalid_gzipped_body_must_redirect(self): + @deferred_f_from_coro_f + async def test_3xx_and_invalid_gzipped_body_must_redirect(self): """Regression test for a failure when redirecting a compressed request. @@ -85,13 +84,14 @@ class TestDefaults(TestManagerBase): "Location": "http://example.com/login", }, ) - ret = self._download(request=req, response=resp) + ret = await self._download(req, resp) assert isinstance(ret, Request), f"Not redirected: {ret!r}" assert to_bytes(ret.url) == resp.headers["Location"], ( "Not redirected to location header" ) - def test_200_and_invalid_gzipped_body_must_fail(self): + @deferred_f_from_coro_f + async def test_200_and_invalid_gzipped_body_must_fail(self): req = Request("http://example.com") body = b"

You are being redirected

" resp = Response( @@ -106,13 +106,14 @@ class TestDefaults(TestManagerBase): }, ) with pytest.raises(BadGzipFile): - self._download(request=req, response=resp) + await self._download(req, resp) class TestResponseFromProcessRequest(TestManagerBase): """Tests middleware returning a response from process_request.""" - def test_download_func_not_called(self): + @deferred_f_from_coro_f + async def test_download_func_not_called(self): resp = Response("http://example.com/index.html") class ResponseMiddleware: @@ -123,19 +124,17 @@ class TestResponseFromProcessRequest(TestManagerBase): req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called -class TestProcessRequestInvalidOutput(TestManagerBase): - """Invalid return value for process_request method should raise an exception""" - - def test_invalid_process_request(self): +class TestInvalidOutput(TestManagerBase): + @deferred_f_from_coro_f + async def test_invalid_process_request(self): + """Invalid return value for process_request method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessRequestMiddleware: @@ -143,18 +142,12 @@ class TestProcessRequestInvalidOutput(TestManagerBase): return 1 self.mwman._add_middleware(InvalidProcessRequestMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - assert isinstance(results[0], Failure) - assert isinstance(results[0].value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._download(req) - -class TestProcessResponseInvalidOutput(TestManagerBase): - """Invalid return value for process_response method should raise an exception""" - - def test_invalid_process_response(self): + @deferred_f_from_coro_f + async def test_invalid_process_response(self): + """Invalid return value for process_response method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessResponseMiddleware: @@ -162,18 +155,12 @@ class TestProcessResponseInvalidOutput(TestManagerBase): return 1 self.mwman._add_middleware(InvalidProcessResponseMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - assert isinstance(results[0], Failure) - assert isinstance(results[0].value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._download(req) - -class TestProcessExceptionInvalidOutput(TestManagerBase): - """Invalid return value for process_exception method should raise an exception""" - - def test_invalid_process_exception(self): + @deferred_f_from_coro_f + async def test_invalid_process_exception(self): + """Invalid return value for process_exception method should raise an exception""" req = Request("http://example.com/index.html") class InvalidProcessExceptionMiddleware: @@ -184,18 +171,15 @@ class TestProcessExceptionInvalidOutput(TestManagerBase): return 1 self.mwman._add_middleware(InvalidProcessExceptionMiddleware()) - download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - assert isinstance(results[0], Failure) - assert isinstance(results[0].value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._download(req) class TestMiddlewareUsingDeferreds(TestManagerBase): """Middlewares using Deferreds should work""" - def test_deferred(self): + @deferred_f_from_coro_f + async def test_deferred(self): resp = Response("http://example.com/index.html") class DeferredMiddleware: @@ -211,12 +195,10 @@ class TestMiddlewareUsingDeferreds(TestManagerBase): self.mwman._add_middleware(DeferredMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called @@ -224,27 +206,27 @@ class TestMiddlewareUsingDeferreds(TestManagerBase): class TestMiddlewareUsingCoro(TestManagerBase): """Middlewares using asyncio coroutines should work""" - def test_asyncdef(self): + @deferred_f_from_coro_f + async def test_asyncdef(self): resp = Response("http://example.com/index.html") class CoroMiddleware: async def process_request(self, request, spider): - await defer.succeed(42) + await succeed(42) return resp self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called @pytest.mark.only_asyncio - def test_asyncdef_asyncio(self): + @deferred_f_from_coro_f + async def test_asyncdef_asyncio(self): resp = Response("http://example.com/index.html") class CoroMiddleware: @@ -255,10 +237,8 @@ class TestMiddlewareUsingCoro(TestManagerBase): self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - dfd = self.mwman.download(download_func, req, self.spider) - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - - assert results[0] is resp + result = await maybe_deferred_to_future( + self.mwman.download(download_func, req, self.spider) + ) + assert result is resp assert not download_func.called diff --git a/tests/test_engine.py b/tests/test_engine.py index ba4c6dc40..8928e4daf 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -487,18 +487,17 @@ def test_request_scheduled_signal(caplog): if "drop" in request.url: raise IgnoreRequest - spider = MySpider() - crawler = get_crawler(spider.__class__) + crawler = get_crawler(MySpider) engine = ExecutionEngine(crawler, lambda _: None) engine.downloader._slot_gc_loop.stop() scheduler = TestScheduler() engine.slot = Slot((), None, Mock(), scheduler) crawler.signals.connect(signal_handler, request_scheduled) keep_request = Request("https://keep.example") - engine._schedule_request(keep_request, spider) + engine._schedule_request(keep_request) drop_request = Request("https://drop.example") caplog.set_level(DEBUG) - engine._schedule_request(drop_request, spider) + engine._schedule_request(drop_request) assert scheduler.enqueued == [keep_request], ( f"{scheduler.enqueued!r} != [{keep_request!r}]" ) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index ddc9b5206..1d671134e 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,12 +1,12 @@ from __future__ import annotations from collections.abc import AsyncIterator, Iterable +from typing import Any from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from scrapy.core.spidermw import SpiderMiddlewareManager @@ -14,7 +14,11 @@ from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen -from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from scrapy.utils.defer import ( + deferred_f_from_coro_f, + deferred_from_coro, + maybe_deferred_to_future, +) from scrapy.utils.test import get_crawler @@ -26,53 +30,51 @@ class TestSpiderMiddleware(TestCase): self.spider = self.crawler._create_spider("foo") self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) - def _scrape_response(self): + async def _scrape_response(self) -> Any: """Execute spider mw manager's scrape_response method and return the result. Raise exception in case of failure. """ scrape_func = mock.MagicMock() - dfd = self.mwman.scrape_response( - scrape_func, self.response, self.request, self.spider + return await maybe_deferred_to_future( + self.mwman.scrape_response( + scrape_func, self.response, self.request, self.spider + ) ) - # catch deferred result and return the value - results = [] - dfd.addBoth(results.append) - self._wait(dfd) - return results[0] class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_input method""" - def test_invalid_process_spider_input(self): + @deferred_f_from_coro_f + async def test_invalid_process_spider_input(self): class InvalidProcessSpiderInputMiddleware: def process_spider_input(self, response, spider): return 1 self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._scrape_response() class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_output method""" - def test_invalid_process_spider_output(self): + @deferred_f_from_coro_f + async def test_invalid_process_spider_output(self): class InvalidProcessSpiderOutputMiddleware: def process_spider_output(self, response, result, spider): return 1 self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._scrape_response() class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_exception method""" - def test_invalid_process_spider_exception(self): + @deferred_f_from_coro_f + async def test_invalid_process_spider_exception(self): class InvalidProcessSpiderOutputExceptionMiddleware: def process_spider_exception(self, response, exception, spider): return 1 @@ -83,15 +85,15 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, _InvalidOutput) + with pytest.raises(_InvalidOutput): + await self._scrape_response() class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): """Re raise the exception by returning None""" - def test_process_spider_exception_return_none(self): + @deferred_f_from_coro_f + async def test_process_spider_exception_return_none(self): class ProcessSpiderExceptionReturnNoneMiddleware: def process_spider_exception(self, response, exception, spider): return None @@ -102,9 +104,8 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) - result = self._scrape_response() - assert isinstance(result, Failure) - assert isinstance(result.value, ZeroDivisionError) + with pytest.raises(ZeroDivisionError): + await self._scrape_response() class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): diff --git a/tox.ini b/tox.ini index 59572442d..e63e44189 100644 --- a/tox.ini +++ b/tox.ini @@ -143,7 +143,7 @@ deps = google-cloud-storage ipython robotexclusionrulesparser - uvloop; platform_system != "Windows" + uvloop; platform_system != "Windows" and implementation_name != "pypy" zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests [testenv:extra-deps-pinned] @@ -159,7 +159,7 @@ deps = google-cloud-storage==1.29.0 ipython==2.0.0 robotexclusionrulesparser==1.6.2 - uvloop==0.14.0; platform_system != "Windows" + uvloop==0.14.0; platform_system != "Windows" and implementation_name != "pypy" zstandard==0.1; implementation_name != "pypy" install_command = {[pinned]install_command} setenv =