diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 06da46ca1..23e60df9f 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -35,9 +35,6 @@ jobs: - python-version: "3.13" env: TOXENV: default-reactor - - python-version: pypy3.10 - env: - TOXENV: pypy3 - python-version: pypy3.11 env: TOXENV: pypy3 @@ -49,7 +46,7 @@ jobs: - python-version: "3.9.21" env: TOXENV: default-reactor-pinned - - python-version: pypy3.10 + - python-version: pypy3.11 env: TOXENV: pypy3-pinned - python-version: "3.9.21" diff --git a/docs/conf.py b/docs/conf.py index 0345ec695..5c5926dbe 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -68,6 +68,14 @@ html_css_files = [ "custom.css", ] +html_context = { + "display_github": True, + "github_user": "scrapy", + "github_repo": "scrapy", + "github_version": "master", + "conf_py_path": "/docs/", +} + # Set canonical URL from the Read the Docs Domain html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "") diff --git a/docs/index.rst b/docs/index.rst index 1a9cf636c..71047f9ef 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -132,7 +132,7 @@ Built-in services topics/telnetconsole :doc:`topics/logging` - Learn how to use Python's builtin logging on Scrapy. + Learn how to use Python's built-in logging on Scrapy. :doc:`topics/stats` Collect statistics about your scraping crawler. diff --git a/docs/news.rst b/docs/news.rst index 6a019e2fb..2a769bc1c 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -15,6 +15,14 @@ Backward-incompatible changes ``True`` when running Scrapy via :ref:`its command-line tool ` to avoid a reactor mismatch exception. +- The ``log_count/*`` stats no longer count some of the early messages that + they counted before. While the earliest log messages, emitted before the + counter is initialized, were never counted, the counter initialization now + happens later than in previous Scrapy versions. You may need to adjust + expected values if you retrieve and compare values of these stats in your + code. + (:issue:`7046`) + - The classes listed below are now :term:`abstract base classes `. They cannot be instantiated directly and their subclasses need to override the abstract methods listed below to be able to be diff --git a/docs/requirements.txt b/docs/requirements.txt index 4b382b11e..81a3119e4 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,5 @@ +pydantic==2.12.3 +scrapy-spider-metadata==0.2.0 sphinx==8.1.3 sphinx-notfound-page==1.0.4 sphinx-rtd-theme==3.0.2 diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 815501e66..dba14e749 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -88,7 +88,7 @@ recommend that such custom components should be written in the following way: 1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the default Scrapy one (e.g. - ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead + ``scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler``), but instead be able to load the class of the fallback component from a special setting (e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use it. @@ -166,7 +166,6 @@ Use a fallback component: .. code-block:: python - from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.utils.misc import build_from_crawler diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index ecde3da43..b4d058754 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -138,7 +138,7 @@ To disable cookies use: Disable retries =============== -Retrying failed HTTP requests can slow down the crawls substantially, specially +Retrying failed HTTP requests can slow down the crawls substantially, especially when sites causes are very slow (or fail) to respond, thus causing a timeout error which gets retried many times, unnecessarily, preventing crawler capacity to be reused for other domains. diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index e1e3dd6b4..0fbd22a27 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -138,6 +138,14 @@ enabled (see :ref:`topics-stats`). .. _topics-extensions-ref-telnetconsole: +Log Count extension +~~~~~~~~~~~~~~~~~~~ + +.. module:: scrapy.extensions.logcount + :synopsis: Basic stats logging + +.. autoclass:: LogCount + Telnet console extension ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -259,7 +267,7 @@ CLOSESPIDER_TIMEOUT Default: ``0`` An integer which specifies a number of seconds. If the spider remains open for -more than that number of second, it will be automatically closed with the +more than that number of seconds, it will be automatically closed with the reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by timeout. diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 56177ba4e..79d5bcce9 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -281,7 +281,7 @@ finishes before starting the next one: Distributed crawls ================== -Scrapy doesn't provide any built-in facility for running crawls in a distribute +Scrapy doesn't provide any built-in facility for running crawls in a distributed (multi-server) manner. However, there are some ways to distribute crawls, which vary depending on how you plan to distribute them. @@ -289,10 +289,10 @@ If you have many spiders, the obvious way to distribute the load is to setup many Scrapyd instances and distribute spider runs among those. If you instead want to run a single (big) spider through many machines, what -you usually do is partition the urls to crawl and send them to each separate +you usually do is partition the URLs to crawl and send them to each separate spider. Here is a concrete example: -First, you prepare the list of urls to crawl and put them into separate +First, you prepare the list of URLs to crawl and put them into separate files/urls:: http://somedomain.com/urls-to-crawl/spider1/part1.list @@ -319,7 +319,7 @@ consider contacting `commercial support`_ if in doubt. Here are some tips to keep in mind when dealing with these kinds of sites: -* rotate your user agent from a pool of well-known ones from browsers (google +* rotate your user agent from a pool of well-known ones from browsers (Google around to get a list of them) * disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use cookies to spot bot behaviour diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 9dc26d703..5e56de05c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -711,7 +711,7 @@ connections (for ``HTTP10DownloadHandler``). so you can safely ignore this setting, unless you really want to use HTTP/1.0 and override :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly, - i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. + i.e. to ``'scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler'``. .. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY @@ -909,8 +909,8 @@ Default: { "data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler", "file": "scrapy.core.downloader.handlers.file.FileDownloadHandler", - "http": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", - "https": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", + "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler", "ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler", } diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 85a08cebd..8ae8ff512 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -24,7 +24,7 @@ If you have `IPython`_ installed, the Scrapy shell will use it (instead of the standard Python console). The `IPython`_ console is much more powerful and provides smart auto-completion and colorized output, among other things. -We highly recommend you install `IPython`_, specially if you're working on +We highly recommend you install `IPython`_, especially if you're working on Unix systems (where `IPython`_ excels). See the `IPython installation guide`_ for more info. diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 8240d5d4b..0a83b248e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -364,6 +364,52 @@ used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`:: Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. +scrapy-spider-metadata parameters +--------------------------------- + +Another alternative to pass spider arguments is the library `scrapy-spider-metadata`_. + +This allows for Scrapy spiders to define, validate, document and pre-process +their arguments as Pydantic models. + +The example shows how to define typed parameters where a string argument +is automatically converted to an integer: + +.. code-block:: python + + import scrapy + from pydantic import BaseModel + from scrapy_spider_metadata import Args + + + class MyParams(BaseModel): + pages: int + + + class BookSpider(Args[MyParams], scrapy.Spider): + name = "bookspider" + start_urls = ["http://books.toscrape.com/catalogue"] + + async def start(self): + for start_url in self.start_urls: + for index in range(1, self.args.pages + 1): + yield scrapy.Request(f"{start_url}/page-{index}.html") + + def parse(self, response): + book_links = response.css("article.product_pod h3 a::attr(href)").getall() + for book_link in book_links: + yield response.follow(book_link, self.parse_book) + + def parse_book(self, response): + yield { + "title": response.css("h1::text").get(), + "price": response.css("p.price_color::text").get(), + } + +This spider can be called from the command line:: + + scrapy crawl bookspider -a pages=2 + .. _start-requests: Start requests @@ -628,7 +674,7 @@ XMLFeedSpider This method is called for the nodes matching the provided tag name (``itertag``). Receives the response and an :class:`~scrapy.Selector` for each node. Overriding this - method is mandatory. Otherwise, you spider won't work. This method + method is mandatory. Otherwise, your spider won't work. This method must return an :ref:`item object `, a :class:`~scrapy.Request` object, or an iterable containing any of them. @@ -938,6 +984,7 @@ Combine SitemapSpider with other sources of urls: def parse_other(self, response): pass # ... scrape other here ... +.. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/params.html .. _Sitemaps: https://www.sitemaps.org/index.html .. _Sitemap index files: https://www.sitemaps.org/protocol.html#index .. _robots.txt: https://www.robotstxt.org/ diff --git a/pyproject.toml b/pyproject.toml index e29393a04..47bdd3a6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -135,6 +135,9 @@ branch = true include = ["scrapy/*"] omit = ["tests/*"] disable_warnings = ["include-ignored"] +patch = [ + "subprocess", +] [tool.coverage.paths] source = [ diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 9f6c752d7..e1e027c95 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -15,8 +15,7 @@ from scrapy.exceptions import UsageError from scrapy.http import Request, Response from scrapy.utils import display from scrapy.utils.asyncgen import collect_asyncgen -from scrapy.utils.defer import aiter_errback, deferred_from_coro -from scrapy.utils.deprecate import argument_is_required +from scrapy.utils.defer import _schedule_coro, aiter_errback, deferred_from_coro from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter from scrapy.utils.spider import spidercls_for_request @@ -285,12 +284,12 @@ class Command(BaseRunSpiderCommand): if opts.pipelines: assert self.pcrawler.engine itemproc = self.pcrawler.engine.scraper.itemproc - needs_spider = argument_is_required(itemproc.process_item, "spider") - for item in items: - if needs_spider: + if hasattr(itemproc, "process_item_async"): + for item in items: + _schedule_coro(itemproc.process_item_async(item)) + else: + for item in items: itemproc.process_item(item, spider) - else: - itemproc.process_item(item) self.add_items(depth, items) self.add_requests(depth, requests) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 4b4d4efc7..76cd376e1 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -95,13 +95,6 @@ def _get_concurrency_delay( ) -> tuple[int, float]: delay: float = settings.getfloat("DOWNLOAD_DELAY") if hasattr(spider, "download_delay"): - warnings.warn( - "The 'download_delay' spider attribute is deprecated. " - "Use Spider.custom_settings or Spider.update_settings() instead. " - "The corresponding setting name is 'DOWNLOAD_DELAY'.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) delay = spider.download_delay if hasattr(spider, "max_concurrent_requests"): diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 93b96c779..bc343e37f 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,7 +1,18 @@ +import warnings + from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import ( HTTP11DownloadHandler as HTTPDownloadHandler, ) +from scrapy.exceptions import ScrapyDeprecationWarning + +warnings.warn( + "The scrapy.core.downloader.handlers.http module is deprecated," + " please import scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" + " instead of its deprecated alias scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + ScrapyDeprecationWarning, + stacklevel=2, +) __all__ = [ "HTTP10DownloadHandler", diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 02beb2f8b..05a71b742 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -2,7 +2,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any -from scrapy.core.downloader.handlers.http import HTTPDownloadHandler +from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.exceptions import NotConfigured from scrapy.utils.boto import is_botocore_available from scrapy.utils.httpobj import urlparse_cached @@ -29,7 +29,7 @@ class S3DownloadHandler: aws_access_key_id: str | None = None, aws_secret_access_key: str | None = None, aws_session_token: str | None = None, - httpdownloadhandler: type[HTTPDownloadHandler] = HTTPDownloadHandler, + httpdownloadhandler: type[HTTP11DownloadHandler] = HTTP11DownloadHandler, **kw: Any, ): if not is_botocore_available(): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 9184731cd..84cc6a0ee 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -35,7 +35,7 @@ from scrapy.utils.defer import ( parallel, parallel_async, ) -from scrapy.utils.deprecate import argument_is_required, method_is_overridden +from scrapy.utils.deprecate import method_is_overridden from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import load_object, warn_on_generator_with_return_value from scrapy.utils.python import global_object_name @@ -110,48 +110,13 @@ class Scraper: crawler.settings["ITEM_PROCESSOR"] ) self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler) - itemproc_methods = [ + self._itemproc_has_async: dict[str, bool] = {} + for method in [ "open_spider", "close_spider", - ] - if not hasattr(self.itemproc, "process_item_async"): - warnings.warn( - f"{global_object_name(itemproc_cls)} doesn't define a process_item_async() method," - f" this is deprecated and the method will be required in future Scrapy versions.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - itemproc_methods.append("process_item") - self._itemproc_has_process_async = False - elif ( - issubclass(itemproc_cls, ItemPipelineManager) - and method_is_overridden(itemproc_cls, ItemPipelineManager, "process_item") - and not method_is_overridden( - itemproc_cls, ItemPipelineManager, "process_item_async" - ) - ): - warnings.warn( - f"{global_object_name(itemproc_cls)} overrides process_item() but doesn't override process_item_async()." - f" This is deprecated. process_item() will be used, but in future Scrapy versions process_item_async() will be used instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - itemproc_methods.append("process_item") - self._itemproc_has_process_async = False - else: - self._itemproc_has_process_async = True - self._itemproc_needs_spider: dict[str, bool] = {} - for method in itemproc_methods: - self._itemproc_needs_spider[method] = argument_is_required( - getattr(self.itemproc, method), "spider" - ) - if self._itemproc_needs_spider[method]: - warnings.warn( - f"The {method}() method of {global_object_name(itemproc_cls)} requires a spider argument," - f" this is deprecated and the argument will not be passed in future Scrapy versions.", - ScrapyDeprecationWarning, - stacklevel=2, - ) + "process_item", + ]: + self._check_deprecated_itemproc_method(method) self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS") self.crawler: Crawler = crawler @@ -159,6 +124,33 @@ class Scraper: assert crawler.logformatter self.logformatter: LogFormatter = crawler.logformatter + def _check_deprecated_itemproc_method(self, method: str) -> None: + itemproc_cls = type(self.itemproc) + if not hasattr(self.itemproc, "process_item_async"): + warnings.warn( + f"{global_object_name(itemproc_cls)} doesn't define a {method}_async() method," + f" this is deprecated and the method will be required in future Scrapy versions.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + self._itemproc_has_async[method] = False + elif ( + issubclass(itemproc_cls, ItemPipelineManager) + and method_is_overridden(itemproc_cls, ItemPipelineManager, method) + and not method_is_overridden( + itemproc_cls, ItemPipelineManager, f"{method}_async" + ) + ): + warnings.warn( + f"{global_object_name(itemproc_cls)} overrides {method}() but doesn't override {method}_async()." + f" This is deprecated. {method}() will be used, but in future Scrapy versions {method}_async() will be used instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + self._itemproc_has_async[method] = False + else: + self._itemproc_has_async[method] = True + def open_spider(self, spider: Spider | None = None) -> Deferred[None]: warnings.warn( "Scraper.open_spider() is deprecated, use open_spider_async() instead", @@ -177,12 +169,12 @@ class Scraper: raise RuntimeError( "Scraper.open_spider() called before Crawler.spider is set." ) - if self._itemproc_needs_spider["open_spider"]: + if self._itemproc_has_async["open_spider"]: + await self.itemproc.open_spider_async() + else: await maybe_deferred_to_future( self.itemproc.open_spider(self.crawler.spider) ) - else: - await maybe_deferred_to_future(self.itemproc.open_spider()) def close_spider(self, spider: Spider | None = None) -> Deferred[None]: warnings.warn( @@ -202,12 +194,13 @@ class Scraper: self.slot.closing = Deferred() self._check_if_closing() await maybe_deferred_to_future(self.slot.closing) - if self._itemproc_needs_spider["close_spider"]: + if self._itemproc_has_async["close_spider"]: + await self.itemproc.close_spider_async() + else: + assert self.crawler.spider await maybe_deferred_to_future( self.itemproc.close_spider(self.crawler.spider) ) - else: - await maybe_deferred_to_future(self.itemproc.close_spider()) def is_idle(self) -> bool: """Return True if there isn't any more spiders to process""" @@ -487,14 +480,12 @@ class Scraper: assert self.crawler.spider is not None # typing self.slot.itemproc_size += 1 try: - if self._itemproc_has_process_async: + if self._itemproc_has_async["process_item"]: output = await self.itemproc.process_item_async(item) else: - if self._itemproc_needs_spider["process_item"]: - d = self.itemproc.process_item(item, self.crawler.spider) - else: - d = self.itemproc.process_item(item) - output = await maybe_deferred_to_future(d) + output = await maybe_deferred_to_future( + self.itemproc.process_item(item, self.crawler.spider) + ) except DropItem as ex: logkws = self.logformatter.dropped(item, ex, response, self.crawler.spider) if logkws is not None: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ef658e9b5..ffbebe152 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any, TypeVar from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks -from scrapy import Spider, signals +from scrapy import Spider from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import ScrapyDeprecationWarning @@ -22,7 +22,6 @@ from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader from scrapy.utils.asyncio import is_asyncio_available from scrapy.utils.defer import deferred_from_coro from scrapy.utils.log import ( - LogCounterHandler, configure_logging, get_scrapy_root_handler, install_scrapy_root_handler, @@ -97,13 +96,6 @@ class Crawler: self.addons.load_settings(self.settings) self.stats = load_object(self.settings["STATS_CLASS"])(self) - handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) - logging.root.addHandler(handler) - # lambda is assigned to Crawler attribute because this way it is not - # garbage collected after leaving the scope - self.__remove_handler = lambda: logging.root.removeHandler(handler) - self.signals.connect(self.__remove_handler, signals.engine_stopped) - lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) self.logformatter = lf_cls.from_crawler(self) diff --git a/scrapy/extensions/logcount.py b/scrapy/extensions/logcount.py new file mode 100644 index 000000000..04e570bbf --- /dev/null +++ b/scrapy/extensions/logcount.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from scrapy import Spider, signals +from scrapy.utils.log import LogCounterHandler + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + + +logger = logging.getLogger(__name__) + + +class LogCount: + """Install a log handler that counts log messages by level. + + The handler installed is :class:`scrapy.utils.log.LogCounterHandler`. + The counts are stored in stats as ``log_count/``. + + .. versionadded:: VERSION + """ + + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler + self.handler: LogCounterHandler | None = None + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + o = cls(crawler) + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) + return o + + def spider_opened(self, spider: Spider) -> None: + self.handler = LogCounterHandler( + self.crawler, level=self.crawler.settings.get("LOG_LEVEL") + ) + logging.root.addHandler(self.handler) + + def spider_closed(self, spider: Spider, reason: str) -> None: + if self.handler: + logging.root.removeHandler(self.handler) + self.handler = None diff --git a/scrapy/middleware.py b/scrapy/middleware.py index be41b52e4..d441be3ba 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -194,17 +194,13 @@ class MiddlewareManager(ABC): obj = await ensure_awaitable(method(obj, *args)) return obj - def open_spider( - self, spider: Spider | None = None - ) -> Deferred[list[None]]: # pragma: no cover + def open_spider(self, spider: Spider) -> Deferred[list[None]]: # pragma: no cover raise NotImplementedError( "MiddlewareManager.open_spider() is no longer implemented" " and will be removed in a future Scrapy version." ) - def close_spider( - self, spider: Spider | None = None - ) -> Deferred[list[None]]: # pragma: no cover + def close_spider(self, spider: Spider) -> Deferred[list[None]]: # pragma: no cover raise NotImplementedError( "MiddlewareManager.close_spider() is no longer implemented" " and will be removed in a future Scrapy version." diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index b7ec928e8..6d2494719 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -14,7 +14,11 @@ from twisted.internet.defer import Deferred, DeferredList from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list -from scrapy.utils.defer import deferred_from_coro, maybeDeferred_coro +from scrapy.utils.defer import ( + deferred_from_coro, + maybe_deferred_to_future, + maybeDeferred_coro, +) from scrapy.utils.python import global_object_name if TYPE_CHECKING: @@ -44,14 +48,13 @@ class ItemPipelineManager(MiddlewareManager): self.methods["process_item"].append(pipe.process_item) self._check_mw_method_spider_arg(pipe.process_item) - def process_item(self, item: Any, spider: Spider | None = None) -> Deferred[Any]: - if spider: - self._set_compat_spider(spider) + def process_item(self, item: Any, spider: Spider) -> Deferred[Any]: warnings.warn( f"{global_object_name(type(self))}.process_item() is deprecated, use process_item_async() instead.", category=ScrapyDeprecationWarning, stacklevel=2, ) + self._set_compat_spider(spider) return deferred_from_coro(self.process_item_async(item)) async def process_item_async(self, item: Any) -> Any: @@ -77,14 +80,26 @@ class ItemPipelineManager(MiddlewareManager): d2.addErrback(eb) return d2 - def open_spider(self, spider: Spider | None = None) -> Deferred[list[None]]: - if spider: - self._warn_spider_arg("open_spider") - self._set_compat_spider(spider) + def open_spider(self, spider: Spider) -> Deferred[list[None]]: + warnings.warn( + f"{global_object_name(type(self))}.open_spider() is deprecated, use open_spider_async() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self._set_compat_spider(spider) return self._process_parallel("open_spider") - def close_spider(self, spider: Spider | None = None) -> Deferred[list[None]]: - if spider: - self._warn_spider_arg("close_spider") - self._set_compat_spider(spider) + async def open_spider_async(self) -> None: + await maybe_deferred_to_future(self._process_parallel("open_spider")) + + def close_spider(self, spider: Spider) -> Deferred[list[None]]: + warnings.warn( + f"{global_object_name(type(self))}.close_spider() is deprecated, use close_spider_async() instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self._set_compat_spider(spider) return self._process_parallel("close_spider") + + async def close_spider_async(self) -> None: + await maybe_deferred_to_future(self._process_parallel("close_spider")) diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 450311e18..124a7f706 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -63,9 +63,10 @@ class ImagesPipeline(FilesPipeline): crawler: Crawler | None = None, ): try: - from PIL import Image # noqa: PLC0415 + from PIL import Image, ImageOps # noqa: PLC0415 self._Image = Image + self._ImageOps = ImageOps except ImportError: raise NotConfigured( "ImagesPipeline requires installing Pillow 8.0.0 or later" @@ -180,8 +181,9 @@ class ImagesPipeline(FilesPipeline): ) -> Iterable[tuple[str, Image.Image, BytesIO]]: path = self.file_path(request, response=response, info=info, item=item) orig_image = self._Image.open(BytesIO(response.body)) + transposed_image = self._ImageOps.exif_transpose(orig_image) - width, height = orig_image.size + width, height = transposed_image.size if width < self.min_width or height < self.min_height: raise ImageException( "Image too small " @@ -190,7 +192,7 @@ class ImagesPipeline(FilesPipeline): ) image, buf = self.convert_image( - orig_image, response_body=BytesIO(response.body) + transposed_image, response_body=BytesIO(response.body) ) yield path, image, buf diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index e1a12be05..18b622546 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -28,7 +28,7 @@ def decode_robotstxt( if to_native_str_type: body_decoded = to_unicode(robotstxt_body) else: - body_decoded = robotstxt_body.decode("utf-8", errors="ignore") + body_decoded = robotstxt_body.decode("utf-8-sig", errors="ignore") except UnicodeDecodeError: # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. # Switch to 'allow all' state. diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 334de6658..625231fe6 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -293,13 +293,21 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]): if isinstance(value, str): try: value_loaded = json.loads(value) - assert isinstance(value_loaded, (dict, list)) + if not isinstance(value_loaded, (dict, list)): + raise ValueError( + f"JSON string for setting '{name}' must evaluate to a dict or list, " + f"got {type(value_loaded).__name__}: {value_loaded!r}" + ) return value_loaded except ValueError: return value.split(",") if isinstance(value, tuple): return list(value) - assert isinstance(value, (dict, list)) + if not isinstance(value, (dict, list)): + raise ValueError( + f"Setting '{name}' must be a dict, list, tuple, or string, " + f"got {type(value).__name__}: {value!r}" + ) return copy.deepcopy(value) def getwithbase(self, name: _SettingsKeyT) -> BaseSettings: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index f306569e4..543e5c043 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -251,8 +251,8 @@ DOWNLOAD_HANDLERS = {} DOWNLOAD_HANDLERS_BASE = { "data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler", "file": "scrapy.core.downloader.handlers.file.FileDownloadHandler", - "http": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", - "https": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler", + "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", + "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler", "s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler", "ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler", } @@ -308,6 +308,7 @@ if sys.platform == "win32": EXTENSIONS = {} EXTENSIONS_BASE = { "scrapy.extensions.corestats.CoreStats": 0, + "scrapy.extensions.logcount.LogCount": 0, "scrapy.extensions.telnet.TelnetConsole": 0, "scrapy.extensions.memusage.MemoryUsage": 0, "scrapy.extensions.memdebug.MemoryDebugger": 0, diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 02adef1e4..f74842ac9 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -28,9 +28,15 @@ class StatsCollector: self._crawler: Crawler = crawler def __getattribute__(self, name): + cached_name = f"_cached_{name}" + try: + return super().__getattribute__(cached_name) + except AttributeError: + pass + original_attr = super().__getattribute__(name) - if name in ( + if name in { "get_value", "get_stats", "set_value", @@ -41,8 +47,10 @@ class StatsCollector: "clear_stats", "open_spider", "close_spider", - ) and callable(original_attr): - return _warn_spider_arg(original_attr) + } and callable(original_attr): + wrapped = _warn_spider_arg(original_attr) + setattr(self, cached_name, wrapped) + return wrapped return original_attr diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index c4739bcee..374c8ecb8 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -93,8 +93,10 @@ def _warn_spider_arg( ): """Decorator to warn if a ``spider`` argument is passed to a function.""" + sig = inspect.signature(func) + def check_args(*args: _P.args, **kwargs: _P.kwargs) -> None: - bound = inspect.signature(func).bind(*args, **kwargs) + bound = sig.bind(*args, **kwargs) if "spider" in bound.arguments: warnings.warn( f"Passing a 'spider' argument to {func.__qualname__}() is deprecated and " diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 533906003..e45d34ac5 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -134,14 +134,21 @@ _scrapy_root_handler: logging.Handler | None = None def install_scrapy_root_handler(settings: Settings) -> None: global _scrapy_root_handler # noqa: PLW0603 # pylint: disable=global-statement + _uninstall_scrapy_root_handler() + logging.root.setLevel(logging.NOTSET) + _scrapy_root_handler = _get_handler(settings) + logging.root.addHandler(_scrapy_root_handler) + + +def _uninstall_scrapy_root_handler() -> None: + global _scrapy_root_handler # noqa: PLW0603 # pylint: disable=global-statement + if ( _scrapy_root_handler is not None and _scrapy_root_handler in logging.root.handlers ): logging.root.removeHandler(_scrapy_root_handler) - logging.root.setLevel(logging.NOTSET) - _scrapy_root_handler = _get_handler(settings) - logging.root.addHandler(_scrapy_root_handler) + _scrapy_root_handler = None def get_scrapy_root_handler() -> logging.Handler | None: diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py index 409920dcb..d7c892828 100644 --- a/tests/mockserver/http.py +++ b/tests/mockserver/http.py @@ -14,6 +14,7 @@ from .http_resources import ( BrokenChunkedResource, BrokenDownloadResource, ChunkedResource, + Compress, ContentLengthHeaderResource, Delay, Drop, @@ -29,6 +30,8 @@ from .http_resources import ( PayloadResource, Raw, RedirectTo, + ResponseHeadersResource, + SetCookie, Status, ) @@ -75,7 +78,10 @@ class Root(resource.Resource): self.putChild(b"contentlength", ContentLengthHeaderResource()) self.putChild(b"nocontenttype", EmptyContentTypeHeaderResource()) self.putChild(b"largechunkedfile", LargeChunkedFileResource()) + self.putChild(b"compress", Compress()) self.putChild(b"duplicate-header", DuplicateHeaderResource()) + self.putChild(b"response-headers", ResponseHeadersResource()) + self.putChild(b"set-cookie", SetCookie()) def getChild(self, name, request): return self diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py index c9083b1d9..d5687fc44 100644 --- a/tests/mockserver/http_resources.py +++ b/tests/mockserver/http_resources.py @@ -1,5 +1,6 @@ from __future__ import annotations +import gzip import json import random from urllib.parse import urlencode @@ -307,3 +308,42 @@ class UriResource(resource.Resource): if request.method != b"CONNECT": return request.uri return b"" + + +class ResponseHeadersResource(resource.Resource): + """Return a response with headers set from the JSON request body""" + + def render(self, request): + body = json.loads(request.content.read().decode()) + for header_name, header_value in body.items(): + request.responseHeaders.addRawHeader(header_name, header_value) + return json.dumps(body).encode("utf-8") + + +class Compress(resource.Resource): + """Compress the data sent in the request url params and set Content-Encoding header""" + + def render(self, request): + data = request.args.get(b"data")[0] + + accept_encoding_header = request.getHeader(b"accept-encoding") + + # include common encoding schemes here + if accept_encoding_header == b"gzip": + request.setHeader(b"Content-Encoding", b"gzip") + return gzip.compress(data) + + # just set this to trigger a test failure if no valid accept-encoding header was set + request.setResponseCode(500) + return b"Did not receive a valid accept-encoding header" + + +class SetCookie(resource.Resource): + """Return a response with a Set-Cookie header for each request url parameter""" + + def render(self, request): + for cookie_name, cookie_values in request.args.items(): + for cookie_value in cookie_values: + cookie = (cookie_name.decode() + "=" + cookie_value.decode()).encode() + request.setHeader(b"Set-Cookie", cookie) + return b"" diff --git a/tests/test_addons.py b/tests/test_addons.py index 0383fa627..457945ea5 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -149,7 +149,7 @@ class TestAddonManager: ) assert ( crawler.settings.get(FALLBACK_SETTING) - == "scrapy.core.downloader.handlers.http.HTTPDownloadHandler" + == "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" ) settings_dict = { diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py index 1edef0b4a..246066485 100644 --- a/tests/test_command_startproject.py +++ b/tests/test_command_startproject.py @@ -17,9 +17,7 @@ from tests.test_commands import TestProjectBase class TestStartprojectCommand(TestProjectBase): def test_startproject(self): - p, out, err = self.proc("startproject", self.project_name) - print(out) - print(err, file=sys.stderr) + p, _, _ = self.proc("startproject", self.project_name) assert p.returncode == 0 assert Path(self.proj_path, "scrapy.cfg").exists() @@ -64,9 +62,7 @@ class TestStartprojectCommand(TestProjectBase): project_path = Path(project_dir, project_name) project_path.mkdir() - p, out, err = self.proc("startproject", project_name, cwd=project_dir) - print(out) - print(err, file=sys.stderr) + p, _, _ = self.proc("startproject", project_name, cwd=project_dir) assert p.returncode == 0 assert Path(project_path, "scrapy.cfg").exists() @@ -151,6 +147,8 @@ class TestStartprojectTemplates(TestProjectBase): project_name, ), cwd=destination, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, env=self.env, ) process.wait() @@ -204,6 +202,8 @@ class TestStartprojectTemplates(TestProjectBase): f"TEMPLATES_DIR={read_only_templates_dir}", ), cwd=destination, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, env=self.env, ) process.wait() @@ -263,6 +263,8 @@ class TestStartprojectTemplates(TestProjectBase): ".", ), cwd=project_dir, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, env=self.env, ) process.wait() @@ -306,6 +308,8 @@ class TestStartprojectTemplates(TestProjectBase): project_name, ), cwd=destination, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, env=self.env, ) process.wait() diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 255a03a2a..47bed925a 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -401,7 +401,7 @@ with multiples lines assert "Got response 200" in str(log) @inlineCallbacks - def test_crawl_multiple(self): + def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture): runner = CrawlerRunner(get_reactor_settings()) runner.crawl( SimpleSpider, @@ -414,11 +414,11 @@ with multiples lines mockserver=self.mockserver, ) - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): yield runner.join() - self._assert_retried(log) - assert "Got response 200" in str(log) + self._assert_retried(caplog.text) + assert "Got response 200" in caplog.text class TestCrawlSpider: diff --git a/tests/test_crawler.py b/tests/test_crawler.py index de20d6b90..2be3b49b3 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -36,7 +36,11 @@ from scrapy.utils.defer import ( deferred_from_coro, maybe_deferred_to_future, ) -from scrapy.utils.log import configure_logging, get_scrapy_root_handler +from scrapy.utils.log import ( + _uninstall_scrapy_root_handler, + configure_logging, + get_scrapy_root_handler, +) from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings from tests.mockserver.http import MockServer @@ -516,10 +520,13 @@ class TestCrawlerLogging: get_crawler(MySpider) assert get_scrapy_root_handler() is None - def test_spider_custom_settings_log_level(self, tmp_path): + @deferred_f_from_coro_f + async def test_spider_custom_settings_log_level(self, tmp_path): log_file = Path(tmp_path, "log.txt") log_file.write_text("previous message\n", encoding="utf-8") + info_count = None + class MySpider(scrapy.Spider): name = "spider" custom_settings = { @@ -527,15 +534,27 @@ class TestCrawlerLogging: "LOG_FILE": str(log_file), } - configure_logging() - assert get_scrapy_root_handler().level == logging.DEBUG - crawler = get_crawler(MySpider) - assert get_scrapy_root_handler().level == logging.INFO - info_count = crawler.stats.get_value("log_count/INFO") - logging.debug("debug message") # noqa: LOG015 - logging.info("info message") # noqa: LOG015 - logging.warning("warning message") # noqa: LOG015 - logging.error("error message") # noqa: LOG015 + async def start(self): + info_count_start = crawler.stats.get_value("log_count/INFO") + logging.debug("debug message") # noqa: LOG015 + logging.info("info message") # noqa: LOG015 + logging.warning("warning message") # noqa: LOG015 + logging.error("error message") # noqa: LOG015 + nonlocal info_count + info_count = ( + crawler.stats.get_value("log_count/INFO") - info_count_start + ) + return + yield + + try: + configure_logging() + assert get_scrapy_root_handler().level == logging.DEBUG + crawler = get_crawler(MySpider) + assert get_scrapy_root_handler().level == logging.INFO + await maybe_deferred_to_future(crawler.crawl()) + finally: + _uninstall_scrapy_root_handler() logged = log_file.read_text(encoding="utf-8") @@ -546,7 +565,7 @@ class TestCrawlerLogging: assert "error message" in logged assert crawler.stats.get_value("log_count/ERROR") == 1 assert crawler.stats.get_value("log_count/WARNING") == 1 - assert crawler.stats.get_value("log_count/INFO") - info_count == 1 + assert info_count == 1 assert crawler.stats.get_value("log_count/DEBUG", 0) == 0 def test_spider_custom_settings_log_append(self, tmp_path): @@ -560,9 +579,12 @@ class TestCrawlerLogging: "LOG_FILE_APPEND": False, } - configure_logging() - get_crawler(MySpider) - logging.debug("debug message") # noqa: LOG015 + try: + configure_logging() + get_crawler(MySpider) + logging.debug("debug message") # noqa: LOG015 + finally: + _uninstall_scrapy_root_handler() logged = log_file.read_text(encoding="utf-8") @@ -617,24 +639,24 @@ class TestAsyncCrawlerRunner(TestBaseCrawler): class TestCrawlerProcess(TestBaseCrawler): def test_crawler_process_accepts_dict(self): - runner = CrawlerProcess({"foo": "bar"}) + runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False) assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") def test_crawler_process_accepts_None(self): - runner = CrawlerProcess() + runner = CrawlerProcess(install_root_handler=False) self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") @pytest.mark.only_asyncio class TestAsyncCrawlerProcess(TestBaseCrawler): def test_crawler_process_accepts_dict(self): - runner = AsyncCrawlerProcess({"foo": "bar"}) + runner = AsyncCrawlerProcess({"foo": "bar"}, install_root_handler=False) assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") def test_crawler_process_accepts_None(self): - runner = AsyncCrawlerProcess() + runner = AsyncCrawlerProcess(install_root_handler=False) self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") @@ -1165,7 +1187,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): ) def test_log_scrapy_info(settings, items, caplog): with caplog.at_level("INFO"): - CrawlerProcess(settings) + CrawlerProcess(settings, install_root_handler=False) assert ( caplog.records[0].getMessage() == f"Scrapy {scrapy.__version__} started (bot: scrapybot)" diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 3784d0933..8fefc0dd7 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -2,9 +2,11 @@ from __future__ import annotations +import gzip import json import sys from abc import ABC, abstractmethod +from http import HTTPStatus from typing import TYPE_CHECKING, Any from unittest import mock @@ -93,6 +95,113 @@ class TestHttpBase(ABC): response = await download_request(download_handler, request) assert response.body == b"" + @pytest.mark.parametrize( + "http_status", + [ + pytest.param(http_status, id=f"status={http_status.value}") + for http_status in HTTPStatus + if http_status.value == 200 or http_status.value // 100 in (4, 5) + ], + ) + @deferred_f_from_coro_f + async def test_download_has_correct_http_status_code( + self, + mockserver: MockServer, + download_handler: DownloadHandlerProtocol, + http_status: HTTPStatus, + ) -> None: + request = Request( + mockserver.url(f"/status?n={http_status.value}", is_secure=self.is_secure) + ) + response = await download_request(download_handler, request) + assert response.status == http_status.value + + @deferred_f_from_coro_f + async def test_server_receives_correct_request_headers( + self, + mockserver: MockServer, + download_handler: DownloadHandlerProtocol, + ) -> None: + request_headers = { + # common request headers + "Accept": "text/html", + "Accept-Charset": "utf-8", + "Accept-Datetime": "Thu, 31 May 2007 20:35:00 GMT", + "Accept-Encoding": "gzip, deflate", + # custom headers + "X-Custom-Header": "Custom Value", + } + + request = Request( + mockserver.url("/echo", is_secure=self.is_secure), + headers=request_headers, + ) + response = await download_request(download_handler, request) + assert response.status == HTTPStatus.OK + body = json.loads(response.body.decode("utf-8")) + assert "headers" in body + for header_name, header_value in request_headers.items(): + assert header_name in body["headers"] + assert body["headers"][header_name] == [header_value] + + @deferred_f_from_coro_f + async def test_server_receives_correct_request_body( + self, + mockserver: MockServer, + download_handler: DownloadHandlerProtocol, + ) -> None: + request_body = { + "message": "It works!", + } + request = Request( + mockserver.url("/echo", is_secure=self.is_secure), + body=json.dumps(request_body), + ) + response = await download_request(download_handler, request) + assert response.status == HTTPStatus.OK + body = json.loads(response.body.decode("utf-8")) + assert json.loads(body["body"]) == request_body + + @deferred_f_from_coro_f + async def test_download_has_correct_response_headers( + self, + mockserver: MockServer, + download_handler: DownloadHandlerProtocol, + ) -> None: + # these headers will be set on the response in the resource and returned + response_headers = { + # common response headers + "Access-Control-Allow-Origin": "*", + "Allow": "Get, Head", + "Age": "12", + "Cache-Control": "max-age=3600", + "Content-Encoding": "gzip", + "Content-MD5": "Q2hlY2sgSW50ZWdyaXR5IQ==", + "Content-Type": "text/html; charset=utf-8", + "Date": "Date: Tue, 15 Nov 1994 08:12:31 GMT", + "Pragma": "no-cache", + "Retry-After": "120", + "Set-Cookie": "CookieName=CookieValue; Max-Age=3600; Version=1", + "WWW-Authenticate": "Basic", + # custom headers + "X-Custom-Header": "Custom Header Value", + } + + request = Request( + mockserver.url("/response-headers", is_secure=self.is_secure), + headers={"content-type": "application/json"}, + body=json.dumps(response_headers), + ) + response = await download_request(download_handler, request) + assert response.status == 200 + for header_name, header_value in response_headers.items(): + assert header_name in response.headers, ( + f"Response was missing expected header {header_name}" + ) + assert response.headers[header_name] == bytes( + header_value, encoding="utf-8" + ) + @deferred_f_from_coro_f async def test_redirect_status( self, mockserver: MockServer, download_handler: DownloadHandlerProtocol @@ -258,6 +367,65 @@ class TestHttpBase(ABC): response = await download_request(download_handler, request) assert response.headers.getlist(b"Set-Cookie") == [b"a=b", b"c=d"] + @deferred_f_from_coro_f + async def test_download_is_not_automatically_gzip_decoded( + self, download_handler: DownloadHandlerProtocol, mockserver: MockServer + ) -> None: + """Test download handler does not automatically decode content using the scheme provided in Content-Encoding header""" + + data = "compress-me" + + # send a request to mock resource that gzip encodes the "data" url parameter + request = Request( + mockserver.url(f"/compress?data={data}", is_secure=self.is_secure), + headers={ + "accept-encoding": "gzip", + }, + ) + response = await download_request(download_handler, request) + + assert response.status == 200 + + # check that the Content-Encoding header is gzip + content_encoding = response.headers[b"Content-Encoding"] + assert content_encoding == b"gzip" + + # check that the response is still encoded + # by checking for the magic number that is always included at the start of a gzip encoding + # see https://datatracker.ietf.org/doc/html/rfc1952#page-5 section 2.3.1 + GZIP_MAGIC = b"\x1f\x8b" + assert response.body[:2] == GZIP_MAGIC, "Response body was not in gzip format" + + # check that a gzip decoding matches the data sent in the request + expected_decoding = bytes(data, encoding="utf-8") + assert gzip.decompress(response.body) == expected_decoding + + @deferred_f_from_coro_f + async def test_no_cookie_processing_or_persistence( + self, mockserver: MockServer, download_handler: DownloadHandlerProtocol + ) -> None: + cookie_name = "foo" + cookie_value = "bar" + + # check that cookies are not modified + request = Request( + mockserver.url( + f"/set-cookie?{cookie_name}={cookie_value}", is_secure=self.is_secure + ) + ) + response = await download_request(download_handler, request) + assert response.status == 200 + set_cookie = response.headers.get(b"Set-Cookie") + assert set_cookie == f"{cookie_name}={cookie_value}".encode() + + # check that cookies are not sent in the next request + request = Request(mockserver.url("/echo", is_secure=self.is_secure)) + response = await download_request(download_handler, request) + assert response.status == 200 + headers = Headers(json.loads(response.text)["headers"]) + assert "Cookie" not in headers + assert "cookie" not in headers + class TestHttp11Base(TestHttpBase): """HTTP 1.1 test case""" diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index f50e7bec7..03b941e4f 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -32,8 +32,10 @@ class TestManagerBase: mwman = DownloaderMiddlewareManager.from_crawler(crawler) crawler.engine = crawler._create_engine() await crawler.engine.open_spider_async() - yield mwman - await crawler.engine.close_spider_async() + try: + yield mwman + finally: + await crawler.engine.close_spider_async() @staticmethod async def _download( @@ -309,7 +311,8 @@ class TestDownloadDeprecated(TestManagerBase): async with self.get_mwman() as mwman: with pytest.warns( ScrapyDeprecationWarning, - match=r"Passing a spider argument to DownloaderMiddlewareManager.download\(\) is deprecated", + match=r"Passing a spider argument to DownloaderMiddlewareManager.download\(\)" + r" is deprecated and the passed value is ignored.", ): ret = await maybe_deferred_to_future( mwman.download(download_func, req, mwman.crawler.spider) diff --git a/tests/test_engine.py b/tests/test_engine.py index 4bfccdb18..430d599c8 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -489,6 +489,7 @@ class TestEngine(TestEngineBase): ) p = subprocess.Popen( args, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, ) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c5bd1b172..aa4477210 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1717,7 +1717,6 @@ class TestFeedExport(TestFeedExportBase): with LogCapture() as log: await self.exported_data(items, settings) - print(log) for fmt in ["json", "xml", "csv"]: assert f"Stored {fmt} feed (2 items)" in str(log) @@ -1738,7 +1737,6 @@ class TestFeedExport(TestFeedExportBase): with LogCapture() as log: await self.exported_data(items, settings) - print(log) for fmt in ["json", "xml", "csv"]: assert f"Error storing {fmt} feed (2 items)" in str(log) diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 3acb9bf7f..c633e867a 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -169,13 +169,33 @@ class TestImagesPipeline: path, new_im, new_buf = next(get_images_gen) assert path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" - assert orig_im == new_im + assert orig_im.copy() == new_im assert buf.getvalue() == new_buf.getvalue() thumb_path, thumb_img, thumb_buf = next(get_images_gen) assert thumb_path == "thumbs/small/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" assert orig_thumb_buf.getvalue() == thumb_buf.getvalue() + def test_get_transposed_images(self): + orig_im = Image.new("RGB", (2, 2), (0, 0, 0)) + orig_im.putpixel((1, 1), (255, 0, 0)) + exif = orig_im.getexif() + exif[274] = 3 + buf = io.BytesIO() + orig_im.save(buf, "PNG", exif=exif) + buf.seek(0) + + resp = Response(url="https://dev.mydeco.com/mydeco.gif", body=buf.getvalue()) + req = Request(url="https://dev.mydeco.com/mydeco.gif") + + get_images_gen = self.pipeline.get_images( + response=resp, request=req, info=object() + ) + + path, new_im, _ = next(get_images_gen) + assert path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg" + assert new_im.getpixel((0, 0)) == (255, 0, 0) + def test_convert_image(self): SIZE = (100, 100) # straight forward case: RGB and JPEG diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 4b8007ead..131a0e36b 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -32,6 +32,10 @@ class DeprecatedSpiderArgPipeline: def close_spider(self, spider): pass + def process_item(self, item, spider): + item["pipeline_passed"] = True + return item + class DeferredPipeline: def cb(self, item): @@ -145,11 +149,32 @@ class TestPipeline: yield crawler.crawl(mockserver=self.mockserver) assert len(self.items) == 1 + @deferred_f_from_coro_f + async def test_deprecated_spider_arg(self, mockserver: MockServer) -> None: + crawler = self._create_crawler(DeprecatedSpiderArgPipeline) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.process_item\(\) requires a spider argument", + ), + ): + await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver)) + + assert len(self.items) == 1 + class TestCustomPipelineManager: def test_deprecated_process_item_spider_arg(self) -> None: class CustomPipelineManager(ItemPipelineManager): - def process_item(self, item, spider): # pylint: disable=signature-differs + def process_item(self, item, spider): # pylint: disable=useless-parent-delegation return super().process_item(item, spider) crawler = get_crawler(DefaultSpider) @@ -190,13 +215,21 @@ class TestCustomPipelineManager: @deferred_f_from_coro_f async def test_integration_no_async_subclass(self, mockserver: MockServer) -> None: class CustomPipelineManager(ItemPipelineManager): - def open_spider(self, spider): # pylint: disable=signature-differs - return super().open_spider(spider) + def open_spider(self, spider): + with pytest.warns( + ScrapyDeprecationWarning, + match=r"CustomPipelineManager.open_spider\(\) is deprecated, use open_spider_async\(\)", + ): + return super().open_spider(spider) - def close_spider(self, spider): # pylint: disable=signature-differs - return super().close_spider(spider) + def close_spider(self, spider): + with pytest.warns( + ScrapyDeprecationWarning, + match=r"CustomPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\)", + ): + return super().close_spider(spider) - def process_item(self, item, spider): # pylint: disable=signature-differs + def process_item(self, item, spider): with pytest.warns( ScrapyDeprecationWarning, match=r"CustomPipelineManager.process_item\(\) is deprecated, use process_item_async\(\)", @@ -222,23 +255,11 @@ class TestCustomPipelineManager: with ( pytest.warns( ScrapyDeprecationWarning, - match=r"The open_spider\(\) method of .+\.CustomPipelineManager requires a spider argument", + match=r"CustomPipelineManager overrides open_spider\(\) but doesn't override open_spider_async\(\)", ), pytest.warns( ScrapyDeprecationWarning, - match=r"The close_spider\(\) method of .+\.CustomPipelineManager requires a spider argument", - ), - pytest.warns( - ScrapyDeprecationWarning, - match=r"The process_item\(\) method of .+\.CustomPipelineManager requires a spider argument", - ), - pytest.warns( - ScrapyDeprecationWarning, - match=r"Passing a spider argument to CustomPipelineManager.open_spider\(\) is deprecated", - ), - pytest.warns( - ScrapyDeprecationWarning, - match=r"Passing a spider argument to CustomPipelineManager.close_spider\(\) is deprecated", + match=r"CustomPipelineManager overrides close_spider\(\) but doesn't override close_spider_async\(\)", ), pytest.warns( ScrapyDeprecationWarning, @@ -294,22 +315,18 @@ class TestCustomPipelineManager: crawler.spider = crawler._create_spider() crawler.signals.connect(_on_item_scraped, signals.item_scraped) with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"CustomPipelineManager doesn't define a open_spider_async\(\) method", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"CustomPipelineManager doesn't define a close_spider_async\(\) method", + ), pytest.warns( ScrapyDeprecationWarning, match=r"CustomPipelineManager doesn't define a process_item_async\(\) method", ), - pytest.warns( - ScrapyDeprecationWarning, - match=r"The open_spider\(\) method of .+\.CustomPipelineManager requires a spider argument", - ), - pytest.warns( - ScrapyDeprecationWarning, - match=r"The close_spider\(\) method of .+\.CustomPipelineManager requires a spider argument", - ), - pytest.warns( - ScrapyDeprecationWarning, - match=r"The process_item\(\) method of .+\.CustomPipelineManager requires a spider argument", - ), ): await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver)) @@ -325,9 +342,12 @@ class TestMiddlewareManagerSpider: def crawler(self) -> Crawler: return get_crawler(Spider) - def test_deprecated_spider_arg_no_crawler_spider(self, crawler: Crawler) -> None: - """Crawler is provided, but doesn't have a spider. The instance passed to the method is - ignored and raises a warning.""" + @deferred_f_from_coro_f + async def test_deprecated_spider_arg_no_crawler_spider( + self, crawler: Crawler + ) -> None: + """Crawler is provided, but doesn't have a spider, the methods raise an exception. + The instance passed to a deprecated method is ignored.""" mwman = ItemPipelineManager(crawler=crawler) with ( pytest.warns( @@ -338,12 +358,16 @@ class TestMiddlewareManagerSpider: ScrapyDeprecationWarning, match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument", ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.process_item\(\) requires a spider argument", + ), ): mwman._add_middleware(DeprecatedSpiderArgPipeline()) with ( pytest.warns( ScrapyDeprecationWarning, - match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated", + match=r"ItemPipelineManager.open_spider\(\) is deprecated, use open_spider_async\(\) instead", ), pytest.raises( ValueError, @@ -351,10 +375,15 @@ class TestMiddlewareManagerSpider: ), ): mwman.open_spider(DefaultSpider()) + with pytest.raises( + ValueError, + match="ItemPipelineManager needs to access self.crawler.spider but it is None", + ): + await mwman.open_spider_async() with ( pytest.warns( ScrapyDeprecationWarning, - match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated", + match=r"ItemPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\) instead", ), pytest.raises( ValueError, @@ -362,59 +391,59 @@ class TestMiddlewareManagerSpider: ), ): mwman.close_spider(DefaultSpider()) + with pytest.raises( + ValueError, + match="ItemPipelineManager needs to access self.crawler.spider but it is None", + ): + await mwman.close_spider_async() def test_deprecated_spider_arg_with_crawler(self, crawler: Crawler) -> None: - """Crawler is provided and has a spider, works. The instance passed to the method is ignored, - even if mismatched, but raises a warning.""" + """Crawler is provided and has a spider, works. The instance passed to a deprecated method + is ignored, even if mismatched.""" mwman = ItemPipelineManager(crawler=crawler) crawler.spider = crawler._create_spider("foo") with pytest.warns( ScrapyDeprecationWarning, - match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated", + match=r"ItemPipelineManager.open_spider\(\) is deprecated, use open_spider_async\(\) instead", ): mwman.open_spider(DefaultSpider()) with pytest.warns( ScrapyDeprecationWarning, - match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated", + match=r"ItemPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\) instead", ): mwman.close_spider(DefaultSpider()) def test_deprecated_spider_arg_without_crawler(self) -> None: - """The first instance passed to the method is used, with a warning. Mismatched ones raise an error.""" + """The first instance passed to a deprecated method is used. Mismatched ones raise an error.""" with pytest.warns( ScrapyDeprecationWarning, match="was called without the crawler argument", ): mwman = ItemPipelineManager() - with ( - pytest.warns( - ScrapyDeprecationWarning, - match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument", - ), - pytest.warns( - ScrapyDeprecationWarning, - match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument", - ), - ): - mwman._add_middleware(DeprecatedSpiderArgPipeline()) + spider = DefaultSpider() with pytest.warns( ScrapyDeprecationWarning, - match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated", + match=r"ItemPipelineManager.open_spider\(\) is deprecated, use open_spider_async\(\) instead", ): - mwman.open_spider(DefaultSpider()) + mwman.open_spider(spider) with ( pytest.warns( ScrapyDeprecationWarning, - match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated", + match=r"ItemPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\) instead", ), pytest.raises( RuntimeError, match="Different instances of Spider were passed" ), ): mwman.close_spider(DefaultSpider()) - mwman.close_spider() + with pytest.warns( + ScrapyDeprecationWarning, + match=r"ItemPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\) instead", + ): + mwman.close_spider(spider) - def test_no_spider_arg_without_crawler(self) -> None: + @deferred_f_from_coro_f + async def test_no_spider_arg_without_crawler(self) -> None: """If no crawler and no spider arg, raise an error.""" with pytest.warns( ScrapyDeprecationWarning, @@ -430,6 +459,10 @@ class TestMiddlewareManagerSpider: ScrapyDeprecationWarning, match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument", ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.process_item\(\) requires a spider argument", + ), ): mwman._add_middleware(DeprecatedSpiderArgPipeline()) with ( @@ -438,4 +471,4 @@ class TestMiddlewareManagerSpider: match="has no known Spider instance", ), ): - mwman.open_spider() + await mwman.open_spider_async() diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index f9874f821..61d79743d 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -116,9 +116,7 @@ class TestProxyConnect: assert "Proxy-Authorization" not in echo["headers"] def _assert_got_response_code(self, code, log): - print(log) assert str(log).count(f"Crawled ({code})") == 1 def _assert_got_tunnel_error(self, log): - print(log) assert "TunnelError" in str(log) diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 6a24d2e90..bc79e3a17 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -129,6 +129,12 @@ class TestDecodeRobotsTxt: decoded_content = decode_robotstxt(robotstxt_body, spider=None) assert decoded_content == "User-agent: *\nDisallow: /\n" + # UTF-8 BOM at the beginning of the file ignored + def test_decode_utf8_bom(self): + robotstxt_body = b"\xef\xbb\xbfUser-agent: *\nDisallow: /\n" + decoded_content = decode_robotstxt(robotstxt_body, spider=None) + assert decoded_content == "User-agent: *\nDisallow: /\n" + class TestPythonRobotParser(BaseRobotParserTest): def setup_method(self): diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 8fd33eb13..977273673 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -16,6 +16,7 @@ from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.asyncio import call_later from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler if TYPE_CHECKING: @@ -604,7 +605,7 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): class TestDeprecatedSpiderArg(TestSpiderMiddleware): @deferred_f_from_coro_f - async def test_deprecated_spider_arg(self): + async def test_deprecated_mw_spider_arg(self): class DeprecatedSpiderArgMiddleware: def process_spider_input(self, response, spider): return None @@ -631,3 +632,26 @@ class TestDeprecatedSpiderArg(TestSpiderMiddleware): ): self.mwman._add_middleware(DeprecatedSpiderArgMiddleware()) await self._scrape_response() + + @deferred_f_from_coro_f + async def test_deprecated_mwman_spider_arg(self): + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a spider argument to SpiderMiddlewareManager.process_start\(\)" + r" is deprecated and the passed value is ignored", + ): + await self.mwman.process_start(DefaultSpider()) + + @deferred_f_from_coro_f + async def test_deprecated_mwman_spider_arg_no_crawler(self): + with pytest.warns( + ScrapyDeprecationWarning, + match=r"MiddlewareManager.__init__\(\) was called without the crawler argument", + ): + mwman = SpiderMiddlewareManager() + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a spider argument to SpiderMiddlewareManager.process_start\(\)" + r" is deprecated, SpiderMiddlewareManager should be instantiated with a Crawler", + ): + await mwman.process_start(DefaultSpider()) diff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.py new file mode 100644 index 000000000..2560f8d7f --- /dev/null +++ b/tests/test_zz_resources.py @@ -0,0 +1,27 @@ +"""Test that certain resources are not leaked during earlier tests.""" + +from __future__ import annotations + +import logging + +from scrapy.utils.log import LogCounterHandler + + +def test_counter_handler() -> None: + """Test that ``LogCounterHandler`` is always properly removed. + + It's added in ``Crawler.crawl{,_async}()`` and removed on engine_stopped. + """ + c = sum(1 for h in logging.root.handlers if isinstance(h, LogCounterHandler)) + assert c == 0 + + +def test_stderr_log_handler() -> None: + """Test that the Scrapy root handler is always properly removed. + + It's added in ``configure_logging()``, called by ``{Async,}CrawlerProcess`` + (without ``install_root_handler=False``). It can be removed with + ``_uninstall_scrapy_root_handler()`` if installing it was really neeeded. + """ + c = sum(1 for h in logging.root.handlers if type(h) is logging.StreamHandler) # pylint: disable=unidiomatic-typecheck + assert c == 0 diff --git a/tests/upper-constraints.txt b/tests/upper-constraints.txt deleted file mode 100644 index 2a335e533..000000000 --- a/tests/upper-constraints.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Request the latest known version or newer of some dependencies to prevent the -# pip dependency resolver from spending too much time backtracking. -attrs>=20.2.0 -Automat>=0.8.0 -botocore>=1.20.30 -itemadapter>=0.1.1 -itemloaders>=1.0.3 -lxml>=4.6.1 -parsel>=1.5.2 -Pillow>=8.0.1 -pyOpenSSL>=17.5 # mitmproxy 4.0.4 -pytest>=6.2.1 -pytest-twisted>=1.13.1 -service_identity>=17.0.0 -six>=1.14.0 -sybil>=2.0.0 -Twisted>=19.10.0 diff --git a/tox.ini b/tox.ini index db6add080..5f49e6a5a 100644 --- a/tox.ini +++ b/tox.ini @@ -10,12 +10,12 @@ minversion = 1.7.0 [test-requirements] deps = attrs - coverage >= 7.4.0 + coverage >= 7.10.6 pexpect >= 4.8.0 pyftpdlib >= 2.0.1 pygments pytest - pytest-cov >= 4.0.0 + pytest-cov >= 7.0.0 pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures @@ -40,9 +40,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 docs scrapy tests --doctest-modules} -install_command = - python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 scrapy tests --doctest-modules} [testenv:typing] basepython = python3.9 @@ -121,10 +119,7 @@ deps = # above, hence we do not install it in pinned environments at the moment setenv = _SCRAPY_PINNED=true -install_command = - python -I -m pip install {opts} {packages} commands = - ; tests for docs fail with parsel < 1.8.0 pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=pinned.junit.xml -o junit_family=legacy --durations=10 scrapy tests} [testenv:pinned] @@ -132,7 +127,6 @@ basepython = {[pinned]basepython} deps = {[pinned]deps} PyDispatcher==2.0.5 -install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = {[pinned]commands} @@ -168,7 +162,6 @@ deps = robotexclusionrulesparser==1.6.2 uvloop==0.14.0; platform_system != "Windows" and implementation_name != "pypy" zstandard==0.1; implementation_name != "pypy" -install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = {[pinned]commands} @@ -181,7 +174,6 @@ commands = basepython = {[pinned]basepython} deps = {[testenv:pinned]deps} commands = {[pinned]commands} --reactor=default -install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -189,7 +181,7 @@ setenv = basepython = pypy3 commands = ; not enabling coverage as it significantly increases the run time - pytest {posargs:--durations=10 docs scrapy tests} + pytest {posargs:--durations=10 scrapy tests} [testenv:pypy3-extra-deps] basepython = pypy3 @@ -198,27 +190,26 @@ deps = commands = {[testenv:pypy3]commands} [testenv:pypy3-pinned] -basepython = pypy3.10 +basepython = pypy3.11 deps = PyPyDispatcher==2.1.0 {[test-requirements]deps} pytest==8.4.0 Protego==0.1.15 Twisted==21.7.0 - cryptography==41.0.5 + cryptography==44.0.2 cssselect==0.9.1 itemadapter==0.1.0 - lxml==4.6.0 + lxml==5.3.2 parsel==1.5.0 - pyOpenSSL==23.3.0 + pyOpenSSL==24.3.0 queuelib==1.4.2 service_identity==18.1.0 - w3lib==1.17.0 + w3lib==1.20.0 zope.interface==5.1.0 commands = - ; disabling both coverage and docs tests + ; disabling coverage pytest {posargs:--durations=10 scrapy tests} -install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -233,10 +224,13 @@ setenv = [testenv:docs] basepython = python3 changedir = {[docs]changedir} -deps = {[docs]deps} +deps = + {[test-requirements]deps} + {[docs]deps} setenv = {[docs]setenv} commands = sphinx-build -W -b html . {envtmpdir}/html + pytest [testenv:docs-coverage] basepython = python3 @@ -269,7 +263,6 @@ basepython = {[pinned]basepython} deps = {[pinned]deps} botocore==1.4.87 -install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands =