From 6cefaa5434da050bc23ffaaf2c1b38be1041bd31 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 12:43:40 +0200 Subject: [PATCH 01/13] Add a stats reference (#7814) --- .pre-commit-config.yaml | 2 +- docs/requirements.in | 2 +- docs/requirements.txt | 2 +- docs/topics/extensions.rst | 21 +- docs/topics/settings.rst | 3 +- docs/topics/stats.rst | 641 ++++++++++++++++++++++++++++++++++ scrapy/core/scheduler.py | 8 +- scrapy/extensions/logcount.py | 2 +- tox.ini | 2 +- 9 files changed, 659 insertions(+), 24 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 311df7052..c27348c7a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,6 @@ repos: hooks: - id: sphinx-lint - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.8.8 + rev: 0.8.9 hooks: - id: sphinx-scrapy diff --git a/docs/requirements.in b/docs/requirements.in index a1f3a7468..257365380 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -5,4 +5,4 @@ sphinx sphinx-notfound-page sphinx-rtd-theme sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9 diff --git a/docs/requirements.txt b/docs/requirements.txt index a5cbad302..87634cea9 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0 # via # -r docs/requirements.in # sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@912ed0507405e16ac60a47dd08195a1cd0ced984 # via -r docs/requirements.in sphinx-sitemap==2.9.0 # via sphinx-scrapy diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 5598ab983..78b38cc3f 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -136,18 +136,10 @@ Core Stats extension Enable the collection of core statistics, provided the stats collection is enabled (see :ref:`topics-stats`). -The following stats are collected: - -* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`). -* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`). -* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`). -* ``finish_reason``: the closing reason string (e.g. ``"finished"``, - ``"closespider_timeout"``). -* ``item_scraped_count``: total number of items that passed all pipelines. -* ``item_dropped_count``: total number of items dropped by a pipeline. -* ``item_dropped_reasons_count/``: per-exception drop count - (e.g. ``item_dropped_reasons_count/DropItem``). -* ``response_received_count``: total number of HTTP responses received. +The following stats are collected: :stat:`elapsed_time_seconds`, +:stat:`finish_reason`, :stat:`finish_time`, :stat:`item_dropped_count`, +:stat:`item_dropped_reasons_count/{exception}`, :stat:`item_scraped_count`, +:stat:`response_received_count`, :stat:`start_time`. Log Count extension ~~~~~~~~~~~~~~~~~~~ @@ -190,7 +182,7 @@ Monitors the memory used by the Scrapy process that runs the spider and: 1. sends a :signal:`memusage_warning_reached` signal when it exceeds :setting:`MEMUSAGE_WARNING_MB` -2. closes the spider with the `"memusage_exceeded"` reason when it exceeds +2. closes the spider with the ``"memusage_exceeded"`` reason when it exceeds :setting:`MEMUSAGE_LIMIT_MB` This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and @@ -214,7 +206,8 @@ An extension for debugging memory usage. It collects information about: * objects left alive that shouldn't. For more info, see :ref:`topics-leaks-trackrefs` To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The -info will be stored in the stats. +info will be stored in the :stat:`memdebug/gc_garbage_count` and +:stat:`memdebug/live_refs/{cls}` stats. .. _topics-extensions-ref-spiderstate: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 81055afc2..b07dff180 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1871,7 +1871,8 @@ Default: ``False`` Setting to ``True`` will log debug information about the requests scheduler. This currently logs (only once) if the requests cannot be serialized to disk. -Stats counter (``scheduler/unserializable``) tracks the number of times this happens. +The :stat:`scheduler/unserializable` stat tracks the number of times this +happens. Example entry in logs:: diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 0cf4a72cc..c702cefe7 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -21,6 +21,8 @@ using the Stats Collector from. Another feature of the Stats Collector is that it's very efficient (when enabled) and extremely efficient (almost unnoticeable) when disabled. +See :ref:`topics-stats-reference` below for the stats that Scrapy sets. + .. _topics-stats-usecases: Common Stats Collector uses @@ -101,3 +103,642 @@ DummyStatsCollector ------------------- .. autoclass:: DummyStatsCollector + +.. _topics-stats-reference: + +Built-in stats reference +======================== + +Scrapy sets the following :ref:`stats `. Components other than +those built into Scrapy may set additional stats; see their documentation. + +Stat keys that contain a ``{placeholder}`` below stand for a family of stats, +one per actual value of the placeholder. + +.. note:: Most stats are set by a specific :ref:`component + `, and are only present if that component is enabled and + its code path is reached. A stat that is missing from + :meth:`~scrapy.statscollectors.StatsCollector.get_stats` output is + equivalent to a counter of 0. + +.. stat:: downloader/exception_count + +``downloader/exception_count`` + Number of exceptions raised while downloading requests. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/exception_type_count/{exception_type} + +``downloader/exception_type_count/{exception_type}`` + Number of exceptions raised while downloading requests, per exception type, + where ``{exception_type}`` is the import path of the exception class, e.g. + ``twisted.internet.error.DNSLookupError``. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_bytes + +``downloader/request_bytes`` + Total size, in bytes, of the requests sent, counting the request line, the + headers and the body. As with :stat:`downloader/request_count`, requests + served from the cache are also counted. + + It is an approximation, reconstructed from each :class:`~scrapy.Request` + object instead of measured on the wire, so it does not account for the + actual bytes that the :ref:`download handler + ` sends, e.g. transport-level overhead. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_count + +``downloader/request_count`` + Number of requests sent. + + Requests that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache are also counted, even though they are never sent, + because it handles requests after + :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/request_method_count/{method} + +``downloader/request_method_count/{method}`` + Number of requests sent, per HTTP method, e.g. ``GET`` or ``POST``. As with + :stat:`downloader/request_count`, requests served from the cache are also + counted. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_bytes + +``downloader/response_bytes`` + Total size, in bytes, of the responses received, counting the status line, + the headers and the body. It covers the same responses as + :stat:`downloader/response_count`. + + The body is counted as received, i.e. still compressed for responses that + used ``Content-Encoding``, because + :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` handles + responses before + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + decompresses them. See :stat:`httpcompression/response_bytes` for + decompressed sizes. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_count + +``downloader/response_count`` + Number of responses received. + + It counts responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache, even though they do not come from the network, and + responses that a downloader middleware consumes before they reach your + spider, e.g. redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware` + turns into new requests. Compare with :stat:`response_received_count`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: downloader/response_status_count/{status_code} + +``downloader/response_status_count/{status_code}`` + Number of responses received, per HTTP status code, e.g. ``200`` or + ``404``. It covers the same responses as :stat:`downloader/response_count`. + + Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`. + +.. stat:: dupefilter/filtered + +``dupefilter/filtered`` + Number of requests dropped as duplicates. + + Set by :class:`~scrapy.dupefilters.RFPDupeFilter`. + +.. stat:: elapsed_time_seconds + +``elapsed_time_seconds`` + Time, as a :class:`float`, in seconds, between the :signal:`spider_opened` + and the :signal:`spider_closed` signals. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: feedexport/failed_count/{storage} + +``feedexport/failed_count/{storage}`` + Number of :ref:`feeds ` that could not be stored, per + :ref:`storage backend `, where ``{storage}`` + is the class name of the storage backend, e.g. ``FileFeedStorage``. + +.. stat:: feedexport/success_count/{storage} + +``feedexport/success_count/{storage}`` + Number of :ref:`feeds ` stored successfully, per + :ref:`storage backend `, where ``{storage}`` + is the class name of the storage backend, e.g. ``FileFeedStorage``. + +.. stat:: file_count + +``file_count`` + Number of files handled by the :ref:`media pipelines + `. + +.. stat:: file_status_count/{status} + +``file_status_count/{status}`` + Number of files handled by the :ref:`media pipelines + `, per status, where ``{status}`` is one of: + + - ``downloaded``: the file was downloaded. + + - ``cached``: the file came from the + :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + cache. + + - ``uptodate``: the file was already in the storage backend and had not + :ref:`expired `, so it was not downloaded again. + +.. stat:: finish_reason + +``finish_reason`` + String indicating why the crawl finished. It matches the *reason* argument + of the :signal:`spider_closed` signal. + + Scrapy uses the following reasons: + + - ``cancelled``: the spider was closed without a more specific reason, + e.g. because :exc:`~scrapy.exceptions.CloseSpider` was raised without + one. + + - ``closespider_errorcount``: see :setting:`CLOSESPIDER_ERRORCOUNT`. + + - ``closespider_itemcount``: see :setting:`CLOSESPIDER_ITEMCOUNT`. + + - ``closespider_pagecount``: see :setting:`CLOSESPIDER_PAGECOUNT`. + + - ``closespider_pagecount_no_item``: see + :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM`. + + - ``closespider_timeout``: see :setting:`CLOSESPIDER_TIMEOUT`. + + - ``closespider_timeout_no_item``: see + :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`. + + - ``finished``: the spider became idle with no pending requests, i.e. it + finished normally. + + - ``memusage_exceeded``: see :setting:`MEMUSAGE_LIMIT_MB`. + + - ``shutdown``: the crawl was interrupted, e.g. by a system signal such + as ``SIGINT`` (:kbd:`Ctrl-C`). + + Third-party components and your own code may use any other reason, e.g. by + raising :exc:`~scrapy.exceptions.CloseSpider` with it. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: finish_time + +``finish_time`` + Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when + the :signal:`spider_closed` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: httpcache/errorrecovery + +``httpcache/errorrecovery`` + Number of times that a stale cached response was used because downloading a + fresh response raised an exception. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/firsthand + +``httpcache/firsthand`` + Number of responses that were downloaded without a matching cache entry to + validate against, i.e. responses for requests counted in + :stat:`httpcache/miss`. + + It is lower than :stat:`httpcache/miss` when some of those requests yield + no response, either because they are dropped (see + :stat:`httpcache/ignore`) or because their download fails. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/hit + +``httpcache/hit`` + Number of requests served from the cache. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/ignore + +``httpcache/ignore`` + Number of requests dropped because they were not in the cache and + :setting:`HTTPCACHE_IGNORE_MISSING` is ``True``. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/invalidate + +``httpcache/invalidate`` + Number of times that a cached response failed validation and was replaced + with a freshly downloaded response. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/miss + +``httpcache/miss`` + Number of requests for which no cache entry could be read, either because + there was none or because reading it failed, in which case the request is + also counted in :stat:`httpcache/retrieve_error`. Those requests are + downloaded (see :stat:`httpcache/firsthand`), or dropped if + :setting:`HTTPCACHE_IGNORE_MISSING` is ``True`` (see + :stat:`httpcache/ignore`). + + Requests with a stale cache entry are not counted here; see + :stat:`httpcache/revalidate` and :stat:`httpcache/invalidate`. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/retrieve_error + +``httpcache/retrieve_error`` + Number of cache entries that could not be read, and hence were treated as + cache misses. Those requests are also counted in :stat:`httpcache/miss`. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/revalidate + +``httpcache/revalidate`` + Number of times that a cached response was successfully validated against + the target server, and hence used instead of the fresh response. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/store + +``httpcache/store`` + Number of responses stored in the cache. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcache/uncacheable + +``httpcache/uncacheable`` + Number of responses not stored in the cache because the + :setting:`HTTPCACHE_POLICY` did not allow it. + + Every response considered for caching is counted either here or in + :stat:`httpcache/store`, so ``httpcache/store + httpcache/uncacheable`` + equals ``httpcache/firsthand + httpcache/invalidate``. + + Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`. + +.. stat:: httpcompression/response_bytes + +``httpcompression/response_bytes`` + Total size, in bytes, of decompressed response bodies, counting only the + body and only responses that were actually decompressed. Compare with + :stat:`downloader/response_bytes`. + + Set by + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`. + +.. stat:: httpcompression/response_count + +``httpcompression/response_count`` + Number of decompressed responses. + + Set by + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`. + +.. stat:: httperror/response_ignored_count + +``httperror/response_ignored_count`` + Number of responses dropped because of their HTTP status code. + + Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`. + +.. stat:: httperror/response_ignored_status_count/{status_code} + +``httperror/response_ignored_status_count/{status_code}`` + Number of responses dropped because of their HTTP status code, per HTTP + status code, e.g. ``404``. + + Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`. + +.. stat:: item_dropped_count + +``item_dropped_count`` + Number of items dropped by an :ref:`item pipeline + `, i.e. number of times that the + :signal:`item_dropped` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: item_dropped_reasons_count/{exception} + +``item_dropped_reasons_count/{exception}`` + Number of items dropped, per exception, where ``{exception}`` is the class + name of the exception that caused the item to be dropped. + + Only :exc:`~scrapy.exceptions.DropItem` and its subclasses drop items, and + each one is counted under its own class name, e.g. + ``item_dropped_reasons_count/DropItem`` for + :exc:`~scrapy.exceptions.DropItem` itself and + ``item_dropped_reasons_count/MyDropItem`` for a ``MyDropItem`` subclass of + it. Any other exception raised by an :ref:`item pipeline + ` triggers the :signal:`item_error` signal instead of + :signal:`item_dropped`, and is not counted here or in + :stat:`item_dropped_count`. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: item_scraped_count + +``item_scraped_count`` + Number of items that passed all :ref:`item pipelines + `, i.e. number of times that the + :signal:`item_scraped` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: items_per_minute + +``items_per_minute`` + Average number of items scraped per minute during the crawl. + + It is ``None`` if the crawl took less than a minute. + + Set by :class:`~scrapy.extensions.logstats.LogStats`. + +.. stat:: log_count/{level} + +``log_count/{level}`` + Number of log messages, per logging level name, e.g. ``INFO`` or + ``WARNING``. + + Only messages that the :setting:`LOG_LEVEL` setting allows are counted. + + Set by :class:`~scrapy.extensions.logcount.LogCount`. + +.. stat:: memdebug/gc_garbage_count + +``memdebug/gc_garbage_count`` + Number of objects in :data:`gc.garbage` when the spider is closed. + + Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires + :setting:`MEMDEBUG_ENABLED` to be ``True``. + +.. stat:: memdebug/live_refs/{cls} + +``memdebug/live_refs/{cls}`` + Number of live objects of class ``{cls}`` when the spider is closed, as + reported by :ref:`trackref `, e.g. + ``memdebug/live_refs/HtmlResponse``. + + Only set for classes with at least 1 live object. + + Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires + :setting:`MEMDEBUG_ENABLED` to be ``True``. + +.. stat:: memusage/limit_reached + +``memusage/limit_reached`` + ``1`` if memory usage exceeded :setting:`MEMUSAGE_LIMIT_MB`, which also + stops the crawl. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/max + +``memusage/max`` + Maximum peak memory usage, in bytes, observed during the crawl. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/startup + +``memusage/startup`` + Peak memory usage, in bytes, when the engine started. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: memusage/warning_reached + +``memusage/warning_reached`` + ``1`` if memory usage exceeded :setting:`MEMUSAGE_WARNING_MB`. + + Set by :class:`~scrapy.extensions.memusage.MemoryUsage`. + +.. stat:: offsite/domains + +``offsite/domains`` + Number of distinct domains for which at least 1 request was dropped for + being offsite. + + Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`. + +.. stat:: offsite/filtered + +``offsite/filtered`` + Number of requests dropped for being offsite. + + Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`. + +.. stat:: request_depth_count/{depth} + +``request_depth_count/{depth}`` + Number of requests scheduled at depth ``{depth}``, e.g. + ``request_depth_count/2``. + + Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`, which + requires :setting:`DEPTH_STATS_VERBOSE` to be ``True`` for this stat. + +.. stat:: request_depth_max + +``request_depth_max`` + Maximum depth reached. + + Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`. + +.. stat:: response_received_count + +``response_received_count`` + Number of responses received, i.e. number of times that the + :signal:`response_received` signal was sent. + + Unlike :stat:`downloader/response_count`, it does not count responses that + a downloader middleware consumes before they reach the engine, e.g. + redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware` + turns into new requests. Both count responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + serves from the cache. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: responses_per_minute + +``responses_per_minute`` + Average number of responses received per minute during the crawl. + + It is ``None`` if the crawl took less than a minute. + + Set by :class:`~scrapy.extensions.logstats.LogStats`. + +.. stat:: retry/count + +``retry/count`` + Number of requests retried. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. stat:: retry/max_reached + +``retry/max_reached`` + Number of requests that were not retried because they had already been + retried :setting:`RETRY_TIMES` times. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. stat:: retry/reason_count/{reason} + +``retry/reason_count/{reason}`` + Number of requests retried, per reason, e.g. + ``retry/reason_count/twisted.internet.error.TimeoutError`` or + ``retry/reason_count/504 Gateway Time-out``. + + Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses. + +.. note:: Code calling + :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` may pass a + custom *stats_base_key*, in which case ``retry`` is replaced with that key + in the 3 stats above. + +.. stat:: robotstxt/exception_count/{exception_type} + +``robotstxt/exception_count/{exception_type}`` + Number of exceptions raised while downloading ``robots.txt`` files, per + exception type, where ``{exception_type}`` is the string representation of + the exception class, e.g. ````. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/forbidden + +``robotstxt/forbidden`` + Number of requests dropped for being disallowed by ``robots.txt``. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/request_count + +``robotstxt/request_count`` + Number of ``robots.txt`` files requested, i.e. 1 per network location for + which at least 1 request was sent. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/response_count + +``robotstxt/response_count`` + Number of ``robots.txt`` responses received. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: robotstxt/response_status_count/{status_code} + +``robotstxt/response_status_count/{status_code}`` + Number of ``robots.txt`` responses received, per HTTP status code, e.g. + ``404``. + + Set by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + +.. stat:: scheduler/dequeued + +``scheduler/dequeued`` + Number of requests read from the :ref:`scheduler `. + +.. stat:: scheduler/dequeued/disk + +``scheduler/dequeued/disk`` + Number of requests read from the disk queue of the :ref:`scheduler + `. + +.. stat:: scheduler/dequeued/memory + +``scheduler/dequeued/memory`` + Number of requests read from the memory queue of the :ref:`scheduler + `. + +.. stat:: scheduler/enqueued + +``scheduler/enqueued`` + Number of requests stored into the :ref:`scheduler `. + +.. stat:: scheduler/enqueued/disk + +``scheduler/enqueued/disk`` + Number of requests stored into the disk queue of the :ref:`scheduler + `. + +.. stat:: scheduler/enqueued/memory + +``scheduler/enqueued/memory`` + Number of requests stored into the memory queue of the :ref:`scheduler + `. + +.. stat:: scheduler/unserializable + +``scheduler/unserializable`` + Number of requests that could not be stored into the disk queue of the + :ref:`scheduler ` because they could not be + :ref:`serialized `, and hence were stored into the + memory queue instead. + +.. stat:: spider_exceptions/count + +``spider_exceptions/count`` + Number of unhandled exceptions raised by spider callbacks. + + Set by the :ref:`scraper `. + +.. stat:: spider_exceptions/{exception} + +``spider_exceptions/{exception}`` + Number of unhandled exceptions raised by spider callbacks, per exception, + where ``{exception}`` is the class name of the exception, e.g. + ``spider_exceptions/ValueError``. + + Set by the :ref:`scraper `. + +.. stat:: start_time + +``start_time`` + Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when + the :signal:`spider_opened` signal was sent. + + Set by :class:`~scrapy.extensions.corestats.CoreStats`. + +.. stat:: urllength/request_ignored_count + +``urllength/request_ignored_count`` + Number of requests dropped for having a URL longer than + :setting:`URLLENGTH_LIMIT`. + + Set by :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`. diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 42c517222..a511b0c7b 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -366,8 +366,8 @@ class Scheduler(BaseScheduler): Unless the received request is filtered out by the Dupefilter, attempt to push it into the disk queue, falling back to pushing it into the memory queue. - Increment the appropriate stats, such as: ``scheduler/enqueued``, - ``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``. + Increment the appropriate stats, such as: :stat:`scheduler/enqueued`, + :stat:`scheduler/enqueued/disk`, :stat:`scheduler/enqueued/memory`. Return ``True`` if the request was stored successfully, ``False`` otherwise. """ @@ -390,8 +390,8 @@ class Scheduler(BaseScheduler): falling back to the disk queue if the memory queue is empty. Return ``None`` if there are no more enqueued requests. - Increment the appropriate stats, such as: ``scheduler/dequeued``, - ``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``. + Increment the appropriate stats, such as: :stat:`scheduler/dequeued`, + :stat:`scheduler/dequeued/disk`, :stat:`scheduler/dequeued/memory`. """ request: Request | None = self.mqs.pop() assert self.stats is not None diff --git a/scrapy/extensions/logcount.py b/scrapy/extensions/logcount.py index e6d51a7d8..fcce64438 100644 --- a/scrapy/extensions/logcount.py +++ b/scrapy/extensions/logcount.py @@ -20,7 +20,7 @@ class LogCount: """Install a log handler that counts log messages by level. The handler installed is :class:`scrapy.utils.log.LogCounterHandler`. - The counts are stored in stats as ``log_count/``. + The counts are stored in the :stat:`log_count/{level}` stat. .. versionadded:: 2.14 """ diff --git a/tox.ini b/tox.ini index 18e1579c9..b59221340 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] requires = - sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 + sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9 envlist = pre-commit pylint From 6f87d3f86334855484c8e1b0888976c70596b0e9 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 13:46:33 +0200 Subject: [PATCH 02/13] Rename the current benchmark for the future (#7839) --- .../{test_benchmark_crawl.py => test_crawl.py} | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) rename tests/benchmarks/{test_benchmark_crawl.py => test_crawl.py} (82%) diff --git a/tests/benchmarks/test_benchmark_crawl.py b/tests/benchmarks/test_crawl.py similarity index 82% rename from tests/benchmarks/test_benchmark_crawl.py rename to tests/benchmarks/test_crawl.py index 4ad0e672b..0fdfe742b 100644 --- a/tests/benchmarks/test_benchmark_crawl.py +++ b/tests/benchmarks/test_crawl.py @@ -50,8 +50,13 @@ class _Pipeline: return item -def test_benchmark_crawl(benchmark: BenchmarkFixture, mockserver: MockServer) -> None: - """Crawl of a set of interlinked pages served over HTTP.""" +def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None: + """Per-request overhead of a crawl over HTTP. + + The pages are small on purpose, so that the cost of parsing them stays + negligible next to the cost of moving requests and responses through the + engine, the middlewares and the download handler. + """ query = urlencode({"total": PAGES, "show": LINKS_PER_PAGE, "order": "desc"}) url = mockserver.url(f"/follow?{query}") settings = {"ITEM_PIPELINES": {_Pipeline: 100}, "LOG_ENABLED": False} From 1f03fbc17e1bf7ecdd7b7df06d7e3cf8c4f654f7 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 15:59:38 +0200 Subject: [PATCH 03/13] Add scrapy.utils.asyncio.sleep() (#7843) --- docs/topics/asyncio.rst | 1 + scrapy/utils/asyncio.py | 20 +++++++++++++++++++- scrapy/utils/defer.py | 11 ++--------- tests/test_crawler_subprocess.py | 5 +++-- tests/test_engine_loop.py | 11 +++++------ tests/test_utils_asyncio.py | 11 +++++++++++ tests/utils/__init__.py | 10 ---------- 7 files changed, 41 insertions(+), 28 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 63c217e93..afccb491d 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -267,6 +267,7 @@ Here are some examples of APIs and patterns that need a replacement: Scrapy provides unified helpers for some of these examples: +.. autofunction:: scrapy.utils.asyncio.sleep .. autofunction:: scrapy.utils.asyncio.call_later .. autofunction:: scrapy.utils.asyncio.create_looping_call .. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 44604c0fe..7c7697f56 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar from twisted.internet.defer import Deferred -from twisted.internet.task import LoopingCall +from twisted.internet.task import LoopingCall, deferLater from twisted.internet.threads import deferToThread from scrapy.utils.asyncgen import as_async_generator @@ -293,6 +293,24 @@ class CallLaterResult: self._delayed_call = None +async def sleep(seconds: float) -> None: + """Sleep for *seconds*. + + .. versionadded:: VERSION + + This uses either :func:`asyncio.sleep` or + :func:`~twisted.internet.task.deferLater`, depending on whether asyncio + support is available. + """ + if is_asyncio_available(): + await asyncio.sleep(seconds) + return + + from twisted.internet import reactor + + await deferLater(reactor, seconds) + + async def run_in_thread( func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs ) -> _T: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d0259b634..7c6235f29 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -27,7 +27,7 @@ from twisted.internet.task import Cooperator from twisted.python import failure from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.asyncio import is_asyncio_available, sleep from scrapy.utils.python import global_object_name if TYPE_CHECKING: @@ -90,14 +90,7 @@ async def _defer_sleep_async() -> None: """Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers before attending pending delayed calls, so do not set delay to zero. """ - if is_asyncio_available(): - await asyncio.sleep(_DEFER_DELAY) - else: - from twisted.internet import reactor - - d: Deferred[None] = Deferred() - reactor.callLater(_DEFER_DELAY, d.callback, None) - await d + await sleep(_DEFER_DELAY) def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index fe2f83161..733b6797d 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -14,7 +14,8 @@ from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn from w3lib import __version__ as w3lib_version -from tests.utils import async_sleep, get_script_run_env +from scrapy.utils.asyncio import sleep +from tests.utils import get_script_run_env from tests.utils.decorators import coroutine_test if TYPE_CHECKING: @@ -244,7 +245,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): p.kill(sig) p.expect_exact("shutting down gracefully") # sending the second signal too fast often causes problems - await async_sleep(0.01) + await sleep(0.01) p.kill(sig) p.expect_exact("forcing unclean shutdown") p.wait() # type: ignore[no-untyped-call] diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index c15c396d3..14ec3d184 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -6,10 +6,9 @@ from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals from scrapy.core.scheduler import BaseScheduler -from scrapy.utils.asyncio import call_later +from scrapy.utils.asyncio import call_later, sleep from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer -from tests.utils import async_sleep from tests.utils.decorators import coroutine_test if TYPE_CHECKING: @@ -65,23 +64,23 @@ class TestMain: async def start(self): yield Request("data:,a") - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.pause() self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b")) # During this time, the scheduler reports having requests but # returns None. - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.unpause() # The scheduler request is processed. - await async_sleep(seconds) + await sleep(seconds) yield Request("data:,c") - await async_sleep(seconds) + await sleep(seconds) self.crawler.engine._slot.scheduler.pause() self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d")) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 9b7eb22fa..4bd54acd4 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -12,7 +12,9 @@ from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.asyncio import ( AsyncioLoopingCall, _parallel_asyncio, + call_later, is_asyncio_available, + sleep, ) from tests.utils.decorators import coroutine_test @@ -26,6 +28,15 @@ async def test_is_asyncio_available(reactor_pytest: str) -> None: assert is_asyncio_available() == (reactor_pytest != "default") +@coroutine_test +async def test_sleep() -> None: + events: list[str] = [] + call_later(0.05, events.append, "call_later") + await sleep(0.1) + events.append("sleep") + assert events == ["call_later", "sleep"] + + @pytest.mark.only_asyncio class TestParallelAsyncio: """Test for scrapy.utils.asyncio.parallel_asyncio(), based on tests.test_utils_defer.TestParallelAsync.""" diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index b0632a7ea..b27c5ade7 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import os from pathlib import Path from typing import TYPE_CHECKING @@ -8,8 +7,6 @@ from typing import TYPE_CHECKING from twisted.internet.defer import Deferred from scrapy.settings import Settings, default_settings -from scrapy.utils.asyncio import is_asyncio_available -from scrapy.utils.defer import maybe_deferred_to_future if TYPE_CHECKING: from collections.abc import Callable @@ -23,13 +20,6 @@ def twisted_sleep(seconds: float): return d -async def async_sleep(seconds: float) -> None: - if is_asyncio_available(): - await asyncio.sleep(seconds) - else: - await maybe_deferred_to_future(twisted_sleep(seconds)) - - def get_script_run_env() -> dict[str, str]: """Return a OS environment dict suitable to run scripts shipped with tests.""" From 14478e3f24258ad3e9b5a3ca8a178ef126c2d4c4 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 17:19:07 +0200 Subject: [PATCH 04/13] Support CONCURRENT_REQUESTS = 0 for unlimited concurrency (#7840) --- docs/topics/settings.rst | 2 +- scrapy/core/downloader/__init__.py | 3 ++- scrapy/core/downloader/handlers/_httpx.py | 7 ++++--- tests/test_core_downloader.py | 18 ++++++++++++++++++ tests/test_downloader_handler_httpx.py | 14 ++++++++++++++ 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index b07dff180..e287c3bd5 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -575,7 +575,7 @@ CONCURRENT_REQUESTS Default: ``16`` The maximum number of concurrent (i.e. simultaneous) requests that will be -performed by the Scrapy downloader. +performed by the Scrapy downloader. Use ``0`` for no limit. .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 7c0ee0eec..eb2079d0c 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -138,7 +138,8 @@ class Downloader: self.active.remove(request) def needs_backout(self) -> bool: - return len(self.active) >= self.total_concurrency + # A total concurrency of 0 means no limit. + return 0 < self.total_concurrency <= len(self.active) @_warn_spider_arg def _get_slot( diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index d5a4e9fcd..8bbffb233 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -92,10 +92,11 @@ class HttpxDownloadHandler(_Base): self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings) self._bind_host: str | None = self._get_bind_address_host() self._limits: httpx.Limits = httpx.Limits( - # hard limit on simultaneous connections - max_connections=self._pool_size_total, + # hard limit on simultaneous connections (None for no limit, which + # is what a CONCURRENT_REQUESTS of 0 means) + max_connections=self._pool_size_total or None, # total number of idle connections in the pool (extra ones are closed) - max_keepalive_connections=self._pool_size_total, + max_keepalive_connections=self._pool_size_total or None, ) self._default_client: httpx.AsyncClient = self._make_client() diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index fdd5edc27..3e4139b3e 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -14,6 +14,7 @@ from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse +from scrapy import Request from scrapy.core.downloader import Downloader, Slot, tls from scrapy.core.downloader.contextfactory import ( _load_context_factory_from_settings, @@ -296,6 +297,23 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase): await self._assert_factory_works(server_url, client_context_factory) +@pytest.mark.parametrize( + ("concurrency", "active", "expected"), + [ + (2, 1, False), + (2, 2, True), + (0, 0, False), + (0, 2, False), + ], +) +def test_needs_backout(concurrency: int, active: int, expected: bool) -> None: + crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency}) + downloader = Downloader(crawler) + downloader.active = {Request(f"https://example.com/{i}") for i in range(active)} + assert downloader.needs_backout() is expected + downloader.close() + + @coroutine_test async def test_fetch_deprecated_spider_arg(): class CustomDownloader(Downloader): diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index 5ceb93382..976daacaf 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -15,6 +15,8 @@ from scrapy.core.downloader.handlers._httpx import ( HttpxDownloadHandler, ) from scrapy.exceptions import DownloadFailedError +from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler from tests.utils.bases.download_handlers_http import ( TestHttpBase, TestHttpProxyBase, @@ -161,3 +163,15 @@ class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase): @pytest.mark.requires_internet class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase): pass + + +@pytest.mark.parametrize(("concurrency", "expected"), [(16, 16), (0, None)]) +@coroutine_test +async def test_pool_limits(concurrency: int, expected: int | None) -> None: + crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency}) + handler = build_from_crawler(HttpxDownloadHandler, crawler) + try: + assert handler._limits.max_connections == expected + assert handler._limits.max_keepalive_connections == expected + finally: + await handler.close() From 8caaac6ecbd49ad950b7a665498ccebe9fbff052 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:05:22 -0300 Subject: [PATCH 05/13] fix(shell): Run IPython prompt in a thread under a running loop (#7816) --- pyproject.toml | 4 +- scrapy/utils/console.py | 28 ++++++----- tests/test_utils_console.py | 98 +++++++++++++++++++++++++++++++++++++ tox.ini | 4 +- 4 files changed, 119 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 576a42e5c..1cbd39946 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,8 +69,8 @@ brotli = [ gcs = ["google-cloud-storage>=1.29.0"] httpx = ["httpx2[http2,socks]>=2.0.0"] images = ["Pillow>=8.3.2"] -ipython = ["ipython>=7.1.0"] -ptpython = ["ptpython>=2.0.1"] +ipython = ["ipython>=8.15.0"] +ptpython = ["ptpython>=3.0.23"] robotparser = ["robotexclusionrulesparser>=1.6.2"] s3 = ["boto3>=1.20.0"] twisted-http2 = ["Twisted[http2]>=21.7.0"] diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 31a4bb32f..23b4401e8 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,8 +1,9 @@ from __future__ import annotations +import asyncio import code from collections.abc import Callable -from functools import wraps +from functools import partial, wraps from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -16,16 +17,8 @@ def _embed_ipython_shell( namespace: dict[str, Any] | None = None, banner: str = "" ) -> EmbedFuncT: """Start an IPython Shell""" - try: - from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415 - from IPython.terminal.ipapp import load_default_config # noqa: PLC0415 - except ImportError: - from IPython.frontend.terminal.embed import ( # type: ignore[import-not-found,no-redef] # noqa: T100,PLC0415 - InteractiveShellEmbed, - ) - from IPython.frontend.terminal.ipapp import ( # type: ignore[import-not-found,no-redef] # noqa: PLC0415 - load_default_config, - ) + from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415 + from IPython.terminal.ipapp import load_default_config # noqa: PLC0415 @wraps(_embed_ipython_shell) def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: @@ -38,6 +31,19 @@ def _embed_ipython_shell( shell = InteractiveShellEmbed.instance( banner1=banner, user_ns=namespace, config=config ) + # If an asyncio event loop is already running in this thread, e.g. when + # inspect_response() is called from a spider callback while using the + # asyncio reactor, prompt_toolkit cannot run its own event loop here, so + # ask it to run the prompt in a separate thread instead. pt_app is None + # when IPython falls back to its simple prompt, which needs no event loop. + # See https://github.com/scrapy/scrapy/issues/5447 + if (pt_app := getattr(shell, "pt_app", None)) is not None: + try: + asyncio.get_running_loop() + except RuntimeError: + pass + else: + pt_app.prompt = partial(pt_app.prompt, in_thread=True) shell() return wrapper diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index ab0c72d8a..0dea8af6d 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -1,10 +1,38 @@ from __future__ import annotations +import subprocess +import sys from importlib.util import find_spec +from io import BytesIO +from typing import TYPE_CHECKING import pytest +from pexpect import EOF from scrapy.utils.console import get_shell_embed_func, start_python_console +from scrapy.utils.test import get_testenv + +if TYPE_CHECKING: + from pathlib import Path + +CONSOLE = """ +from scrapy.utils.console import start_python_console + +start_python_console(banner="SHELL-READY", shells=["ipython"]) +""" + +CONSOLE_IN_RUNNING_LOOP = """ +import asyncio + +from scrapy.utils.console import start_python_console + + +async def main(): + start_python_console(banner="SHELL-READY", shells=["ipython"]) + + +asyncio.run(main()) +""" def test_get_shell_embed_func(): @@ -61,6 +89,76 @@ def test_get_shell_embed_func_default(): assert shell.__name__ == expected +@pytest.mark.skipif(find_spec("IPython") is None, reason="IPython is not installed") +class TestIPythonShell: + """Starting an IPython shell, with and without an asyncio event loop already + running in the calling thread. The latter happens when inspect_response() is + called from a spider callback while using the asyncio reactor.""" + + @staticmethod + def _env(tmp_path: Path) -> dict[str, str]: + env = get_testenv() + # Keep IPython away from the profile and history of the user running the tests. + env["IPYTHONDIR"] = str(tmp_path) + return env + + def test_simple_prompt(self, tmp_path: Path) -> None: + """IPython falls back to its simple prompt, which needs no event loop, + when stdin is not a TTY.""" + env = self._env(tmp_path) + p = subprocess.run( + [sys.executable, "-c", CONSOLE_IN_RUNNING_LOOP], + check=False, + capture_output=True, + encoding="utf-8", + timeout=60, + env=env, + stdin=subprocess.DEVNULL, + ) + output = p.stdout + p.stderr + assert "SHELL-READY" in output + assert p.returncode == 0, output + + @pytest.mark.skipif( + sys.platform == "win32", reason="requires a POSIX pseudo-terminal" + ) + @pytest.mark.parametrize( + "script", + [CONSOLE, CONSOLE_IN_RUNNING_LOOP], + ids=["no_running_loop", "running_loop"], + ) + def test_tty(self, tmp_path: Path, script: str) -> None: + """IPython uses prompt_toolkit, which needs an event loop of its own, + when stdin is a TTY.""" + # pexpect only defines spawn, which needs a pseudo-terminal, on POSIX. + from pexpect import spawn # noqa: PLC0415 + + env = self._env(tmp_path) + env.pop("IPY_TEST_SIMPLE_PROMPT", None) + env["TERM"] = "xterm" + logfile = BytesIO() + p = spawn( + sys.executable, + ["-c", script], + env=env, + timeout=60, + ) + p.logfile_read = logfile + try: + # Wait for the prompt, which prompt_toolkit draws once it is done + # querying the terminal, before typing into it. + p.expect(r"In \[") + p.sendline("21*2") + p.expect_exact("42") + p.sendline("exit()") + p.expect(EOF) + finally: + p.close() + output = logfile.getvalue().decode() + assert "Traceback" not in output + assert p.exitstatus == 0, output + + def test_start_python_console_exit(monkeypatch: pytest.MonkeyPatch) -> None: def embed(namespace: dict[str, object], banner: str) -> None: raise SystemExit diff --git a/tox.ini b/tox.ini index b59221340..8331e2ab6 100644 --- a/tox.ini +++ b/tox.ini @@ -190,8 +190,8 @@ deps = brotlicffi==1.2.0.0; implementation_name == "pypy" google-cloud-storage==1.29.0 httpx2[http2,socks]==2.0.0 - ipython==7.1.0 - ptpython==2.0.1 + ipython==8.15.0 + ptpython==3.0.23 robotexclusionrulesparser==1.6.2 uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy" zstandard==0.16.0; implementation_name != "pypy" From 24de06bcd3bcf20b787e33b1f65a3512a226d1d9 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 31 Jul 2026 20:02:52 +0200 Subject: [PATCH 06/13] CI: Install dependencies with uv (#7838) * CI: Install dependencies with uv * Setup GitHub Actions hardening * CI: Use uv for mitmproxy, benchmarks and cache keys * Fix mitmproxy install on PyPy --- .github/dependabot.yml | 12 +++++++++ .github/workflows/auto-close-llm-pr.yml | 6 +++-- .github/workflows/checks.yml | 32 +++++++++++++++++----- .github/workflows/codspeed.yml | 24 +++++++++++++---- .github/workflows/publish.yml | 35 +++++++++++++++++++----- .github/workflows/tests-macos.yml | 35 +++++++++++++++++++----- .github/workflows/tests-ubuntu.yml | 36 +++++++++++++++++++------ .github/workflows/tests-windows.yml | 35 +++++++++++++++++++----- .pre-commit-config.yaml | 7 ++++- docs/requirements.in | 2 +- docs/requirements.txt | 2 +- tox.ini | 3 ++- 12 files changed, 182 insertions(+), 47 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..623d48cfa --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: monthly + groups: + github-actions: + patterns: + - "*" + cooldown: + default-days: 7 diff --git a/.github/workflows/auto-close-llm-pr.yml b/.github/workflows/auto-close-llm-pr.yml index 160b39488..15120b0d9 100644 --- a/.github/workflows/auto-close-llm-pr.yml +++ b/.github/workflows/auto-close-llm-pr.yml @@ -1,5 +1,7 @@ name: Auto-close LLM PRs -on: +# The workflow only reads the pull request body through the API, it never +# checks out or runs pull request code, so pull_request_target is safe here. +on: # zizmor: ignore[dangerous-triggers] pull_request_target: types: [opened] permissions: @@ -11,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check PR body and close if LLM-written - uses: actions/github-script@v6 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ed2388a59..331fade61 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,4 +1,8 @@ name: Checks + +permissions: + contents: read + on: push: branches: @@ -13,6 +17,10 @@ concurrency: jobs: checks: runs-on: ubuntu-latest + env: + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: @@ -38,21 +46,31 @@ jobs: TOXENV: twinecheck steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + docs/requirements.txt + pyproject.toml + tox.ini + - name: Run check env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - uses: pre-commit/action@v3.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 6930c4c1c..82b16eea2 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -22,24 +22,38 @@ permissions: {} jobs: benchmark: runs-on: ubuntu-latest + env: + # Make uv use the interpreter that actions/setup-python installed + # instead of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system permissions: contents: read id-token: write # OIDC authentication with CodSpeed steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python 3.14 - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.14' + + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + - name: Install dependencies + # tox must stay on PATH for the CodSpeed action to invoke it. run: | - pip install --upgrade pip - pip install --upgrade tox + uv tool install --with tox-uv tox tox -n -e benchmark - name: Run benchmarks - uses: CodSpeedHQ/action@v4 + uses: CodSpeedHQ/action@f22792bfac16f3e14eb9fbea76f4a48e9cc22b93 # v4.19.1 with: mode: simulation run: tox -e benchmark diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7779bbb6b..697647131 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,8 @@ name: Publish + +permissions: + contents: read + on: push: tags: @@ -9,8 +13,28 @@ concurrency: cancel-in-progress: true jobs: + build: + name: Build distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + - run: | + python -m pip install --upgrade build + python -m build + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: python-package-distributions + path: dist/ + publish: name: Upload release to PyPI + needs: + - build runs-on: ubuntu-latest environment: name: pypi @@ -18,12 +42,9 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - python-version: "3.14" - - run: | - python -m pip install --upgrade build - python -m build + name: python-package-distributions + path: dist/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 566b34e50..af2a0206a 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,4 +1,8 @@ name: macOS + +permissions: + contents: read + on: push: branches: @@ -15,6 +19,9 @@ jobs: runs-on: macos-latest env: PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: @@ -31,25 +38,39 @@ jobs: TOXENV: no-reactor steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + run: uv tool install mitmproxy + - name: Run tests env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox - name: Upload coverage report if: ${{ matrix.coverage }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: report_type: test_results diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index ad2bcfce7..60a2bec21 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,4 +1,8 @@ name: Ubuntu + +permissions: + contents: read + on: push: branches: @@ -15,6 +19,9 @@ jobs: runs-on: ubuntu-latest env: PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: @@ -92,10 +99,12 @@ jobs: coverage: true steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} @@ -105,21 +114,32 @@ jobs: sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + - name: Install mitmproxy - run: pipx install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + # mitmproxy has no PyPy wheels, so run it on CPython regardless of the + # interpreter under test. + run: uv tool install --python cpython mitmproxy - name: Run tests env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox - name: Upload coverage report if: ${{ matrix.coverage }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: report_type: test_results diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index c33f96b12..884ff8e7c 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,4 +1,8 @@ name: Windows + +permissions: + contents: read + on: push: branches: @@ -15,6 +19,9 @@ jobs: runs-on: windows-latest env: PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} + # Make uv use the interpreter that actions/setup-python installed instead + # of downloading one of its own. + UV_PYTHON_PREFERENCE: only-system strategy: fail-fast: false matrix: @@ -55,25 +62,39 @@ jobs: TOXENV: extra-deps steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + - name: Set up uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + cache-dependency-glob: | + pyproject.toml + tox.ini + + - name: Install mitmproxy + env: + # mitmproxy needs a newer Python than the oldest matrix entries, so let + # uv download one where no system interpreter is new enough. + UV_PYTHON_PREFERENCE: system + run: uv tool install mitmproxy + - name: Run tests env: ${{ matrix.env }} - run: | - pip install -U tox - tox + run: uvx --with tox-uv tox - name: Upload coverage report if: ${{ matrix.coverage }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: report_type: test_results diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c27348c7a..c2cb5056d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,11 @@ repos: hooks: - id: sphinx-lint - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.8.9 + rev: 0.8.10 hooks: - id: sphinx-scrapy +- repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.28.0 + hooks: + - id: zizmor + args: [--no-progress, --fix] diff --git a/docs/requirements.in b/docs/requirements.in index 257365380..3783dd1dc 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -5,4 +5,4 @@ sphinx sphinx-notfound-page sphinx-rtd-theme sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10 diff --git a/docs/requirements.txt b/docs/requirements.txt index 87634cea9..0f5969401 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0 # via # -r docs/requirements.in # sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@912ed0507405e16ac60a47dd08195a1cd0ced984 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8 # via -r docs/requirements.in sphinx-sitemap==2.9.0 # via sphinx-scrapy diff --git a/tox.ini b/tox.ini index 8331e2ab6..edde83356 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,8 @@ [tox] requires = - sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9 + sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10 + tox-uv envlist = pre-commit pylint From 11d1712a2cdfde86be037553f18780c45d534222 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 1 Aug 2026 12:51:45 +0500 Subject: [PATCH 07/13] Force CI job names to not include "true" for "coverage". (#7848) --- .github/workflows/tests-macos.yml | 1 + .github/workflows/tests-ubuntu.yml | 1 + .github/workflows/tests-windows.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index af2a0206a..7e928cd76 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -16,6 +16,7 @@ concurrency: jobs: tests: + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) runs-on: macos-latest env: PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 60a2bec21..f929be829 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -16,6 +16,7 @@ concurrency: jobs: tests: + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) runs-on: ubuntu-latest env: PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 884ff8e7c..5d1b1d818 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -16,6 +16,7 @@ concurrency: jobs: tests: + name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }}) runs-on: windows-latest env: PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }} From a499dc9511005e026860c5ce72296ba1ff3aae1b Mon Sep 17 00:00:00 2001 From: Adrian Date: Sat, 1 Aug 2026 17:30:50 +0200 Subject: [PATCH 08/13] CI: Reduce the test job matrix while maintaining coverage (#7844) --- .github/workflows/tests-macos.yml | 20 ++++++++++---------- .github/workflows/tests-ubuntu.yml | 8 -------- .github/workflows/tests-windows.yml | 12 ------------ 3 files changed, 10 insertions(+), 30 deletions(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 7e928cd76..9a09aa67d 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -26,17 +26,17 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] - env: - - TOXENV: py include: - - python-version: '3.14' - env: - TOXENV: py - coverage: true - - python-version: '3.14' - env: - TOXENV: no-reactor + - python-version: "3.10" + env: + TOXENV: py + - python-version: "3.14" + env: + TOXENV: py + coverage: true + - python-version: "3.14" + env: + TOXENV: no-reactor steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f929be829..cd726a2fe 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -51,10 +51,6 @@ jobs: env: TOXENV: no-reactor coverage: true - # pinned due to https://github.com/pypy/pypy/issues/5388 - - python-version: pypy3.11-7.3.20 - env: - TOXENV: pypy3 # min deps - python-version: "3.10.19" @@ -65,10 +61,6 @@ jobs: env: TOXENV: min-default-reactor coverage: true - - python-version: "3.10.19" - env: - TOXENV: min-no-reactor - coverage: true # pinned due to https://github.com/pypy/pypy/issues/5388 - python-version: pypy3.11-7.3.20 env: diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 5d1b1d818..254e9395e 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -30,22 +30,10 @@ jobs: - python-version: "3.10" env: TOXENV: py - - python-version: "3.11" - env: - TOXENV: py - - python-version: "3.12" - env: - TOXENV: py - - python-version: "3.13" - env: - TOXENV: py - python-version: "3.14" env: TOXENV: py coverage: true - - python-version: "3.14" - env: - TOXENV: default-reactor - python-version: "3.14" env: TOXENV: no-reactor From e83c709574addde91240a3b6bb7eee26c4fd8b32 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 3 Aug 2026 12:44:48 +0200 Subject: [PATCH 09/13] Docs: a job directory belongs to one Scrapy version (#7861) --- docs/topics/jobs.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index dcff10772..c3043204b 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -83,6 +83,14 @@ stopping it cleanly. Forced, sudden or otherwise unclean shutdown can lead to data corruption in the job directory, which may prevent the spider from resuming correctly. +Scrapy version changes +---------------------- + +The contents of a job directory are an implementation detail of the Scrapy +version that wrote them. A job must be resumed with the same Scrapy version +that paused it; after upgrading or downgrading Scrapy, start a new job with a +new job directory. + Cookies expiration ------------------ From 2b2e18199b0bbae9dfe64a111d7fe37de7b7da9a Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 3 Aug 2026 12:49:15 +0200 Subject: [PATCH 10/13] Type downloader middleware tests (#7858) --- pyproject.toml | 12 ---- scrapy/http/request/__init__.py | 4 +- tests/test_downloadermiddleware_cookies.py | 66 +++++++++-------- tests/test_downloadermiddleware_httpauth.py | 8 ++- tests/test_downloadermiddleware_httpcache.py | 31 ++++---- ...st_downloadermiddleware_httpcompression.py | 58 +++++++++++---- tests/test_downloadermiddleware_httpproxy.py | 6 +- tests/test_downloadermiddleware_offsite.py | 20 +++--- tests/test_downloadermiddleware_redirect.py | 20 ++++-- ...wnloadermiddleware_redirect_metarefresh.py | 16 +++-- tests/test_downloadermiddleware_retry.py | 69 +++++++++--------- tests/test_downloadermiddleware_robotstxt.py | 32 ++++----- tests/test_robotstxt_interface.py | 72 ++++++++++--------- 13 files changed, 236 insertions(+), 178 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1cbd39946..11e971a35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,17 +125,6 @@ module = [ "tests.test_contracts", "tests.test_core_downloader", "tests.test_downloader_handler_twisted_ftp", - "tests.test_downloadermiddleware_cookies", - "tests.test_downloadermiddleware_httpauth", - "tests.test_downloadermiddleware_httpcache", - "tests.test_downloadermiddleware_httpcompression", - "tests.test_downloadermiddleware_httpproxy", - "tests.test_downloadermiddleware_offsite", - "tests.test_downloadermiddleware_redirect", - "tests.test_downloadermiddleware_redirect_base", - "tests.test_downloadermiddleware_redirect_metarefresh", - "tests.test_downloadermiddleware_retry", - "tests.test_downloadermiddleware_robotstxt", "tests.test_downloaderslotssettings", "tests.test_dupefilters", "tests.test_engine_loop", @@ -167,7 +156,6 @@ module = [ "tests.test_request_cb_kwargs", "tests.test_request_dict", "tests.test_request_left", - "tests.test_robotstxt_interface", "tests.test_scheduler_base", "tests.test_settings", "tests.test_spider", diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 7d67bb6d7..7c53b6b48 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -50,7 +50,9 @@ class VerboseCookie(TypedDict): secure: NotRequired[bool] -CookiesT: TypeAlias = dict[str | bytes, str | bytes] | list[VerboseCookie] +CookiesT: TypeAlias = ( + dict[str | bytes, str | bytes | bool | float | int] | list[VerboseCookie] +) RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 7ad103f27..8d999d952 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -1,5 +1,6 @@ import logging from collections.abc import Iterable +from typing import Any import pytest @@ -219,7 +220,7 @@ class TestCookiesMiddleware: def test_complex_cookies(self): # merge some cookies into jar - cookies = [ + cookies: list[VerboseCookie] = [ { "name": "C1", "value": "value1", @@ -483,13 +484,13 @@ class TestCookiesMiddleware: def _test_cookie_redirect( self, - source, - target, + source: str | dict[str, Any], + target: str | dict[str, Any], *, - cookies1, - cookies2, - ): - input_cookies = {"a": "b"} + cookies1: bool, + cookies2: bool, + ) -> None: + input_cookies: CookiesT = {"a": "b"} if not isinstance(source, dict): source = {"url": source} @@ -551,11 +552,11 @@ class TestCookiesMiddleware: def _test_cookie_header_redirect( self, - source, - target, + source: str | dict[str, Any], + target: str | dict[str, Any], *, - cookies2, - ): + cookies2: bool, + ) -> None: """Test the handling of a user-defined Cookie header when building a redirect follow-up request. @@ -623,14 +624,14 @@ class TestCookiesMiddleware: def _test_user_set_cookie_domain_followup( self, - url1, - url2, - domain, + url1: str, + url2: str, + domain: str, *, - cookies1, - cookies2, - ): - input_cookies = [ + cookies1: bool, + cookies2: bool, + ) -> None: + input_cookies: list[VerboseCookie] = [ { "name": "a", "value": "b", @@ -686,16 +687,16 @@ class TestCookiesMiddleware: def _test_server_set_cookie_domain_followup( self, - url1, - url2, - domain, + url1: str, + url2: str, + domain: str, *, - cookies, - ): + cookies: bool, + ) -> None: request1 = Request(url1) self.mw.process_request(request1) - input_cookies = [ + input_cookies: list[VerboseCookie] = [ { "name": "a", "value": "b", @@ -747,8 +748,14 @@ class TestCookiesMiddleware: ) def _test_cookie_redirect_scheme_change( - self, secure, from_scheme, to_scheme, cookies1, cookies2, cookies3 - ): + self, + secure: bool | object, + from_scheme: str, + to_scheme: str, + cookies1: bool, + cookies2: bool, + cookies3: bool, + ) -> None: """When a redirect causes the URL scheme to change from *from_scheme* to *to_scheme*, while domain and port remain the same, and given a cookie on the initial request with its secure attribute set to @@ -756,10 +763,11 @@ class TestCookiesMiddleware: initial request (*cookies1*), if it should be kept by the redirect middleware (*cookies2*), and if it should be present on the Cookie header in the redirected request (*cookie3*).""" - cookie_kwargs = {} + cookie: VerboseCookie = {"name": "a", "value": "b"} if secure is not UNSET: - cookie_kwargs["secure"] = secure - input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}] + assert isinstance(secure, bool) + cookie["secure"] = secure + input_cookies = [cookie] request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies) self.mw.process_request(request1) diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index 827133d2e..dd5af3bc5 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -1,3 +1,5 @@ +from typing import Any + import pytest from w3lib.http import basic_auth_header @@ -10,8 +12,10 @@ from scrapy.utils.test import get_crawler _DOMAIN_NOT_SET = object() -def make_mw(user="", passwd="", domain=_DOMAIN_NOT_SET): - settings: dict = { +def make_mw( + user: str = "", passwd: str = "", domain: str | object = _DOMAIN_NOT_SET +) -> HttpAuthMiddleware: + settings: dict[str, Any] = { "HTTPAUTH_USER": user, "HTTPAUTH_PASS": passwd, } diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 6e8486eb8..dc8228470 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -94,14 +94,14 @@ class TestBase: finally: mw.spider_closed(crawler.spider) - def assertEqualResponse(self, response1, response2): + def assertEqualResponse(self, response1: Response, response2: Response) -> None: assert response1.url == response2.url assert response1.status == response2.status assert response1.headers == response2.headers assert response1.body == response2.body -class StorageTestMixin: +class StorageTestMixin(TestBase): """Mixin containing storage-specific test methods.""" def _corrupt_cache_entry( @@ -135,6 +135,8 @@ class StorageTestMixin: def test_corrupted_cache_entry_is_a_miss(self, caplog): with self._middleware() as mw: spider = mw.crawler.spider + assert spider + assert mw.crawler.stats mw.storage.store_response(spider, self.request, self.response) self._corrupt_cache_entry(mw.storage, spider, self.request) @@ -155,6 +157,8 @@ class StorageTestMixin: def test_corrupted_cache_entry_ignore_missing(self): with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw: spider = mw.crawler.spider + assert spider + assert mw.crawler.stats mw.storage.store_response(spider, self.request, self.response) self._corrupt_cache_entry(mw.storage, spider, self.request) @@ -180,7 +184,7 @@ class StorageTestMixin: self.assertEqualResponse(response, cached_response) -class PolicyTestMixin: +class PolicyTestMixin(TestBase): """Mixin containing policy-specific test methods.""" def test_dont_cache(self): @@ -302,6 +306,7 @@ class DummyPolicyTestMixin(PolicyTestMixin): assert mw.process_request(self.request) is None fresh_response = self.response.replace(body=b"new body") response = mw.process_response(self.request, fresh_response) + assert isinstance(response, Response) self.assertEqualResponse(self.response, response) assert "cached" in response.flags assert mw.stats.get_value("httpcache/revalidate") == 1 @@ -313,12 +318,12 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): @staticmethod def _process_requestresponse( mw: HttpCacheMiddleware, request: Request, response: Response | None - ) -> Response | Request: - result = None + ) -> Response: + result: Request | Response | None = None try: result = mw.process_request(request) if result: - assert isinstance(result, (Request, Response)) + assert isinstance(result, Response) return result assert response is not None result = mw.process_response(request, response) @@ -346,6 +351,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): res2 = self._process_requestresponse(mw, req0, res0) assert "cached" not in res2.flags res3 = mw.process_request(req0) + assert isinstance(res3, Response) assert "cached" in res3.flags self.assertEqualResponse(res2, res3) # request with no-cache directive must not return cached response @@ -634,6 +640,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin): assert mw.process_request(req0) is None res1 = mw.process_exception(req0, e("foo")) # Use cached response as recovery + assert isinstance(res1, Response) assert "cached" in res1.flags self.assertEqualResponse(res0, res1) # Do not use cached response for unhandled exceptions @@ -684,26 +691,22 @@ class DbmStorageTestMixin(StorageTestMixin): class TestFilesystemStorageWithDummyPolicy( - TestBase, FilesystemStorageTestMixin, DummyPolicyTestMixin + FilesystemStorageTestMixin, DummyPolicyTestMixin ): policy_class = "scrapy.extensions.httpcache.DummyPolicy" class TestFilesystemStorageWithRFC2616Policy( - TestBase, FilesystemStorageTestMixin, RFC2616PolicyTestMixin + FilesystemStorageTestMixin, RFC2616PolicyTestMixin ): policy_class = "scrapy.extensions.httpcache.RFC2616Policy" -class TestDbmStorageWithDummyPolicy( - TestBase, DbmStorageTestMixin, DummyPolicyTestMixin -): +class TestDbmStorageWithDummyPolicy(DbmStorageTestMixin, DummyPolicyTestMixin): policy_class = "scrapy.extensions.httpcache.DummyPolicy" -class TestDbmStorageWithRFC2616Policy( - TestBase, DbmStorageTestMixin, RFC2616PolicyTestMixin -): +class TestDbmStorageWithRFC2616Policy(DbmStorageTestMixin, RFC2616PolicyTestMixin): policy_class = "scrapy.extensions.httpcache.RFC2616Policy" diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 55f06b396..fa0707491 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -3,6 +3,7 @@ from importlib.util import find_spec from io import BytesIO from logging import WARNING from pathlib import Path +from typing import Any import pytest from w3lib.encoding import resolve_encoding @@ -15,6 +16,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWar from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider +from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler from tests import tests_datadir @@ -72,6 +74,7 @@ class TestHttpCompression: def setup_method(self): self.crawler = get_crawler(Spider) self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) + assert self.crawler.stats self.crawler.stats.open_spider() def _getresponse(self, coding: str) -> Response: @@ -96,7 +99,8 @@ class TestHttpCompression: ) return response - def assertStatsEqual(self, key, value): + def assertStatsEqual(self, key: str, value: Any) -> None: + assert self.crawler.stats assert self.crawler.stats.get_value(key) == value, str( self.crawler.stats.get_stats() ) @@ -145,6 +149,7 @@ class TestHttpCompression: def test_process_response_gzip(self): response = self._getresponse("gzip") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"gzip" @@ -159,6 +164,7 @@ class TestHttpCompression: _skip_if_no_br() response = self._getresponse("br") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"br" newresponse = self.mw.process_response(request, response) @@ -172,6 +178,7 @@ class TestHttpCompression: if find_spec("brotli") is not None or find_spec("brotlicffi") is not None: pytest.skip("Requires not having brotli support") response = self._getresponse("br") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"br" caplog.clear() @@ -201,6 +208,7 @@ class TestHttpCompression: if not check_key.startswith("zstd-"): continue response = self._getresponse(check_key) + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"zstd" newresponse = self.mw.process_response(request, response) @@ -216,6 +224,7 @@ class TestHttpCompression: if find_spec("zstandard") is not None: pytest.skip("Requires not having zstandard support") response = self._getresponse("zstd-static-content-size") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"zstd" caplog.clear() @@ -239,6 +248,7 @@ class TestHttpCompression: def test_process_response_rawdeflate(self): response = self._getresponse("rawdeflate") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"deflate" @@ -251,6 +261,7 @@ class TestHttpCompression: def test_process_response_zlibdelate(self): response = self._getresponse("zlibdeflate") + assert response.request request = response.request assert response.headers["Content-Encoding"] == b"deflate" @@ -275,6 +286,7 @@ class TestHttpCompression: def test_multipleencodings(self): response = self._getresponse("gzip") response.headers["Content-Encoding"] = ["uuencode", "gzip"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -282,6 +294,7 @@ class TestHttpCompression: def test_multi_compression_single_header(self): response = self._getresponse("gzip-deflate") + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -293,6 +306,7 @@ class TestHttpCompression: ) -> None: response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = [b"gzip, foo, deflate"] + assert response.request request = response.request caplog.clear() with caplog.at_level( @@ -315,6 +329,7 @@ class TestHttpCompression: def test_multi_compression_multiple_header(self): response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = ["gzip", "deflate"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -324,6 +339,7 @@ class TestHttpCompression: def test_multi_compression_multiple_header_invalid_compression(self): response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -332,6 +348,7 @@ class TestHttpCompression: def test_multi_compression_single_and_multiple_header(self): response = self._getresponse("gzip-deflate-gzip") response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -341,6 +358,7 @@ class TestHttpCompression: def test_multi_compression_single_and_multiple_header_invalid_compression(self): response = self._getresponse("gzip-deflate") response.headers["Content-Encoding"] = ["gzip", "foo,deflate"] + assert response.request request = response.request newresponse = self.mw.process_response(request, response) assert newresponse is not response @@ -397,9 +415,7 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_bytes", len(plainbody)) def test_process_response_no_content_type_header(self): - headers = { - "Content-Encoding": "identity", - } + headers = {b"Content-Encoding": b"identity"} plainbody = ( b"Some page" b'' @@ -414,6 +430,7 @@ class TestHttpCompression: newresponse = self.mw.process_response(request, response) assert isinstance(newresponse, respcls) + assert isinstance(newresponse, HtmlResponse) assert newresponse.body == plainbody assert newresponse.encoding == resolve_encoding("gb2312") self.assertStatsEqual("httpcompression/response_count", 1) @@ -422,6 +439,7 @@ class TestHttpCompression: def test_process_response_gzipped_contenttype(self): response = self._getresponse("gzip") response.headers["Content-Type"] = "application/gzip" + assert response.request request = response.request newresponse = self.mw.process_response(request, response) @@ -434,6 +452,7 @@ class TestHttpCompression: def test_process_response_gzip_app_octetstream_contenttype(self): response = self._getresponse("gzip") response.headers["Content-Type"] = "application/octet-stream" + assert response.request request = response.request newresponse = self.mw.process_response(request, response) @@ -446,6 +465,7 @@ class TestHttpCompression: def test_process_response_gzip_binary_octetstream_contenttype(self): response = self._getresponse("x-gzip") response.headers["Content-Type"] = "binary/octet-stream" + assert response.request request = response.request newresponse = self.mw.process_response(request, response) @@ -504,6 +524,7 @@ class TestHttpCompression: def test_process_response_head_request_no_decode_required(self): response = self._getresponse("gzip") response.headers["Content-Type"] = "application/gzip" + assert response.request request = response.request request.method = "HEAD" response = response.replace(body=None) @@ -513,7 +534,7 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_bytes", None) - def _test_compression_bomb_setting(self, compression_id): + def _test_compression_bomb_setting(self, compression_id: str) -> None: settings = {"DOWNLOAD_MAXSIZE": 1_000_000} crawler = get_crawler(Spider, settings_dict=settings) spider = crawler._create_spider("scrapytest.org") @@ -521,9 +542,12 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B + assert response.request with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response) - assert exc_info.value.__cause__.decompressed_size < 1_100_000 + cause = exc_info.value.__cause__ + assert isinstance(cause, _DecompressionMaxSizeExceeded) + assert cause.decompressed_size < 1_100_000 def test_compression_bomb_setting_br(self): _skip_if_no_br() @@ -549,6 +573,7 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse("bomb-gzip") # 11_511_612 B + assert response.request caplog.clear() with ( caplog.at_level( @@ -565,7 +590,7 @@ class TestHttpCompression: ) ] - def _test_compression_bomb_spider_attr(self, compression_id): + def _test_compression_bomb_spider_attr(self, compression_id: str) -> None: class DownloadMaxSizeSpider(Spider): download_maxsize = 1_000_000 @@ -575,9 +600,12 @@ class TestHttpCompression: mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") + assert response.request with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response) - assert exc_info.value.__cause__.decompressed_size < 1_100_000 + cause = exc_info.value.__cause__ + assert isinstance(cause, _DecompressionMaxSizeExceeded) + assert cause.decompressed_size < 1_100_000 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_compression_bomb_spider_attr_br(self): @@ -599,7 +627,7 @@ class TestHttpCompression: self._test_compression_bomb_spider_attr("zstd") - def _test_compression_bomb_request_meta(self, compression_id): + def _test_compression_bomb_request_meta(self, compression_id: str) -> None: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") mw = HttpCompressionMiddleware.from_crawler(crawler) @@ -607,9 +635,12 @@ class TestHttpCompression: response = self._getresponse(f"bomb-{compression_id}") response.meta["download_maxsize"] = 1_000_000 + assert response.request with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response) - assert exc_info.value.__cause__.decompressed_size < 1_100_000 + cause = exc_info.value.__cause__ + assert isinstance(cause, _DecompressionMaxSizeExceeded) + assert cause.decompressed_size < 1_100_000 def test_compression_bomb_request_meta_br(self): _skip_if_no_br() @@ -789,7 +820,7 @@ class TestHttpCompression: self._test_download_warnsize_request_meta(caplog, "zstd") - def _get_truncated_response(self, compression_id): + def _get_truncated_response(self, compression_id: str) -> Response: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") mw = HttpCompressionMiddleware.from_crawler(crawler) @@ -797,7 +828,10 @@ class TestHttpCompression: response = self._getresponse(compression_id) truncated_body = response.body[: len(response.body) // 2] response = response.replace(body=truncated_body) - return mw.process_response(response.request, response) + assert response.request + new_response = mw.process_response(response.request, response) + assert isinstance(new_response, Response) + return new_response def test_process_truncated_response_br(self): _skip_if_no_br() diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 7ed848764..54d4601a7 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -14,7 +14,8 @@ class TestHttpProxyMiddleware: self._oldenv = os.environ.copy() def teardown_method(self): - os.environ = self._oldenv + os.environ.clear() + os.environ.update(self._oldenv) def test_not_enabled(self): crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False}) @@ -22,7 +23,8 @@ class TestHttpProxyMiddleware: HttpProxyMiddleware.from_crawler(crawler) def test_no_environment_proxies(self): - os.environ = {"dummy_proxy": "reset_env_and_do_not_raise"} + os.environ.clear() + os.environ["dummy_proxy"] = "reset_env_and_do_not_raise" mw = HttpProxyMiddleware() for url in ("http://e.com", "https://e.com", "file:///tmp/a"): diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index 78efb0191..cb17c2553 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -1,4 +1,5 @@ import re +from typing import Any import pytest @@ -53,7 +54,7 @@ def test_process_request_dont_filter(value, filtered): crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["dont_filter"] = value request = Request("https://b.example", **kwargs) @@ -82,7 +83,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) - kwargs = {"meta": {}} + kwargs: dict[str, Any] = {"meta": {}} if allow_offsite is not UNSET: kwargs["meta"]["allow_offsite"] = allow_offsite if dont_filter is not UNSET: @@ -105,7 +106,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): ) def test_process_request_no_allowed_domains(value): crawler = get_crawler(Spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["allowed_domains"] = value crawler.spider = crawler._create_spider(name="a", **kwargs) @@ -152,7 +153,7 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): mw.spider_opened(crawler.spider) request = Request(url) if allowed: - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) else: with pytest.raises(IgnoreRequest): mw.request_scheduled(request, crawler.spider) @@ -172,7 +173,7 @@ def test_request_scheduled_dont_filter(value, filtered): crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["dont_filter"] = value request = Request("https://b.example", **kwargs) @@ -180,7 +181,7 @@ def test_request_scheduled_dont_filter(value, filtered): with pytest.raises(IgnoreRequest): mw.request_scheduled(request, crawler.spider) else: - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) @pytest.mark.parametrize( @@ -193,14 +194,14 @@ def test_request_scheduled_dont_filter(value, filtered): ) def test_request_scheduled_no_allowed_domains(value): crawler = get_crawler(Spider) - kwargs = {} + kwargs: dict[str, Any] = {} if value is not UNSET: kwargs["allowed_domains"] = value crawler.spider = crawler._create_spider(name="a", **kwargs) mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) request = Request("https://example.com") - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) def test_request_scheduled_invalid_domains(): @@ -210,7 +211,7 @@ def test_request_scheduled_invalid_domains(): mw = OffsiteMiddleware.from_crawler(crawler) mw.spider_opened(crawler.spider) request = Request("https://a.example") - assert mw.request_scheduled(request, crawler.spider) is None + mw.request_scheduled(request, crawler.spider) for letter in ("b", "c"): request = Request(f"https://{letter}.example") with pytest.raises(IgnoreRequest): @@ -227,6 +228,7 @@ def test_repeated_offsite_domain(): with pytest.raises(IgnoreRequest): mw.process_request(req1) assert "other.org" in mw.domains_seen + assert crawler.stats assert crawler.stats.get_value("offsite/domains") == 1 assert crawler.stats.get_value("offsite/filtered") == 1 with pytest.raises(IgnoreRequest): diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index ef2774a93..de97aaadb 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -309,7 +309,7 @@ class TestRedirectMiddleware(TestRedirectBase): url = "http://www.example.com/301" url2 = "http://www.example.com/redirected" - def _test_passthrough(req): + def _test_passthrough(req: Request) -> None: rsp = Response(url, headers={"Location": url2}, status=301, request=req) r = self.mw.process_response(req, rsp) assert r is rsp @@ -404,15 +404,17 @@ def test_response_referrer_policy(policy, source_url, target_url, expected_refer status=301, headers={"Location": target_url, **extra_headers}, ) - source_request = redirect_mw.process_response(source_request, response_redirect) - assert isinstance(source_request, Request) + target_request = redirect_mw.process_response(source_request, response_redirect) + assert isinstance(target_request, Request) - assert source_request.headers.get("Referer") == expected_referrer + assert target_request.headers.get("Referer") == expected_referrer def test_no_warning_when_referer_middleware_present(caplog): crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=MagicMock()) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=MagicMock() + ) mw = build_from_crawler(RedirectMiddleware, crawler) caplog.clear() with caplog.at_level(logging.WARNING): @@ -426,7 +428,9 @@ def test_no_warning_when_referer_middleware_present(caplog): def test_warning_redirect_middleware(caplog): crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=None) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=None + ) mw = build_from_crawler(RedirectMiddleware, crawler) with caplog.at_level(logging.WARNING): mw._engine_started() @@ -449,7 +453,9 @@ def test_warning_subclass(caplog): pass crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=None) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=None + ) mw = build_from_crawler(MyRedirectMiddleware, crawler) with caplog.at_level(logging.WARNING): mw._engine_started() diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py index aeae759a0..83dc6825f 100644 --- a/tests/test_downloadermiddleware_redirect_metarefresh.py +++ b/tests/test_downloadermiddleware_redirect_metarefresh.py @@ -21,7 +21,7 @@ from tests.utils.redirect import ( ) -def meta_refresh_body(url, interval=5): +def meta_refresh_body(url: str, interval: int = 5) -> bytes: html = f"""""" return html.encode("utf-8") @@ -34,10 +34,14 @@ class TestMetaRefreshMiddleware(TestRedirectBase): crawler = get_crawler(Spider) self.mw = self.mwcls.from_crawler(crawler) - def _body(self, interval=5, url="http://example.org/newpage"): + def _body( + self, interval: int = 5, url: str = "http://example.org/newpage" + ) -> bytes: return meta_refresh_body(url, interval) - def get_response(self, request, location): + def get_response( + self, request: Request, location: str, status: int = 302 + ) -> Response: return HtmlResponse(request.url, body=self._body(url=location)) def test_meta_refresh(self): @@ -75,7 +79,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase): assert "Content-Length" not in req2.headers, ( "Content-Length header must not be present in redirected request" ) - assert not req2.body, f"Redirected body must be empty, not '{req2.body}'" + assert not req2.body, f"Redirected body must be empty, not {req2.body!r}" def test_ignore_tags_default(self): req = Request(url="http://example.org") @@ -142,7 +146,9 @@ def test_meta_refresh_schemes(url, location, target): def test_warning_meta_refresh_middleware(caplog): crawler = get_crawler() - crawler.get_spider_middleware = MagicMock(return_value=None) + crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign] + return_value=None + ) mw = build_from_crawler(MetaRefreshMiddleware, crawler) with caplog.at_level(logging.WARNING): mw._engine_started() diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 410427b84..ab52590c7 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -31,6 +31,7 @@ class TestRetry: req = Request("http://www.scrapytest.org/503") rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) assert req2.priority < req.priority def test_404(self): @@ -53,9 +54,9 @@ class TestRetry: rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 1 + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.meta["retry_times"] == 1 def test_dont_retry_exc(self): req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True}) @@ -68,18 +69,19 @@ class TestRetry: rsp = Response("http://www.scrapytest.org/503", body=b"", status=503) # first retry - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 1 + req2 = self.mw.process_response(req, rsp) + assert isinstance(req2, Request) + assert req2.meta["retry_times"] == 1 # second retry - req = self.mw.process_response(req, rsp) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 2 + req3 = self.mw.process_response(req2, rsp) + assert isinstance(req3, Request) + assert req3.meta["retry_times"] == 2 # discard it - assert self.mw.process_response(req, rsp) is rsp + assert self.mw.process_response(req3, rsp) is rsp + assert self.crawler.stats assert self.crawler.stats.get_value("retry/max_reached") == 1 assert ( self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable") @@ -131,6 +133,7 @@ class TestRetry: self._test_retry_exception(req, exc("foo")) stats = self.crawler.stats + assert stats assert stats.get_value("retry/max_reached") == len(exceptions) assert stats.get_value("retry/count") == len(exceptions) * 2 assert ( @@ -149,29 +152,30 @@ class TestRetry: req = Request(f"http://www.scrapytest.org/{exc.__name__}") self._test_retry_exception(req, exc("foo"), mw) - def _test_retry_exception(self, req, exception, mw=None): + def _test_retry_exception( + self, req: Request, exception: Exception, mw: RetryMiddleware | None = None + ) -> None: if mw is None: mw = self.mw # first retry - req = mw.process_exception(req, exception) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 1 + req2 = mw.process_exception(req, exception) + assert isinstance(req2, Request) + assert req2.meta["retry_times"] == 1 # second retry - req = mw.process_exception(req, exception) - assert isinstance(req, Request) - assert req.meta["retry_times"] == 2 + req3 = mw.process_exception(req2, exception) + assert isinstance(req3, Request) + assert req3.meta["retry_times"] == 2 # discard it - req = mw.process_exception(req, exception) - assert req is None + assert mw.process_exception(req3, exception) is None class TestMaxRetryTimes: invalid_url = "http://www.scrapytest.org/invalid_url" - def get_middleware(self, settings=None): + def get_middleware(self, settings: dict[str, Any] | None = None) -> RetryMiddleware: crawler = get_crawler(DefaultSpider, settings or {}) crawler.spider = crawler._create_spider() return RetryMiddleware.from_crawler(crawler) @@ -275,20 +279,18 @@ class TestMaxRetryTimes: def _test_retry( self, - req, - exception, - max_retry_times, - middleware=None, - ): - middleware = middleware or self.mw - + req: Request, + exception: Exception, + max_retry_times: int, + middleware: RetryMiddleware, + ) -> None: for _ in range(max_retry_times): - req = middleware.process_exception(req, exception) - assert isinstance(req, Request) + result = middleware.process_exception(req, exception) + assert isinstance(result, Request) + req = result # discard it - req = middleware.process_exception(req, exception) - assert req is None + assert middleware.process_exception(req, exception) is None class TestGetRetryRequest: @@ -428,7 +430,7 @@ class TestGetRetryRequest: def test_no_spider(self): request = Request("https://example.com") with pytest.raises(TypeError): - get_retry_request(request) # pylint: disable=missing-kwoa + get_retry_request(request) # type: ignore[call-arg] # pylint: disable=missing-kwoa def test_max_retry_times_setting(self): max_retry_times = 0 @@ -471,6 +473,7 @@ class TestGetRetryRequest: request, spider=spider, ) + assert new_request assert new_request.priority == priority_adjust def test_priority_adjust_argument(self): @@ -482,6 +485,7 @@ class TestGetRetryRequest: spider=spider, priority_adjust=priority_adjust, ) + assert new_request assert new_request.priority == priority_adjust def test_log_extra_retry_success(self, caplog: pytest.LogCaptureFixture) -> None: @@ -732,6 +736,7 @@ class TestGetRetryRequest: reason=expected_reason, stats_base_key=stats_key, ) + assert spider.crawler.stats for stat in ( f"{stats_key}/count", f"{stats_key}/reason_count/{expected_reason}", diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 1f2575f8f..793a2b5be 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio -from typing import TYPE_CHECKING from unittest import mock import pytest @@ -19,9 +18,6 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from tests.utils.decorators import coroutine_test from tests.utils.robotstxt import rerp_available -if TYPE_CHECKING: - from scrapy.crawler import Crawler - class TestRobotsTxtMiddleware: def setup_method(self) -> None: @@ -39,7 +35,7 @@ class TestRobotsTxtMiddleware: with pytest.raises(NotConfigured): RobotsTxtMiddleware(self.crawler) - def _get_successful_crawler(self) -> Crawler: + def _get_successful_crawler(self) -> mock.MagicMock: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) ROBOTS = """ @@ -54,8 +50,8 @@ Disallow: /some/randome/page.html """.encode() response = TextResponse("http://site.local/robots.txt", body=ROBOTS) - async def return_response(request): - deferred = Deferred() + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.callback, response) return await maybe_deferred_to_future(deferred) @@ -130,15 +126,15 @@ Disallow: /some/randome/page.html Request("http://site.local/static/", meta=meta), middleware ) - def _get_garbage_crawler(self) -> Crawler: + def _get_garbage_crawler(self) -> mock.MagicMock: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response( "http://site.local/robots.txt", body=b"GIF89a\xd3\x00\xfe\x00\xa2" ) - async def return_response(request): - deferred = Deferred() + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.callback, response) return await maybe_deferred_to_future(deferred) @@ -154,13 +150,13 @@ Disallow: /some/randome/page.html await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) await self.assertNotIgnored(Request("http://site.local/static/"), middleware) - def _get_emptybody_crawler(self) -> Crawler: + def _get_emptybody_crawler(self) -> mock.MagicMock: crawler = self.crawler crawler.settings.set("ROBOTSTXT_OBEY", True) response = Response("http://site.local/robots.txt") - async def return_response(request): - deferred = Deferred() + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.callback, response) return await maybe_deferred_to_future(deferred) @@ -180,8 +176,8 @@ Disallow: /some/randome/page.html self.crawler.settings.set("ROBOTSTXT_OBEY", True) err = CannotResolveHostError("Robotstxt address not found") - async def return_failure(request): - deferred = Deferred() + async def return_failure(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.errback, failure.Failure(err)) return await maybe_deferred_to_future(deferred) @@ -208,8 +204,8 @@ Disallow: /some/randome/page.html async def test_ignore_robotstxt_request(self): self.crawler.settings.set("ROBOTSTXT_OBEY", True) - async def ignore_request(request): - deferred = Deferred() + async def ignore_request(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() call_later(0, deferred.errback, failure.Failure(IgnoreRequest())) return await maybe_deferred_to_future(deferred) @@ -236,7 +232,7 @@ Disallow: /some/randome/page.html @coroutine_test async def test_robotstxt_local_file(self): middleware = RobotsTxtMiddleware(self._get_emptybody_crawler()) - middleware.process_request_2 = mock.MagicMock() + middleware.process_request_2 = mock.MagicMock() # type: ignore[method-assign] await middleware.process_request(Request("data:text/plain,Hello World data")) assert not middleware.process_request_2.called diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index ea67877f8..755f29959 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -1,3 +1,7 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import pytest from scrapy.robotstxt import ( @@ -10,22 +14,32 @@ from scrapy.robotstxt import ( from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER from tests.utils.robotstxt import rerp_available +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + class BaseRobotParserTest: - def _setUp(self, parser_cls): + parser_cls: type[RobotParser] + + def _setUp(self, parser_cls: type[RobotParser]) -> None: self.parser_cls = parser_cls + def _parse(self, robotstxt_body: bytes) -> RobotParser: + # The parser backends only use the crawler to get the spider to log with. + return self.parser_cls.from_crawler(None, robotstxt_body) # type: ignore[arg-type] + def test_allowed(self): robotstxt_robotstxt_body = ( b"User-agent: * \nDisallow: /disallowed \nAllow: /allowed \nCrawl-delay: 10" ) - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://www.site.local/allowed", "*") assert not rp.allowed("https://www.site.local/disallowed", "*") - def test_allowed_wildcards(self): + def test_allowed_wildcards(self) -> None: robotstxt_robotstxt_body = b"""User-agent: first Disallow: /disallowed/*/end$ @@ -33,9 +47,7 @@ class BaseRobotParserTest: Allow: /*allowed Disallow: / """ - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://www.site.local/disallowed", "first") assert not rp.allowed("https://www.site.local/disallowed/xyz/end", "first") @@ -46,23 +58,19 @@ class BaseRobotParserTest: assert rp.allowed("https://www.site.local/is_still_allowed", "second") assert rp.allowed("https://www.site.local/is_allowed_too", "second") - def test_length_based_precedence(self): + def test_length_based_precedence(self) -> None: robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://www.site.local/page", "*") - def test_order_based_precedence(self): + def test_order_based_precedence(self) -> None: robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page" - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert not rp.allowed("https://www.site.local/page", "*") def test_empty_response(self): """empty response should equal 'allow all'""" - rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b"") + rp = self._parse(b"") assert rp.allowed("https://site.local/", "*") assert rp.allowed("https://site.local/", "chrome") assert rp.allowed("https://site.local/index.html", "*") @@ -71,9 +79,7 @@ class BaseRobotParserTest: def test_garbage_response(self): """garbage response should be discarded, equal 'allow all'""" robotstxt_robotstxt_body = b"GIF89a\xd3\x00\xfe\x00\xa2" - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://site.local/", "*") assert rp.allowed("https://site.local/", "chrome") assert rp.allowed("https://site.local/index.html", "*") @@ -81,12 +87,12 @@ class BaseRobotParserTest: def test_crawl_delay(self): robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n" - rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body) + rp = self._parse(robotstxt_body) assert rp.crawl_delay("*") == 10.0 def test_crawl_delay_unset(self): robotstxt_body = b"User-agent: *\nDisallow: /private\n" - rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body) + rp = self._parse(robotstxt_body) assert rp.crawl_delay("*") is None def test_unicode_url_and_useragent(self): @@ -100,9 +106,7 @@ class BaseRobotParserTest: User-Agent: UnicödeBöt Disallow: /some/randome/page.html""".encode() - rp = self.parser_cls.from_crawler( - crawler=None, robotstxt_body=robotstxt_robotstxt_body - ) + rp = self._parse(robotstxt_robotstxt_body) assert rp.allowed("https://site.local/", "*") assert not rp.allowed("https://site.local/admin/", "*") assert not rp.allowed("https://site.local/static/", "*") @@ -117,15 +121,13 @@ class TestRobotParser: def test_crawl_delay_unsupported(self): class AllowAllRobotParser(RobotParser): @classmethod - def from_crawler(cls, crawler, robotstxt_body): + def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self: return cls() - def allowed(self, url, user_agent): + def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool: return True - rp = AllowAllRobotParser.from_crawler( - crawler=None, robotstxt_body=b"User-agent: *\nCrawl-delay: 10\n" - ) + rp = AllowAllRobotParser() assert rp.crawl_delay("*") is None @@ -162,21 +164,21 @@ class TestPythonRobotParser(BaseRobotParserTest): not STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support length based directives precedence.", ) - def test_length_based_precedence(self): + def test_length_based_precedence(self) -> None: super().test_length_based_precedence() @pytest.mark.skipif( STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support order based directives precedence.", ) - def test_order_based_precedence(self): + def test_order_based_precedence(self) -> None: super().test_order_based_precedence() @pytest.mark.skipif( not STDLIB_IMPROVED_ROBOTFILEPARSER, reason="RobotFileParser from this Python version does not support wildcards.", ) - def test_allowed_wildcards(self): + def test_allowed_wildcards(self) -> None: super().test_allowed_wildcards() @@ -185,7 +187,7 @@ class TestRerpRobotParser(BaseRobotParserTest): def setup_method(self): super()._setUp(RerpRobotParser) - def test_length_based_precedence(self): + def test_length_based_precedence(self) -> None: pytest.skip("Rerp does not support length based directives precedence.") @@ -193,5 +195,5 @@ class TestProtegoRobotParser(BaseRobotParserTest): def setup_method(self): super()._setUp(ProtegoRobotParser) - def test_order_based_precedence(self): + def test_order_based_precedence(self) -> None: pytest.skip("Protego does not support order based directives precedence.") From a9f177030685281550481888a33e45cb0fc1c9ee Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 3 Aug 2026 13:03:28 +0200 Subject: [PATCH 11/13] Docs: sort and compact the component-settings list (#7862) --- docs/topics/components.rst | 47 +++++++++++++------------------------- 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/docs/topics/components.rst b/docs/topics/components.rst index c0df86922..354375577 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -9,37 +9,22 @@ A Scrapy component is any class whose objects are built using That includes the classes that you may assign to the following settings: -- :setting:`ADDONS` - -- :setting:`TWISTED_DNS_RESOLVER` - -- :setting:`DOWNLOAD_HANDLERS` - -- :setting:`DOWNLOADER_MIDDLEWARES` - -- :setting:`DUPEFILTER_CLASS` - -- :setting:`EXTENSIONS` - -- :setting:`FEED_EXPORTERS` - -- :setting:`FEED_STORAGES` - -- :setting:`ITEM_PIPELINES` - -- :setting:`SCHEDULER` - -- :setting:`SCHEDULER_DISK_QUEUE` - -- :setting:`SCHEDULER_MEMORY_QUEUE` - -- :setting:`SCHEDULER_PRIORITY_QUEUE` - -- :setting:`SCHEDULER_START_DISK_QUEUE` - -- :setting:`SCHEDULER_START_MEMORY_QUEUE` - -- :setting:`SPIDER_MIDDLEWARES` +- :setting:`ADDONS` +- :setting:`DOWNLOAD_HANDLERS` +- :setting:`DOWNLOADER_MIDDLEWARES` +- :setting:`DUPEFILTER_CLASS` +- :setting:`EXTENSIONS` +- :setting:`FEED_EXPORTERS` +- :setting:`FEED_STORAGES` +- :setting:`ITEM_PIPELINES` +- :setting:`SCHEDULER` +- :setting:`SCHEDULER_DISK_QUEUE` +- :setting:`SCHEDULER_MEMORY_QUEUE` +- :setting:`SCHEDULER_PRIORITY_QUEUE` +- :setting:`SCHEDULER_START_DISK_QUEUE` +- :setting:`SCHEDULER_START_MEMORY_QUEUE` +- :setting:`SPIDER_MIDDLEWARES` +- :setting:`TWISTED_DNS_RESOLVER` Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to From cde7af87fa715ba354be315ca9159c386f1c7774 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 3 Aug 2026 13:55:39 +0200 Subject: [PATCH 12/13] Cover passing spider to a deprecated Downloader.fetch() (#7863) --- tests/test_engine_download.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_engine_download.py b/tests/test_engine_download.py index 962808d96..09b998f6a 100644 --- a/tests/test_engine_download.py +++ b/tests/test_engine_download.py @@ -116,6 +116,18 @@ class TestEngineDownloadAsync: engine._slot.add_request.assert_called_once_with(request) engine._slot.remove_request.assert_called_once_with(request) + @coroutine_test + async def test_download_async_fetch_needs_spider(self, engine): + engine._downloader_fetch_needs_spider = True + request = Request("http://example.com") + response = Response("http://example.com", body=b"test body") + engine.spider = Mock() + engine.downloader.fetch.return_value = defer.succeed(response) + + result = await self._download(engine, request) + assert result == response + engine.downloader.fetch.assert_called_once_with(request, engine.spider) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestEngineDownload(TestEngineDownloadAsync): From 298c9e610ec29dea378a706e925ad0d897007410 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 3 Aug 2026 14:16:23 +0200 Subject: [PATCH 13/13] Type tests related to requests and responses (#7864) --- pyproject.toml | 11 ---- scrapy/http/request/form.py | 8 +-- tests/test_http_headers.py | 29 +++++----- tests/test_http_request.py | 3 +- tests/test_http_request_form.py | 75 +++++++++++++++---------- tests/test_http_response_text.py | 15 +++-- tests/test_request_attribute_binding.py | 5 ++ tests/test_request_cb_kwargs.py | 26 ++++++--- tests/test_request_dict.py | 58 +++++++++++-------- tests/test_request_left.py | 43 +++++++------- tests/utils/bases/http_request.py | 59 ++++++++++--------- tests/utils/bases/http_response.py | 48 ++++++++-------- 12 files changed, 214 insertions(+), 166 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 11e971a35..609c70708 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -136,11 +136,6 @@ module = [ "tests.test_feedexport_storages", "tests.test_feedexport_uri_params", "tests.test_http2_client_protocol", - "tests.test_http_headers", - "tests.test_http_request", - "tests.test_http_request_form", - "tests.test_http_response", - "tests.test_http_response_text", "tests.test_item", "tests.test_linkextractors", "tests.test_loader", @@ -152,10 +147,6 @@ module = [ "tests.test_pipeline_media", "tests.test_pipelines", "tests.test_pqueues", - "tests.test_request_attribute_binding", - "tests.test_request_cb_kwargs", - "tests.test_request_dict", - "tests.test_request_left", "tests.test_scheduler_base", "tests.test_settings", "tests.test_spider", @@ -166,8 +157,6 @@ module = [ "tests.test_squeues", "tests.test_squeues_request", "tests.test_stats", - "tests.utils.bases.http_request", - "tests.utils.bases.http_response", "tests.utils.bases.spider", ] check_untyped_defs = false diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index f1a8dbf3b..12745292b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from warnings import warn @@ -34,7 +34,7 @@ if TYPE_CHECKING: FormdataVType: TypeAlias = str | Iterable[str] FormdataKVType: TypeAlias = tuple[str, FormdataVType] -FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None +FormdataType: TypeAlias = Mapping[str, FormdataVType] | Iterable[FormdataKVType] | None class FormRequest(Request): @@ -100,7 +100,7 @@ class FormRequest(Request): super().__init__(*args, **kwargs) if formdata: - items = formdata.items() if isinstance(formdata, dict) else formdata + items = formdata.items() if isinstance(formdata, Mapping) else formdata form_query_str = _urlencode(items, self.encoding) if self.method == "POST": self.headers.setdefault( @@ -248,7 +248,7 @@ def _get_inputs( if clickable and clickable[0] not in formdata and clickable[0] is not None: values.append(clickable) - formdata_items = formdata.items() if isinstance(formdata, dict) else formdata + formdata_items = formdata.items() if isinstance(formdata, Mapping) else formdata values.extend((k, v) for k, v in formdata_items if v is not None) return values diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index aff3562e3..e7ed17615 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -6,9 +6,6 @@ from scrapy.http import Headers class TestHeaders: - def assertSortedEqual(self, first, second, msg=None): - assert sorted(first) == sorted(second), msg - def test_basics(self): h = Headers({"Content-Type": "text/html", "Content-Length": 1234}) assert h["Content-Type"] @@ -39,7 +36,7 @@ class TestHeaders: assert h["X-Forwarded-For"] == b"ip2" assert h.get("X-Forwarded-For") == b"ip2" assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"] - assert h.getlist("X-Forwarded-For") is not hlist + assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap] def test_multivalue_for_one_header(self): h = Headers((("a", "b"), ("a", "c"))) @@ -49,19 +46,19 @@ class TestHeaders: def test_encode_utf8(self): h = Headers({"key": "\xa3"}, encoding="utf-8") - key, val = dict(h).popitem() + key, val = dict(h.items()).popitem() assert isinstance(key, bytes), key assert isinstance(val[0], bytes), val[0] assert val[0] == b"\xc2\xa3" def test_encode_latin1(self): h = Headers({"key": "\xa3"}, encoding="latin1") - _, val = dict(h).popitem() + _, val = dict(h.items()).popitem() assert val[0] == b"\xa3" def test_encode_multiple(self): h = Headers({"key": ["\xa3"]}, encoding="utf-8") - _, val = dict(h).popitem() + _, val = dict(h.items()).popitem() assert val[0] == b"\xc2\xa3" def test_delete_and_contains(self): @@ -75,7 +72,7 @@ class TestHeaders: h = Headers() hlist = ["ip1", "ip2"] olist = h.setdefault("X-Forwarded-For", hlist) - assert h.getlist("X-Forwarded-For") is not hlist + assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap] assert h.getlist("X-Forwarded-For") is olist h = Headers() @@ -87,16 +84,16 @@ class TestHeaders: idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]} h = Headers(idict) - assert dict(h) == { + assert dict(h.items()) == { b"Content-Type": [b"text/html"], b"X-Forwarded-For": [b"ip1", b"ip2"], } - self.assertSortedEqual(h.keys(), [b"X-Forwarded-For", b"Content-Type"]) - self.assertSortedEqual( - h.items(), - [(b"X-Forwarded-For", [b"ip1", b"ip2"]), (b"Content-Type", [b"text/html"])], - ) - self.assertSortedEqual(h.values(), [b"ip2", b"text/html"]) + assert sorted(h.keys()) == [b"Content-Type", b"X-Forwarded-For"] + assert sorted(h.items()) == [ + (b"Content-Type", [b"text/html"]), + (b"X-Forwarded-For", [b"ip1", b"ip2"]), + ] + assert set(h.values()) == {b"ip2", b"text/html"} def test_update(self): h = Headers() @@ -162,4 +159,4 @@ class TestHeaders: with pytest.raises(TypeError, match="Unsupported value type"): Headers().setdefault("foo", object()) with pytest.raises(TypeError, match="Unsupported value type"): - Headers().setlist("foo", [object()]) + Headers().setlist("foo", [object()]) # type: ignore[list-item] diff --git a/tests/test_http_request.py b/tests/test_http_request.py index e58ae8f39..b9cec93a1 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,4 +1,5 @@ import xmlrpc.client +from typing import Any import pytest @@ -17,7 +18,7 @@ class TestXmlRpcRequest(TestRequestBase): default_method = "POST" default_headers = {b"Content-Type": [b"text/xml"]} - def _test_request(self, **kwargs): + def _test_request(self, **kwargs: Any) -> None: r = self.request_class("http://scrapytest.org/rpc2", **kwargs) assert r.headers[b"Content-Type"] == b"text/xml" assert r.body == to_bytes( diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py index 5e965e8dc..cb18a0b03 100644 --- a/tests/test_http_request_form.py +++ b/tests/test_http_request_form.py @@ -2,6 +2,7 @@ from __future__ import annotations import re import warnings +from typing import TYPE_CHECKING, Any from urllib.parse import parse_qs, unquote_to_bytes import pytest @@ -12,20 +13,32 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode from tests.utils.bases.http_request import TestRequestBase +if TYPE_CHECKING: + from scrapy import Request -def _buildresponse(body, **kwargs): + +def _buildresponse(body: bytes | str, **kwargs: Any) -> HtmlResponse: kwargs.setdefault("body", body) kwargs.setdefault("url", "http://example.com") kwargs.setdefault("encoding", "utf-8") return HtmlResponse(**kwargs) -def _qs(req, encoding="utf-8", to_unicode=False): - qs = req.body if req.method == "POST" else req.url.partition("?")[2] - uqs = unquote_to_bytes(qs) - if to_unicode: - uqs = uqs.decode(encoding) - return parse_qs(uqs, True) +def _query_string(req: Request) -> bytes: + return req.body if req.method == "POST" else req.url.partition("?")[2].encode() + + +def _qs(req: Request) -> dict[bytes, list[bytes]]: + return parse_qs(unquote_to_bytes(_query_string(req)), True) + + +def _qs_unicode(req: Request, encoding: str = "utf-8") -> dict[str, list[str]]: + qs = unquote_to_bytes(_query_string(req)).decode(encoding) + return parse_qs(qs, True) + + +def _assert_query_equal(first: bytes, second: bytes) -> None: + assert sorted(to_unicode(first).split("&")) == sorted(to_unicode(second).split("&")) # FormRequest.from_response() is deprecated in favor of form2request, so the @@ -34,11 +47,6 @@ def _qs(req, encoding="utf-8", to_unicode=False): class TestFormRequest(TestRequestBase): request_class = FormRequest - def assertQueryEqual(self, first, second, msg=None): - first = to_unicode(first).split("&") - second = to_unicode(second).split("&") - assert sorted(first) == sorted(second), msg - def test_init_not_deprecated(self): # Building a request directly from form data is not deprecated. with warnings.catch_warnings(): @@ -75,20 +83,22 @@ class TestFormRequest(TestRequestBase): assert fs[b"b"] == [b"2"] assert fs.get(b"c") is None - data = {"a": "1", "b": "2"} + mapping = {"a": "1", "b": "2"} fs = _qs( - self.request_class("http://www.example.com/", method="GET", formdata=data) + self.request_class( + "http://www.example.com/", method="GET", formdata=mapping + ) ) assert fs[b"a"] == [b"1"] assert fs[b"b"] == [b"2"] def test_default_encoding_bytes(self): # using default encoding (utf-8) - data = {b"one": b"two", b"price": b"\xc2\xa3 100"} + data: dict[Any, Any] = {b"one": b"two", b"price": b"\xc2\xa3 100"} r2 = self.request_class("http://www.example.com", formdata=data) assert r2.method == "POST" assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two") + _assert_query_equal(r2.body, b"price=%C2%A3+100&one=two") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_default_encoding_textual_data(self): @@ -97,26 +107,26 @@ class TestFormRequest(TestRequestBase): r2 = self.request_class("http://www.example.com", formdata=data) assert r2.method == "POST" assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") + _assert_query_equal(r2.body, b"price=%C2%A3+100&%C2%B5+one=two") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_default_encoding_mixed_data(self): # using default encoding (utf-8) - data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} + data: dict[Any, Any] = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"} r2 = self.request_class("http://www.example.com", formdata=data) assert r2.method == "POST" assert r2.encoding == "utf-8" - self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") + _assert_query_equal(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_custom_encoding_bytes(self): - data = {b"\xb5 one": b"two", b"price": b"\xa3 100"} + data: dict[Any, Any] = {b"\xb5 one": b"two", b"price": b"\xa3 100"} r2 = self.request_class( "http://www.example.com", formdata=data, encoding="latin1" ) assert r2.method == "POST" assert r2.encoding == "latin1" - self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two") + _assert_query_equal(r2.body, b"price=%A3+100&%B5+one=two") assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded" def test_custom_encoding_textual_data(self): @@ -131,7 +141,7 @@ class TestFormRequest(TestRequestBase): # using multiples values for a single key data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]} r3 = self.request_class("http://www.example.com", formdata=data) - self.assertQueryEqual( + _assert_query_equal( r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100" ) @@ -173,7 +183,7 @@ class TestFormRequest(TestRequestBase): assert req.method == "POST" assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert set(fs["test £"]) == {"val1", "val2"} assert set(fs["one"]) == {"two", "three"} assert fs["test2"] == ["xxx µ"] @@ -196,7 +206,7 @@ class TestFormRequest(TestRequestBase): assert req.method == "POST" assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True, encoding="latin1") + fs = _qs_unicode(req, encoding="latin1") assert set(fs["test £"]) == {"val1", "val2"} assert set(fs["one"]) == {"two", "three"} assert fs["test2"] == ["xxx µ"] @@ -218,7 +228,7 @@ class TestFormRequest(TestRequestBase): assert req.method == "POST" assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded" assert req.url == "http://www.example.com/this/post.php" - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert set(fs["test £"]) == {"val1", "val2"} assert set(fs["one"]) == {"two", "three"} assert fs["test2"] == ["xxx µ"] @@ -305,7 +315,10 @@ class TestFormRequest(TestRequestBase): """ ) - req = self.request_class.from_response(response, formdata={"two": None}) + req = self.request_class.from_response( + response, + formdata={"two": None}, # type: ignore[arg-type] + ) fs = _qs(req) assert fs[b"one"] == [b"1"] assert b"two" not in fs @@ -450,7 +463,7 @@ class TestFormRequest(TestRequestBase): req = self.request_class.from_response( response, clickdata={"name": "price in \u00a3"} ) - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert fs["price in \u00a3"] def test_from_response_unicode_clickdata_latin1(self): @@ -466,7 +479,7 @@ class TestFormRequest(TestRequestBase): req = self.request_class.from_response( response, clickdata={"name": "price in \u00a5"} ) - fs = _qs(req, to_unicode=True, encoding="latin1") + fs = _qs_unicode(req, encoding="latin1") assert fs["price in \u00a5"] def test_from_response_multiple_forms_clickdata(self): @@ -737,7 +750,7 @@ class TestFormRequest(TestRequestBase): """ ) req = self.request_class.from_response(res) - fs = _qs(req, to_unicode=True) + fs = _qs_unicode(req) assert fs == {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]} def test_from_response_radio(self): @@ -1022,7 +1035,7 @@ class TestFormRequest(TestRequestBase): with pytest.raises( ValueError, match="formdata should be a dict or iterable of tuples" ): - FormRequest.from_response(response, formdata=123) + FormRequest.from_response(response, formdata=123) # type: ignore[arg-type] def test_form_response_with_custom_invalid_formdata_value_error(self): """Test that a ValueError is raised for fault-inducing iterable formdata input""" @@ -1037,7 +1050,7 @@ class TestFormRequest(TestRequestBase): with pytest.raises( ValueError, match="formdata should be a dict or iterable of tuples" ): - FormRequest.from_response(response, formdata=("a",)) + FormRequest.from_response(response, formdata=("a",)) # type: ignore[arg-type] def test_get_form_with_xpath_no_form_parent(self): """Test that _get_from raised a ValueError when an XPath selects an element diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py index 04315ad89..efa63e049 100644 --- a/tests/test_http_response_text.py +++ b/tests/test_http_response_text.py @@ -1,6 +1,7 @@ from __future__ import annotations import codecs +from typing import cast from unittest import mock import pytest @@ -14,6 +15,12 @@ from tests.utils.bases.http_response import TestResponseBase class TestTextResponse(TestResponseBase): response_class = TextResponse + def _links_response(self) -> TextResponse: + return cast("TextResponse", super()._links_response()) + + def _links_response_no_href(self) -> TextResponse: + return cast("TextResponse", super()._links_response_no_href()) + def test_follow_None_encoding(self): # unlike the base Response, TextResponse.follow() falls back to the # response encoding when encoding is None instead of raising @@ -21,7 +28,7 @@ class TestTextResponse(TestResponseBase): req = r.follow("foo", encoding=None) assert req.encoding == "cp1252" - def test_replace(self): + def test_replace(self) -> None: super().test_replace() r1 = self.response_class( "http://www.example.com", body="hello", encoding="cp852" @@ -344,7 +351,7 @@ class TestTextResponse(TestResponseBase): def test_follow_selector_list(self): resp = self._links_response() with pytest.raises(ValueError, match="SelectorList"): - resp.follow(resp.css("a")) + resp.follow(resp.css("a")) # type: ignore[arg-type] def test_follow_selector_invalid(self): resp = self._links_response() @@ -616,7 +623,7 @@ class CustomResponse(TextResponse): class TestCustomResponse(TestTextResponse): response_class = CustomResponse - def test_copy(self): + def test_copy(self) -> None: super().test_copy() r1 = self.response_class( url="https://example.org", @@ -632,7 +639,7 @@ class TestCustomResponse(TestTextResponse): assert r1.lost == "lost" assert r2.lost is None - def test_replace(self): + def test_replace(self) -> None: super().test_replace() r1 = self.response_class( url="https://example.org", diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index a624d2097..a2a4e8fdb 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -85,6 +85,7 @@ class TestCrawl: url = self.mockserver.url("/status?n=200") crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) response = crawler.spider.meta["responses"][0] assert response.request.url == url @@ -94,6 +95,7 @@ class TestCrawl: url = self.mockserver.url(f"/status?n={status}") crawler = get_crawler(SingleRequestSpider) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) failure = crawler.spider.meta["failure"] response = failure.value.response assert failure.request.url == url @@ -111,6 +113,7 @@ class TestCrawl: }, ) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) failure = crawler.spider.meta["failure"] assert failure.request.url == url assert isinstance(failure.value, ZeroDivisionError) @@ -178,6 +181,7 @@ class TestCrawl: }, ) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) response = crawler.spider.meta["responses"][0] assert response.body == b"Caught ZeroDivisionError" assert response.request.url == OVERRIDDEN_URL @@ -201,6 +205,7 @@ class TestCrawl: }, ) yield crawler.crawl(seed=url, mockserver=self.mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) response = crawler.spider.meta["responses"][0] assert response.body == b"Caught ZeroDivisionError" assert response.request.url == url diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index b88893b2b..6c26aa878 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -49,6 +49,7 @@ class InjectArgumentsSpiderMiddleware: async for element in result: if ( isinstance(element, Request) + and element.callback and element.callback.__name__ == "parse_spider_mw_2" ): element.cb_kwargs["from_process_spider_output"] = True @@ -68,7 +69,12 @@ class KeywordArgumentsSpider(MockServerSpider): checks: list[bool] = [] + def _inc_checks(self, count: int = 1) -> None: + assert self.crawler.stats + self.crawler.stats.inc_value("boolean_checks", count) + async def start(self): + assert self.mockserver data = {"key": "value", "number": 123, "callback": "some_callback"} yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data) yield Request( @@ -89,9 +95,10 @@ class KeywordArgumentsSpider(MockServerSpider): yield Request(self.mockserver.url("/spider_mw"), self.parse_spider_mw) def parse_first(self, response, key, number): + assert self.mockserver self.checks.append(key == "value") self.checks.append(number == 123) - self.crawler.stats.inc_value("boolean_checks", 2) + self._inc_checks(2) yield response.follow( self.mockserver.url("/two"), self.parse_second, @@ -100,28 +107,28 @@ class KeywordArgumentsSpider(MockServerSpider): def parse_second(self, response, new_key): self.checks.append(new_key == "new_value") - self.crawler.stats.inc_value("boolean_checks") + self._inc_checks() def parse_general(self, response, **kwargs): if response.url.endswith("/general_with"): self.checks.append(kwargs["key"] == "value") self.checks.append(kwargs["number"] == 123) self.checks.append(kwargs["callback"] == "some_callback") - self.crawler.stats.inc_value("boolean_checks", 3) + self._inc_checks(3) elif response.url.endswith("/general_without"): self.checks.append(kwargs == {}) - self.crawler.stats.inc_value("boolean_checks") + self._inc_checks() def parse_no_kwargs(self, response): self.checks.append(response.url.endswith("/no_kwargs")) - self.crawler.stats.inc_value("boolean_checks") + self._inc_checks() def parse_default(self, response, key, number=None, default=99): self.checks.append(response.url.endswith("/default")) self.checks.append(key == "value") self.checks.append(number == 123) self.checks.append(default == 99) - self.crawler.stats.inc_value("boolean_checks", 4) + self._inc_checks(4) def parse_takes_less(self, response, key, callback): """ @@ -140,17 +147,18 @@ class KeywordArgumentsSpider(MockServerSpider): ): self.checks.append(bool(from_process_request)) self.checks.append(bool(from_process_response)) - self.crawler.stats.inc_value("boolean_checks", 2) + self._inc_checks(2) def parse_spider_mw(self, response, from_process_spider_input, from_process_start): + assert self.mockserver self.checks.append(bool(from_process_spider_input)) self.checks.append(bool(from_process_start)) - self.crawler.stats.inc_value("boolean_checks", 2) + self._inc_checks(2) return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2) def parse_spider_mw_2(self, response, from_process_spider_output): self.checks.append(bool(from_process_spider_output)) - self.crawler.stats.inc_value("boolean_checks", 1) + self._inc_checks() class TestCallbackKeywordArguments: diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index 78ff18b15..c7596e45d 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,7 +1,10 @@ +from typing import Any + import pytest +from twisted.python.failure import Failure from scrapy import Request, Spider -from scrapy.http import JsonRequest +from scrapy.http import JsonRequest, Response from scrapy.utils.request import request_from_dict @@ -10,7 +13,7 @@ class CustomRequest(Request): class TestRequestSerialization: - def setup_method(self): + def setup_method(self) -> None: self.spider = MethodsSpider() def test_basic(self): @@ -42,12 +45,14 @@ class TestRequestSerialization: r = Request("http://www.example.com", body=b"\xc2\xa3") self._assert_serializes_ok(r) - def _assert_serializes_ok(self, request, spider=None): + def _assert_serializes_ok( + self, request: Request, spider: Spider | None = None + ) -> None: d = request.to_dict(spider=spider) request2 = request_from_dict(d, spider=spider) self._assert_same_request(request, request2) - def _assert_same_request(self, r1, r2): + def _assert_same_request(self, r1: Request, r2: Request) -> None: assert r1.__class__ == r2.__class__ assert r1.url == r2.url assert r1.callback == r2.callback @@ -64,6 +69,7 @@ class TestRequestSerialization: assert r1.dont_filter == r2.dont_filter assert r1.flags == r2.flags if isinstance(r1, JsonRequest): + assert isinstance(r2, JsonRequest) assert r1.dumps_kwargs == r2.dumps_kwargs def test_request_class(self): @@ -83,8 +89,8 @@ class TestRequestSerialization: def test_reference_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider.parse_item_reference, - errback=self.spider.handle_error_reference, + callback=self.spider.parse_item_reference, # type: ignore[arg-type,misc] + errback=self.spider.handle_error_reference, # type: ignore[arg-type,misc] ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) @@ -94,8 +100,8 @@ class TestRequestSerialization: def test_private_reference_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._MethodsSpider__parse_item_reference, - errback=self.spider._MethodsSpider__handle_error_reference, + callback=self.spider._MethodsSpider__parse_item_reference, # type: ignore[attr-defined] + errback=self.spider._MethodsSpider__handle_error_reference, # type: ignore[attr-defined] ) self._assert_serializes_ok(r, spider=self.spider) request_dict = r.to_dict(spider=self.spider) @@ -105,7 +111,7 @@ class TestRequestSerialization: def test_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._MethodsSpider__parse_item_private, + callback=self.spider._MethodsSpider__parse_item_private, # type: ignore[attr-defined] errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -113,7 +119,7 @@ class TestRequestSerialization: def test_mixin_private_callback_serialization(self): r = Request( "http://www.example.com", - callback=self.spider._SpiderMixin__mixin_callback, + callback=self.spider._SpiderMixin__mixin_callback, # type: ignore[attr-defined] errback=self.spider.handle_error, ) self._assert_serializes_ok(r, spider=self.spider) @@ -127,7 +133,7 @@ class TestRequestSerialization: self._assert_serializes_ok(r, spider=self.spider) def test_unserializable_callback1(self): - r = Request("http://www.example.com", callback=lambda x: x) + r = Request("http://www.example.com", callback=lambda x: x) # type: ignore[misc] with pytest.raises( ValueError, match="is not an instance method in: None: pass spider = MySpider() r = Request("http://www.example.com", callback=spider.parse) - spider.parse = None + spider.parse = None # type: ignore[method-assign,assignment] with pytest.raises(ValueError, match="is not an instance method in: None: pass class SpiderDelegation: - def delegated_callback(self, response): + def delegated_callback(self, response: Response) -> None: pass -def parse_item(response): +def parse_item(response: Response) -> None: pass -def handle_error(failure): +def handle_error(failure: Failure) -> None: pass -def private_parse_item(response): +def private_parse_item(response: Response) -> None: pass -def private_handle_error(failure): +def private_handle_error(failure: Failure) -> None: pass @@ -197,15 +205,17 @@ class MethodsSpider(Spider, SpiderMixin): __parse_item_reference = private_parse_item __handle_error_reference = private_handle_error - def __init__(self, **kwargs): + def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self.delegated_callback = SpiderDelegation().delegated_callback - def parse_item(self, response): + def parse_item(self, response: Response) -> None: pass - def handle_error(self, failure): + def handle_error(self, failure: Failure) -> None: pass - def __parse_item_private(self, response): # pylint: disable=unused-private-member + def __parse_item_private( # pylint: disable=unused-private-member + self, response: Response + ) -> None: pass diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 726e0573a..46a16ad1e 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -1,57 +1,62 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + from scrapy.signals import request_left_downloader from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer from tests.utils.decorators import inline_callbacks_test +if TYPE_CHECKING: + from scrapy import Request + from scrapy.crawler import Crawler + from tests.mockserver.http import MockServer + class SignalCatcherSpider(Spider): name = "signal_catcher" - def __init__(self, crawler, url, *args, **kwargs): + def __init__(self, crawler: Crawler, url: str, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) crawler.signals.connect(self.on_request_left, signal=request_left_downloader) self.caught_times = 0 self.start_urls = [url] @classmethod - def from_crawler(cls, crawler, *args, **kwargs): + def from_crawler( + cls, crawler: Crawler, *args: Any, **kwargs: Any + ) -> SignalCatcherSpider: return cls(crawler, *args, **kwargs) - def on_request_left(self, request, spider): + def on_request_left(self, request: Request, spider: Spider) -> None: self.caught_times += 1 class TestCatching: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - @inline_callbacks_test - def test_success(self): + def test_success(self, mockserver: MockServer): crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl(self.mockserver.url("/status?n=200")) + yield crawler.crawl(mockserver.url("/status?n=200")) + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 @inline_callbacks_test - def test_timeout(self): + def test_timeout(self, mockserver: MockServer): crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1}) - yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) + yield crawler.crawl(mockserver.url("/delay?n=0.2")) + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 @inline_callbacks_test - def test_disconnect(self): + def test_disconnect(self, mockserver: MockServer): crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl(self.mockserver.url("/drop")) + yield crawler.crawl(mockserver.url("/drop")) + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 @inline_callbacks_test def test_noconnect(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl("http://thereisdefinetelynosuchdomain.com") + assert isinstance(crawler.spider, SignalCatcherSpider) assert crawler.spider.caught_times == 1 diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.py index 3a1e588ef..2b712f3ab 100644 --- a/tests/utils/bases/http_request.py +++ b/tests/utils/bases/http_request.py @@ -3,8 +3,9 @@ from abc import ABC, abstractmethod from typing import Any import pytest +from twisted.python.failure import Failure -from scrapy.http import Headers, Request +from scrapy.http import Headers, Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.utils.request import request_to_curl @@ -22,15 +23,15 @@ class TestRequestBase(ABC): def test_init(self): # Request requires url in the __init__ method with pytest.raises(TypeError): - self.request_class() + self.request_class() # type: ignore[call-arg] # url argument must be basestring with pytest.raises(TypeError): - self.request_class(123) + self.request_class(123) # type: ignore[arg-type] # priority argument must be an integer with pytest.raises(TypeError, match="Request priority not an integer"): - self.request_class("http://www.example.com", priority="1") + self.request_class("http://www.example.com", priority="1") # type: ignore[arg-type] r = self.request_class("http://www.example.com") assert isinstance(r.url, str) @@ -205,14 +206,17 @@ class TestRequestBase(ABC): def test_copy(self): """Test Request copy""" - def somecallback(): + def somecallback(response: Response) -> None: + pass + + def someerrback(failure: Failure) -> None: pass r1 = self.request_class( "http://www.example.com", flags=["f1", "f2"], callback=somecallback, - errback=somecallback, + errback=someerrback, ) r1.meta["foo"] = "bar" r1.cb_kwargs["key"] = "value" @@ -220,7 +224,7 @@ class TestRequestBase(ABC): # make sure callbaclks are copied assert r1.callback is somecallback - assert r1.errback is somecallback + assert r1.errback is someerrback assert r2.callback is r1.callback assert r2.errback is r1.errback @@ -251,7 +255,7 @@ class TestRequestBase(ABC): def test_copy_inherited_classes(self): """Test Request children copies preserve their class""" - class CustomRequest(self.request_class): + class CustomRequest(self.request_class): # type: ignore[misc,name-defined] pass r1 = CustomRequest("http://www.example.com") @@ -283,7 +287,9 @@ class TestRequestBase(ABC): assert r4.dont_filter is False # the cls argument allows changing the resulting class - custom_request_cls = type("CustomRequest", (self.request_class,), {}) + custom_request_cls: type[Request] = type( + "CustomRequest", (self.request_class,), {} + ) r5 = r1.replace(cls=custom_request_cls) assert isinstance(r5, custom_request_cls) assert r5.url == r1.url @@ -295,33 +301,36 @@ class TestRequestBase(ABC): def test_immutable_attributes(self): r = self.request_class("http://example.com") with pytest.raises(AttributeError): - r.url = "http://example2.com" + r.url = "http://example2.com" # type: ignore[misc] with pytest.raises(AttributeError): - r.body = "xxx" + r.body = "xxx" # type: ignore[misc,assignment] def test_callback_and_errback(self): - def a_function(): + def a_callback(response: Response) -> None: + pass + + def an_errback(failure: Failure) -> None: pass r1 = self.request_class("http://example.com") assert r1.callback is None assert r1.errback is None - r2 = self.request_class("http://example.com", callback=a_function) - assert r2.callback is a_function + r2 = self.request_class("http://example.com", callback=a_callback) + assert r2.callback is a_callback assert r2.errback is None - r3 = self.request_class("http://example.com", errback=a_function) + r3 = self.request_class("http://example.com", errback=an_errback) assert r3.callback is None - assert r3.errback is a_function + assert r3.errback is an_errback r4 = self.request_class( url="http://example.com", - callback=a_function, - errback=a_function, + callback=a_callback, + errback=an_errback, ) - assert r4.callback is a_function - assert r4.errback is a_function + assert r4.callback is a_callback + assert r4.errback is an_errback r5 = self.request_class( url="http://example.com", @@ -329,18 +338,18 @@ class TestRequestBase(ABC): errback=NO_CALLBACK, ) assert r5.callback is NO_CALLBACK - assert r5.errback is NO_CALLBACK + assert r5.errback is NO_CALLBACK # type: ignore[comparison-overlap] def test_callback_and_errback_type(self): with pytest.raises(TypeError): - self.request_class("http://example.com", callback="a_function") + self.request_class("http://example.com", callback="a_function") # type: ignore[arg-type] with pytest.raises(TypeError): - self.request_class("http://example.com", errback="a_function") + self.request_class("http://example.com", errback="a_function") # type: ignore[arg-type] with pytest.raises(TypeError): self.request_class( url="http://example.com", - callback="a_function", - errback="a_function", + callback="a_function", # type: ignore[arg-type] + errback="a_function", # type: ignore[arg-type] ) def test_setters(self): diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.py index 2fbf6527a..78e14e7b0 100644 --- a/tests/utils/bases/http_response.py +++ b/tests/utils/bases/http_response.py @@ -7,7 +7,7 @@ import pytest from w3lib.encoding import resolve_encoding from scrapy.exceptions import NotSupported -from scrapy.http import Headers, Request, Response +from scrapy.http import Headers, Request, Response, TextResponse from scrapy.link import Link from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS from tests import get_testdata @@ -15,6 +15,8 @@ from tests import get_testdata if TYPE_CHECKING: from collections.abc import Iterable + from parsel import Selector + class TestResponseBase(ABC): @property @@ -25,14 +27,14 @@ class TestResponseBase(ABC): def test_init(self): # Response requires url in the constructor with pytest.raises(TypeError): - self.response_class() + self.response_class() # type: ignore[call-arg] assert isinstance( self.response_class("http://example.com/"), self.response_class ) with pytest.raises(TypeError): - self.response_class(b"http://example.com") + self.response_class(b"http://example.com") # type: ignore[arg-type] with pytest.raises(TypeError): - self.response_class(url="http://example.com", body={}) + self.response_class(url="http://example.com", body={}) # type: ignore[arg-type] # body can be str or None assert isinstance( self.response_class("http://example.com/", body=b""), @@ -67,12 +69,12 @@ class TestResponseBase(ABC): r = self.response_class("http://www.example.com", status=301) assert r.status == 301 - r = self.response_class("http://www.example.com", status="301") + r = self.response_class("http://www.example.com", status="301") # type: ignore[arg-type] assert r.status == 301 with pytest.raises(ValueError, match=r"invalid literal for int\(\)"): - self.response_class("http://example.com", status="lala200") + self.response_class("http://example.com", status="lala200") # type: ignore[arg-type] - def test_copy(self): + def test_copy(self) -> None: """Test Response copy""" r1 = self.response_class("http://www.example.com", body=b"Some body") @@ -121,7 +123,7 @@ class TestResponseBase(ABC): def test_copy_inherited_classes(self): """Test Response children copies preserve their class""" - class CustomResponse(self.response_class): + class CustomResponse(self.response_class): # type: ignore[misc,name-defined] pass r1 = CustomResponse("http://www.example.com") @@ -129,7 +131,7 @@ class TestResponseBase(ABC): assert isinstance(r2, CustomResponse) - def test_replace(self): + def test_replace(self) -> None: """Test Response.replace() method""" hdrs = Headers({"key": "value"}) r1 = self.response_class("http://www.example.com") @@ -146,7 +148,9 @@ class TestResponseBase(ABC): assert r4.body == b"" assert not r4.flags - def _assert_response_values(self, response, encoding, body): + def _assert_response_values( + self, response: TextResponse, encoding: str, body: str | bytes + ) -> None: if isinstance(body, str): body_unicode = body body_bytes = body.encode(encoding) @@ -160,15 +164,15 @@ class TestResponseBase(ABC): assert response.body == body_bytes assert response.text == body_unicode - def _assert_response_encoding(self, response, encoding): + def _assert_response_encoding(self, response: TextResponse, encoding: str) -> None: assert response.encoding == resolve_encoding(encoding) def test_immutable_attributes(self): r = self.response_class("http://example.com") with pytest.raises(AttributeError): - r.url = "http://example2.com" + r.url = "http://example2.com" # type: ignore[misc] with pytest.raises(AttributeError): - r.body = "xxx" + r.body = "xxx" # type: ignore[misc,assignment] def test_setter_mutable_lazy_loading(self): """Mutable attributes are set internally to None only until they are @@ -256,7 +260,7 @@ class TestResponseBase(ABC): def test_follow_None_url(self): r = self.response_class("http://example.com") with pytest.raises(ValueError, match="url can't be None"): - r.follow(None) + r.follow(None) # type: ignore[arg-type] def test_follow_None_encoding(self): r = self.response_class("http://example.com") @@ -325,20 +329,20 @@ class TestResponseBase(ABC): r = self.response_class("http://example.com") if self.response_class == Response: with pytest.raises(TypeError): - list(r.follow_all(urls=None)) + list(r.follow_all(urls=None)) # type: ignore[arg-type] with pytest.raises(TypeError): - list(r.follow_all(urls=12345)) + list(r.follow_all(urls=12345)) # type: ignore[arg-type] with pytest.raises(ValueError, match="url can't be None"): - list(r.follow_all(urls=[None])) + list(r.follow_all(urls=[None])) # type: ignore[list-item] else: with pytest.raises( ValueError, match="Please supply exactly one of the following arguments" ): - list(r.follow_all(urls=None)) + list(r.follow_all(urls=None)) # type: ignore[arg-type] with pytest.raises(TypeError): - list(r.follow_all(urls=12345)) + list(r.follow_all(urls=12345)) # type: ignore[arg-type] with pytest.raises(ValueError, match="url can't be None"): - list(r.follow_all(urls=[None])) + list(r.follow_all(urls=[None])) # type: ignore[list-item] @pytest.mark.xfail( not W3LIB_STRIPS_URLS, @@ -384,14 +388,14 @@ class TestResponseBase(ABC): def _assert_followed_url( self, - follow_obj: str | Link, + follow_obj: str | Link | Selector, target_url: str, response: Response | None = None, encoding: str | None = None, ) -> None: if response is None: response = self._links_response() - req = response.follow(follow_obj) + req = response.follow(follow_obj) # type: ignore[arg-type] assert req.url == target_url if encoding is not None: assert req.encoding == encoding