From deb7e2861e616bbfefb96433bd72e0d50055cd0e Mon Sep 17 00:00:00 2001 From: Gaurav Yadav Date: Tue, 30 Jun 2026 18:19:47 +0530 Subject: [PATCH 01/65] Fix _get_tag_name() crash for non-string elem.tag (#7686) (#7687) * Fix _get_tag_name() crash for non-string elem.tag (#7686) * test: improve non-string tag test accuracy and add direct unit test * test: use minimal payload for non-string tag test * test: address Adrian+syncrain PR feedback - remove first docstring line (Adrian: unnecessary) - replace weak isinstance assert with no-op call - keep Cython function mention (Adrian: wording is great) * test: replace silent call with assert results == [] per Adrian * chore: drop accidental pyproject.toml and uv.lock changes --- scrapy/utils/sitemap.py | 9 +++++---- tests/test_utils_sitemap.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 1520a4ff0..03b7bf3b1 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -97,10 +97,11 @@ class Sitemap: @staticmethod def _get_tag_name(elem: lxml.etree._Element) -> str: - if TYPE_CHECKING: - assert isinstance(elem.tag, str) - _, _, localname = elem.tag.partition("}") - return localname or elem.tag + tag = elem.tag + if not isinstance(tag, str): + return "" + _, _, localname = tag.partition("}") + return localname or tag def sitemap_urls_from_robots( diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index 3599c4824..ac57e1739 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -311,3 +311,14 @@ def test_xml_entity_expansion(): """ ) assert list(s) == [{"loc": "http://127.0.0.1:8000/"}] + + +def test_sitemap_non_string_tag(): + """With recover=True and resolve_entities=False, libxml2 >= 2.14.6 (used + by lxml >= 6.1.1) preserves undeclared entity reference nodes whose + .tag is a non-string ``Cython function`` object instead of a ``str``. + _get_tag_name must handle this gracefully instead of raising + AttributeError. + """ + results = list(Sitemap(b"&k;")) + assert results == [] From 00098cb596d0d3957236ba1ba193b09172696bc5 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 30 Jun 2026 18:27:32 +0500 Subject: [PATCH 02/65] Assorted docstring fixes (#7698) * Smaller fixes. * Add more code blocks in docstrings. * Queue stuff. * Response stuff. * Round 2. * Address feedback. --- scrapy/addons.py | 2 +- scrapy/commands/__init__.py | 2 +- scrapy/core/downloader/contextfactory.py | 9 +++- scrapy/core/downloader/handlers/ftp.py | 6 +-- scrapy/core/downloader/handlers/http11.py | 3 +- scrapy/core/http2/agent.py | 6 +-- scrapy/core/http2/protocol.py | 4 +- scrapy/core/scheduler.py | 12 ++--- scrapy/core/scraper.py | 4 +- scrapy/crawler.py | 12 ++--- .../downloadermiddlewares/httpcompression.py | 2 +- scrapy/downloadermiddlewares/redirect.py | 7 ++- scrapy/downloadermiddlewares/retry.py | 8 ++- scrapy/exceptions.py | 2 +- scrapy/extensions/logstats.py | 2 +- scrapy/extensions/throttle.py | 2 +- scrapy/http/cookies.py | 6 +-- scrapy/http/response/html.py | 5 +- scrapy/http/response/xml.py | 5 +- scrapy/item.py | 2 +- scrapy/link.py | 4 +- scrapy/logformatter.py | 24 +++++---- scrapy/mail.py | 2 - scrapy/pipelines/__init__.py | 2 +- scrapy/pipelines/files.py | 2 +- scrapy/pipelines/images.py | 2 +- scrapy/pqueues.py | 7 +-- scrapy/settings/__init__.py | 11 ++-- scrapy/shell.py | 5 +- scrapy/spidermiddlewares/base.py | 8 +-- scrapy/spidermiddlewares/httperror.py | 2 +- scrapy/spidermiddlewares/referer.py | 2 +- scrapy/spiders/feed.py | 29 ++++++----- scrapy/utils/datatypes.py | 2 - scrapy/utils/defer.py | 52 +++++++++++-------- scrapy/utils/deprecate.py | 15 +++--- scrapy/utils/log.py | 3 +- scrapy/utils/response.py | 2 +- scrapy/utils/signal.py | 7 ++- scrapy/utils/trackref.py | 4 +- scrapy/utils/url.py | 4 +- 41 files changed, 151 insertions(+), 139 deletions(-) diff --git a/scrapy/addons.py b/scrapy/addons.py index 2e12f8c8a..470be47c1 100644 --- a/scrapy/addons.py +++ b/scrapy/addons.py @@ -64,7 +64,7 @@ class AddonManager: :param settings: The :class:`~scrapy.settings.BaseSettings` object from \ which to read the early add-on configuration - :type settings: :class:`~scrapy.settings.Settings` + :type settings: :class:`~scrapy.settings.BaseSettings` """ for clspath in build_component_list(settings["ADDONS"]): addoncls = load_object(clspath) diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 598e8060e..d9c919db9 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -73,7 +73,7 @@ class ScrapyCommand(ABC): def long_desc(self) -> str: """A long description of the command. Return short description when not available. It cannot contain newlines since contents will be formatted - by optparser which removes newlines and wraps text. + by argparse which removes newlines and wraps text. """ return self.short_desc() diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index c1796c723..a934cbbc7 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -47,8 +47,13 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): instance. The purpose of this custom class is to provide a ``creatorForNetloc()`` - method that returns a ``_ScrapyClientTLSOptions`` instance configured based - on TLS settings provided to the factory. + method that returns: + + - a ``_ScrapyClientTLSOptions26`` or ``_ScrapyClientTLSOptions`` instance + configured based on TLS settings provided to the factory (when the + certificate verification is disabled); + - a result of ``optionsForClientTLS()`` called with those TLS settings + (when the certificate verification is enabled). """ def __init__( diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 9064aa53a..07ff4a74e 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -2,9 +2,9 @@ An asynchronous FTP file download handler for scrapy which somehow emulates an http response. FTP connection parameters are passed using the request meta field: -- ftp_user (required) -- ftp_password (required) -- ftp_passive (by default, enabled) sets FTP connection passive mode +- ftp_user (optional, falls back to FTP_USER) +- ftp_password (optional, falls back to FTP_PASSWORD) +- ftp_passive (optional, falls back to FTP_PASSIVE_MODE) sets FTP connection passive mode - ftp_local_filename - If not given, file data will come in the response.body, as a normal scrapy Response, which will imply that the entire file will be on memory. diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1fc504c59..17dca5acb 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -104,7 +104,6 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): self._disconnect_timeout: int = 1 async def download_request(self, request: Request) -> Response: - """Return a deferred for the HTTP download""" if hasattr(self._crawler.spider, "download_maxsize"): # pragma: no cover warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE") if hasattr(self._crawler.spider, "download_warnsize"): # pragma: no cover @@ -283,7 +282,7 @@ def _tunnel_request_data( class _TunnelingAgent(Agent): - """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS + """An agent that uses a ``_TunnelingTCP4ClientEndpoint`` to make HTTPS downloads. It may look strange that we have chosen to subclass Agent and not ProxyAgent but consider that after the tunnel is opened the proxy is transparent to the client; thus the agent should behave like there is no diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index 7137a0f2b..aa55e29a0 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -114,11 +114,7 @@ class H2ConnectionPool: d.errback(ResponseFailed(errors)) def close_connections(self) -> None: - """Close all the HTTP/2 connections and remove them from pool - - Returns: - Deferred that fires when all connections have been closed - """ + """Close all the HTTP/2 connections and remove them from pool.""" for conn in self._connections.values(): assert conn.transport is not None # typing conn.transport.abortConnection() diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 39703f976..7136e829e 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -101,7 +101,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): uri is used to verify that incoming client requests have correct base URL. settings -- Scrapy project settings - conn_lost_deferred -- Deferred fires with the reason: Failure to notify + conn_lost_deferred -- Deferred that fires with the list of underlying exceptions to notify that connection was lost tls_verbose_logging -- Whether to log TLS details """ @@ -375,7 +375,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def _handle_events(self, events: list[Event]) -> None: """Private method which acts as a bridge between the events - received from the HTTP/2 data and IH2EventsHandler + received from the HTTP/2 data and the handlers in this class. Arguments: events -- A list of events that the remote peer triggered by sending data diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 7217da942..82e31b90b 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -131,10 +131,10 @@ class Scheduler(BaseScheduler): (:setting:`SCHEDULER_PRIORITY_QUEUE`) that sort requests by :attr:`~scrapy.http.Request.priority`. - By default, a single, memory-based priority queue is used for all requests. - When using :setting:`JOBDIR`, a disk-based priority queue is also created, + By default, memory-based priority queues are used for all requests. + When using :setting:`JOBDIR`, disk-based priority queues are also created, and only unserializable requests are stored in the memory-based priority - queue. For a given priority value, requests in memory take precedence over + queues. For a given priority value, requests in memory take precedence over requests in disk. Each priority queue stores requests in separate internal queues, one per @@ -209,8 +209,8 @@ class Scheduler(BaseScheduler): ------------------------- While pending requests are below the configured values of - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` - or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent + :setting:`CONCURRENT_REQUESTS` or + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent concurrently. As a result, the first few requests of a crawl may not follow the desired @@ -342,7 +342,7 @@ class Scheduler(BaseScheduler): def open(self, spider: Spider) -> Deferred[None] | None: """ (1) initialize the memory queue - (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory + (2) initialize the disk queue if the ``jobdir`` argument wasn't empty (3) return the result of the dupefilter's ``open`` method """ self.spider: Spider = spider diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 466ce656d..58e37ce5e 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -441,7 +441,7 @@ class Scraper: self, output: Any, response: Response | Failure ) -> Deferred[None]: """Process each Request/Item (given in the output parameter) returned - from the given spider. + from the spider. Items are sent to the item pipelines, requests are scheduled. """ @@ -451,7 +451,7 @@ class Scraper: self, output: Any, response: Response | Failure ) -> None: """Process each Request/Item (given in the output parameter) returned - from the given spider. + from the spider. Items are sent to the item pipelines, requests are scheduled. """ diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 9828fe6f5..c72752915 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -531,8 +531,8 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): """ Run a crawler with the provided arguments. - It will call the given Crawler's :meth:`~Crawler.crawl` method, while - keeping track of it so it can be stopped later. + It will call the given Crawler's :meth:`~Crawler.crawl_async` method, + while keeping track of it so it can be stopped later. If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler` instance, this method will try to create one using this parameter as @@ -773,7 +773,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner): """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS - resolver based on :setting:`DNSCACHE_ENABLED`. + resolver based on :setting:`TWISTED_DNS_RESOLVER`. If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. @@ -875,10 +875,10 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): When using a reactor it adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE` and installs a DNS resolver based - on :setting:`DNSCACHE_ENABLED`. + on :setting:`TWISTED_DNS_RESOLVER`. - If ``stop_after_crawl`` is True, the reactor will be stopped after all - crawlers have finished, using :meth:`join`. + If ``stop_after_crawl`` is True, the reactor/event loop will be stopped + after all crawlers have finished, using :meth:`join`. :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 2b1721ced..6ca04a50e 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -60,7 +60,7 @@ else: class HttpCompressionMiddleware: - """This middleware allows compressed (gzip, deflate) traffic to be + """This middleware allows compressed (gzip, deflate etc.) traffic to be sent/received from websites""" def __init__( diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 821f41699..45af5c67a 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -196,10 +196,7 @@ class BaseRedirectMiddleware: class RedirectMiddleware(BaseRedirectMiddleware): - """ - Handle redirection of requests based on response status - and meta-refresh html tag. - """ + """Handle redirection of requests based on response status.""" @_warn_spider_arg def process_response( @@ -251,6 +248,8 @@ class RedirectMiddleware(BaseRedirectMiddleware): class MetaRefreshMiddleware(BaseRedirectMiddleware): + """Handle redirection of requests based on meta-refresh html tag.""" + enabled_setting = "METAREFRESH_ENABLED" def __init__(self, settings: BaseSettings): diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5f125cae4..a0a0b60a2 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -5,9 +5,6 @@ problems such as a connection timeout or HTTP 500 error. You can change the behaviour of this middleware by modifying the scraping settings: RETRY_TIMES - how many times to retry a failed page RETRY_HTTP_CODES - which HTTP response codes to retry - -Failed pages are collected on the scraping process and rescheduled at the end, -once the spider has finished crawling all regular (non-failed) pages. """ from __future__ import annotations @@ -70,8 +67,9 @@ def get_retry_request( and :ref:`stats `, and to provide extra logging context (see :func:`logging.debug`). - *reason* is a string or an :class:`Exception` object that indicates the - reason why the request needs to be retried. It is used to name retry stats. + *reason* is a string, an :class:`Exception` subclass or an + :class:`Exception` object that indicates the reason why the request needs + to be retried. It is used to name retry stats. *max_retry_times* is a number that determines the maximum number of times that *request* can be retried. If not specified or ``None``, the number is diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 204132973..5330eab48 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -115,7 +115,7 @@ class UsageError(Exception): class ScrapyDeprecationWarning(Warning): """Warning category for deprecated features, since the default - DeprecationWarning is silenced on Python 2.7+ + :exc:`DeprecationWarning` is silenced. """ diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index 3d7674905..6c94d947e 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) class LogStats: """Log basic scraping stats periodically like: - * RPM - Requests per Minute + * RPM - Responses per Minute * IPM - Items per Minute """ diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index cdb0671ae..542ff1cdc 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -116,7 +116,7 @@ class AutoThrottle: # It works better with problematic sites. new_delay = max(target_delay, new_delay) - # Make sure self.mindelay <= new_delay <= self.max_delay + # Make sure self.mindelay <= new_delay <= self.maxdelay new_delay = min(max(self.mindelay, new_delay), self.maxdelay) # Dont adjust delay if response status != 200 and new delay is smaller diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 599f20947..8edeae01c 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -136,9 +136,9 @@ class _DummyLock: class WrappedRequest: - """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class - - see http://docs.python.org/library/urllib2.html#urllib2.Request + """Wraps a :class:`scrapy.Request` class with methods defined by + the :class:`urllib.request.Request` class to interact with + the :class:`http.cookiejar.CookieJar` class. """ def __init__(self, request: Request): diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index 70c08c11d..6d3a9ee12 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -1,6 +1,7 @@ """ -This module implements the HtmlResponse class which adds encoding -discovering through HTML encoding declarations to the TextResponse class. +This module implements the :class:`HtmlResponse` class which is used as a +content type marker by :class:`~scrapy.selector.Selector` and can be used in +``isinstance()`` checks. See documentation in docs/topics/request-response.rst """ diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py index 6d9c4cb73..847fb2b3c 100644 --- a/scrapy/http/response/xml.py +++ b/scrapy/http/response/xml.py @@ -1,6 +1,7 @@ """ -This module implements the XmlResponse class which adds encoding -discovering through XML encoding declarations to the TextResponse class. +This module implements the :class:`XmlResponse` class which is used as a +content type marker by :class:`~scrapy.selector.Selector` and can be used in +``isinstance()`` checks. See documentation in docs/topics/request-response.rst """ diff --git a/scrapy/item.py b/scrapy/item.py index 1cc0ae584..d5adc1efb 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -1,7 +1,7 @@ """ Scrapy Item -See documentation in docs/topics/item.rst +See documentation in docs/topics/items.rst """ from __future__ import annotations diff --git a/scrapy/link.py b/scrapy/link.py index 046630403..8211faccd 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -9,7 +9,9 @@ its documentation in: docs/topics/link-extractors.rst class Link: """Link objects represent an extracted link by the LinkExtractor. - Using the anchor tag sample below to illustrate the parameters:: + Using the anchor tag sample below to illustrate the parameters: + + .. code-block:: html Dont follow this one diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index a50064e08..bfb2d5dff 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -58,18 +58,20 @@ class LogFormatter: logging an action the method must return ``None``. Here is an example on how to create a custom log formatter to lower the severity level of - the log message when an item is dropped from the pipeline:: + the log message when an item is dropped from the pipeline: - class PoliteLogFormatter(logformatter.LogFormatter): - def dropped(self, item, exception, response, spider): - return { - 'level': logging.INFO, # lowering the level from logging.WARNING - 'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s", - 'args': { - 'exception': exception, - 'item': item, - } - } + .. code-block:: python + + class PoliteLogFormatter(logformatter.LogFormatter): + def dropped(self, item, exception, response, spider): + return { + "level": logging.INFO, # lowering the level from logging.WARNING + "msg": "Dropped: %(exception)s" + os.linesep + "%(item)s", + "args": { + "exception": exception, + "item": item, + }, + } """ def crawled( diff --git a/scrapy/mail.py b/scrapy/mail.py index fbd11ad1d..97123e63c 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -1,7 +1,5 @@ """ Mail sending helpers - -See documentation in docs/topics/email.rst """ from __future__ import annotations diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 84fb5f85e..383c461e6 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -1,7 +1,7 @@ """ Item pipeline -See documentation in docs/item-pipeline.rst +See documentation in docs/topics/item-pipeline.rst """ from __future__ import annotations diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 0066fd38f..44c422430 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -423,7 +423,7 @@ class FTPFilesStore: class FilesPipeline(MediaPipeline): - """Abstract pipeline that implement the file downloading + """Pipeline that implements file downloading. This pipeline tries to minimize network transfers and file processing, doing stat of the files and determining if file is new, up-to-date or diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 762b0fdf1..79b6c4f27 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -47,7 +47,7 @@ class ImageException(FileException): class ImagesPipeline(FilesPipeline): - """Abstract pipeline that implement the image thumbnail generation logic""" + """Pipeline that implements the handling logic specific to images.""" MEDIA_NAME: str = "image" diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 2efc43d4b..41411ceaf 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -92,9 +92,10 @@ class ScrapyPriorityQueue: - The :data:`~scrapy.Request.priority` of the request. For each combination of the above seen, this class creates an instance of - *downstream_queue_cls* with *key* set to a subdirectory of the persistence - directory, named as the request priority (e.g. ``1``), with an ``s`` suffix - in case of a start request (e.g. ``1s``). + *downstream_queue_cls* (or *start_queue_cls* for start requests if it was + passed) with *key* set to a subdirectory of the persistence directory, + named as the negated request priority (e.g. ``-1``), with an ``s`` suffix + in case of a start request (e.g. ``-1s``). """ @classmethod diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index be298e9b1..932463ba9 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -454,9 +454,10 @@ class BaseSettings(MutableMapping[str, Any]): """ Store a key/value attribute with a given priority. - Settings should be populated *before* configuring the Crawler object - (through the :meth:`~scrapy.crawler.Crawler.configure` method), - otherwise they won't have any effect. + Settings should be populated *before* the Crawler object applies them + (in the :meth:`~scrapy.crawler.Crawler.crawl_async` or + :meth:`~scrapy.crawler.Crawler.crawl` method), otherwise they won't + have any effect. :param name: the setting name :type name: str @@ -613,7 +614,7 @@ class BaseSettings(MutableMapping[str, Any]): """ Make a deep copy of current settings. - This method returns a new instance of the :class:`Settings` class, + This method returns a new instance of this class, populated with the same values and their priorities. Modifications to the new object won't be reflected on the original @@ -658,7 +659,7 @@ class BaseSettings(MutableMapping[str, Any]): Make a copy of current settings and convert to a dict. This method returns a new dict populated with the same values - and their priorities as the current settings. + as the current settings. Modifications to the returned dict won't be reflected on the original settings. diff --git a/scrapy/shell.py b/scrapy/shell.py index 44dcd880e..dfea00c46 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -75,7 +75,7 @@ if TYPE_CHECKING: # running event loop. # # Side note: it should be possible to remove _request_deferred() by using -# engine.download_async() instead of engine.schedule(), losing the usual stuff +# engine.download_async() instead of engine.crawl(), losing the usual stuff # like spider middlewares (none of which should be important). # # Other architecture problems: @@ -188,7 +188,8 @@ class Shell: async def _schedule(self, request: Request, spider: Spider | None) -> Response: """Send the request to the engine, wait for the result. - Runs in the reactor thread. + Runs in the reactor thread when using the reactor, or in the asyncio + event loop thread otherwise. """ if not self.spider: await self._open_spider(spider) diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py index e09f2d10e..6c62dccb8 100644 --- a/scrapy/spidermiddlewares/base.py +++ b/scrapy/spidermiddlewares/base.py @@ -75,7 +75,7 @@ class BaseSpiderMiddleware: ) -> Request | None: """Return a processed request from the spider output. - This method is called with a single request from the start seeds or the + This method is called with a single request from ``start()`` or the spider output. It should return the same or a different request, or ``None`` to ignore it. @@ -84,7 +84,7 @@ class BaseSpiderMiddleware: :param response: the response being processed :type response: :class:`~scrapy.http.Response` object or ``None`` for - start seeds + start requests :return: the processed request or ``None`` """ @@ -93,7 +93,7 @@ class BaseSpiderMiddleware: def get_processed_item(self, item: Any, response: Response | None) -> Any: """Return a processed item from the spider output. - This method is called with a single item from the start seeds or the + This method is called with a single item from ``start()`` or the spider output. It should return the same or a different item, or ``None`` to ignore it. @@ -102,7 +102,7 @@ class BaseSpiderMiddleware: :param response: the response being processed :type response: :class:`~scrapy.http.Response` object or ``None`` for - start seeds + start items :return: the processed item or ``None`` """ diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 94b6dfbb5..156b73e7e 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) class HttpError(IgnoreRequest): - """A non-200 response was filtered""" + """A non-2xx response was filtered""" def __init__(self, response: Response, *args: Any, **kwargs: Any): self.response = response diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 264f685c1..1305874d5 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -92,7 +92,7 @@ class ReferrerPolicy(ABC): ) def origin(self, url: str) -> str | None: - """Return serialized origin (scheme, host, path) for a request or response URL.""" + """Return serialized origin (scheme, host, port) for a request or response URL.""" return self.strip_url(url, origin_only=True) def potentially_trustworthy(self, url: str) -> bool: diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 395183613..925f31ede 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -54,17 +54,21 @@ class XMLFeedSpider(Spider): return response def parse_node(self, response: Response, selector: Selector) -> Any: - """This method must be overridden with your custom spider functionality""" + """This method is called for the nodes matching the provided tag name + (itertag). Receives the response and an Selector for each node. + + This method must return either an item, a request, or a list + containing any of them. + + This method must be overridden with your custom spider functionality. + """ if hasattr(self, "parse_item"): # backward compatibility return self.parse_item(response, selector) raise NotImplementedError def parse_nodes(self, response: Response, nodes: Iterable[Selector]) -> Any: """This method is called for the nodes matching the provided tag name - (itertag). Receives the response and an Selector for each node. - Overriding this method is mandatory. Otherwise, you spider won't work. - This method must return either an item, a request, or a list - containing any of them. + (itertag). Receives the response and an iterable of Selectors. """ for selector in nodes: @@ -113,6 +117,9 @@ class CSVFeedSpider(Spider): It receives a CSV file in a response; iterates through each of its rows, and calls parse_row with a dict containing each field's data. + This spider also gives the opportunity to override adapt_response and + process_results methods for pre and post-processing purposes. + You can set some options regarding the CSV file, such as the delimiter, quotechar and the file's headers. """ @@ -136,16 +143,14 @@ class CSVFeedSpider(Spider): return response def parse_row(self, response: Response, row: dict[str, str]) -> Any: - """This method must be overridden with your custom spider functionality""" + """Receives a response and a dict (representing each row) with a key for + each provided (or detected) header of the CSV file. + + This method must be overridden with your custom spider functionality. + """ raise NotImplementedError def parse_rows(self, response: Response) -> Any: - """Receives a response and a dict (representing each row) with a key for - each provided (or detected) header of the CSV file. This spider also - gives the opportunity to override adapt_response and - process_results methods for pre and post-processing purposes. - """ - for row in csviter( response, self.delimiter, self.headers, quotechar=self.quotechar ): diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index dd0e062d0..c020ff4b9 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -1,8 +1,6 @@ """ This module contains data types used by Scrapy which are not included in the Python Standard Library. - -This module must not depend on any module outside the Standard Library. """ from __future__ import annotations diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 29a34d4ef..d0259b634 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -103,7 +103,7 @@ async def _defer_sleep_async() -> None: def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover warnings.warn( "scrapy.utils.defer.defer_result() is deprecated, use" - " twisted.internet.defer.success() and twisted.internet.defer.fail()," + " twisted.internet.defer.succeed() and twisted.internet.defer.fail()," " plus an explicit sleep if needed, or explicit reactor.callLater().", category=ScrapyDeprecationWarning, stacklevel=2, @@ -469,22 +469,22 @@ def _maybeDeferred_coro( def deferred_to_future(d: Deferred[_T]) -> Future[_T]: """Return an :class:`asyncio.Future` object that wraps *d*. - This function requires - :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be - installed. + This function requires an installed asyncio reactor or a running asyncio + event loop, see :ref:`using-asyncio`. - When :ref:`using the asyncio reactor `, you cannot await - on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy - callables defined as coroutines `, you can only await on - ``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects - allows you to wait on them:: + In this state you cannot await on :class:`~twisted.internet.defer.Deferred` + objects from :ref:`Scrapy callables defined as coroutines + `, you can only await on ``Future`` objects. Wrapping + ``Deferred`` objects into ``Future`` objects allows you to wait on them: + + .. code-block:: python class MySpider(Spider): ... + async def parse(self, response): - additional_request = scrapy.Request('https://example.org/price') - deferred = self.crawler.engine.download(additional_request) - additional_response = await deferred_to_future(deferred) + deferred = some_dfd_helper() + result = await deferred_to_future(deferred) .. versionchanged:: 2.14 This function no longer installs an asyncio loop if called before the @@ -492,7 +492,10 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]: in this case. """ if not is_asyncio_available(): - raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.") + raise RuntimeError( + "deferred_to_future() requires an installed asyncio reactor" + " or a running asyncio event loop." + ) return d.asFuture(asyncio.get_event_loop()) @@ -501,23 +504,26 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: defined as a coroutine `. What you can await in Scrapy callables defined as coroutines depends on the - value of :setting:`TWISTED_REACTOR`: + value of :setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED`: - - When :ref:`using the asyncio reactor `, you can only - await on :class:`asyncio.Future` objects. + - When :ref:`using the asyncio reactor `, or :ref:`not + using a reactor at all `, you can only await + on :class:`asyncio.Future` objects. - - When not using the asyncio reactor, you can only await on - :class:`~twisted.internet.defer.Deferred` objects. + - When :ref:`using a non-asyncio reactor `, you can only + await on :class:`~twisted.internet.defer.Deferred` objects. - If you want to write code that uses ``Deferred`` objects but works with any - reactor, use this function on all ``Deferred`` objects:: + If you want to write code that uses ``Deferred`` objects but works in both + of these states, use this function on all ``Deferred`` objects: + + .. code-block:: python class MySpider(Spider): ... + async def parse(self, response): - additional_request = scrapy.Request('https://example.org/price') - deferred = self.crawler.engine.download(additional_request) - additional_response = await maybe_deferred_to_future(deferred) + deferred = some_dfd_helper() + result = await maybe_deferred_to_future(deferred) """ if not is_asyncio_available(): return d diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 359f819d7..4fd50fdad 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -43,15 +43,18 @@ def create_deprecated_class( It can be used to rename a base class in a library. For example, if we have - class OldName(SomeClass): - # ... + .. code-block:: python - and we want to rename it to NewName, we can do the following:: + class OldName(SomeClass): ... - class NewName(SomeClass): - # ... + and we want to rename it to NewName, we can do the following: - OldName = create_deprecated_class('OldName', NewName) + .. code-block:: python + + class NewName(SomeClass): ... + + + OldName = create_deprecated_class("OldName", NewName) Then, if user class inherits from OldName, warning is issued. Also, if some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)`` diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index aa77e692a..7645b235e 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -248,8 +248,7 @@ def logformatter_adapter( ) -> tuple[int, str, dict[str, Any] | tuple[Any, ...]]: """ Helper that takes the dictionary output from the methods in LogFormatter - and adapts it into a tuple of positional arguments for logger.log calls, - handling backward compatibility as well. + and adapts it into a tuple of positional arguments for logger.log calls. """ level = logkws.get("level", logging.INFO) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c068d9b1e..7747a7b9b 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -88,7 +88,7 @@ def open_in_browser( def parse_details(self, response): - if "item name" not in response.body: + if "item name" not in response.text: open_in_browser(response) """ # circular imports diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index 919f67240..eca95e225 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -39,7 +39,7 @@ def send_catch_log( *arguments: TypingAny, **named: TypingAny, ) -> list[tuple[TypingAny, TypingAny]]: - """Like ``pydispatcher.robust.sendRobust()`` but it also logs errors and returns + """Like ``pydispatch.robust.sendRobust()`` but it also logs errors and returns Failures instead of exceptions. """ dont_log = named.pop("dont_log", ()) @@ -172,9 +172,8 @@ async def _send_catch_log_asyncio( Returns a coroutine that completes once all signal handlers have finished. - This function requires - :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be - installed. + This function requires an installed asyncio reactor or a running asyncio + event loop. .. versionadded:: 2.14 """ diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 87df10a02..22f9eadd0 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -4,9 +4,7 @@ references to live object instances. If you want live objects for a particular class to be tracked, you only have to subclass from object_ref (instead of object). -About performance: This library has a minimal performance impact when enabled, -and no performance penalty at all when disabled (as object_ref becomes just an -alias to object in that case). +This library has a minimal performance impact. .. note:: PyPy uses a tracing garbage collector, so objects may remain in the ``live_refs`` longer than expected, even after they diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 4d2bbdda2..f67853ece 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -117,8 +117,8 @@ def strip_url( - ``strip_credentials`` removes "user:password@" - ``strip_default_port`` removes ":80" (resp. ":443", ":21") from http:// (resp. https://, ftp://) URLs - - ``origin_only`` replaces path component with "/", also dropping - query and fragment components ; it also strips credentials + - ``origin_only`` replaces the path component with "/", also dropping + the query component; it also strips credentials - ``strip_fragment`` drops any #fragment component """ From a6d6a48aa600d1b4c3deb6409c17d0d7f55db312 Mon Sep 17 00:00:00 2001 From: Adrian Date: Tue, 30 Jun 2026 16:26:22 +0200 Subject: [PATCH 03/65] Keep Item fields in definition order (#7694) --- scrapy/item.py | 37 ++++++++++++++++++++++++++++++------- tests/test_feedexport.py | 2 +- tests/test_item.py | 24 ++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index d5adc1efb..4d99ea79d 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -25,6 +25,25 @@ class Field(dict[str, Any]): """Container of field metadata""" +def _ordered_field_names(cls: type) -> list[str]: + """Return the names of the :class:`Field` attributes of *cls* in definition + order. + + Fields declared in base classes come first, ordered from the topmost base + to the most derived class. Within each class, fields keep their definition + order. A field redefined in a subclass keeps the position of its first + definition. + """ + names: list[str] = [] + seen: set[str] = set() + for base in reversed(cls.__mro__): + for name, value in vars(base).items(): + if isinstance(value, Field) and name not in seen: + seen.add(name) + names.append(name) + return names + + class ItemMeta(ABCMeta): """Metaclass_ of :class:`Item` that handles field definitions. @@ -39,13 +58,9 @@ class ItemMeta(ABCMeta): _class = super().__new__(mcs, "x_" + class_name, new_bases, attrs) fields = getattr(_class, "fields", {}) - new_attrs = {} - for n in dir(_class): - v = getattr(_class, n) - if isinstance(v, Field): - fields[n] = v - elif n in attrs: - new_attrs[n] = attrs[n] + for n in _ordered_field_names(_class): + fields[n] = getattr(_class, n) + new_attrs = {n: v for n, v in attrs.items() if not isinstance(v, Field)} new_attrs["fields"] = fields new_attrs["_class"] = _class @@ -80,6 +95,14 @@ class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta): #: those populated. The keys are the field names and the values are the #: :class:`Field` objects used in the :ref:`Item declaration #: `. + #: + #: Fields are kept in definition order: fields declared in base classes + #: come first, followed by fields declared in subclasses, and a field + #: redefined in a subclass keeps the position of its first definition. + #: + #: .. versionchanged:: VERSION + #: Fields are now returned in definition order rather than alphabetical + #: order. fields: dict[str, Field] def __init__(self, *args: Any, **kwargs: Any): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index d7ea60cc8..7d751f188 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -746,7 +746,7 @@ class TestFeedExport(TestFeedExportBase): ] formats = { - "csv": b"baz,egg,foo\r\n,spam1,bar1\r\n", + "csv": b"foo,egg,baz\r\nbar1,spam1,\r\n", "json": b'[\n{"hello": "world2", "foo": "bar2"}\n]', "jsonlines": ( b'{"foo": "bar1", "egg": "spam1"}\n{"hello": "world2", "foo": "bar2"}\n' diff --git a/tests/test_item.py b/tests/test_item.py index 34b054e12..7b4c2e918 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -142,6 +142,30 @@ class TestItem: self.assertSortedEqual(list(item.keys()), ["new"]) self.assertSortedEqual(list(item.values()), ["New"]) + def test_fields_order(self): + class TestItem(Item): + name = Field() + keys = Field() + values = Field() + + assert list(TestItem.fields) == ["name", "keys", "values"] + + def test_fields_order_inheritance(self): + class ParentItem(Item): + name = Field() + keys = Field() + values = Field() + + class TestItem(ParentItem): + extra = Field() + keys = Field(serializer=str) + + # Inherited fields come first, in their definition order, followed by + # the fields newly defined in the subclass. A redefined field keeps the + # position of its first definition while taking the new metadata. + assert list(TestItem.fields) == ["name", "keys", "values", "extra"] + assert TestItem.fields["keys"] == {"serializer": str} + def test_metaclass_inheritance(self): class ParentItem(Item): name = Field() From fc5216f15611e40d795f5ab566acff8f9ca0af1e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 1 Jul 2026 11:50:32 +0500 Subject: [PATCH 04/65] Clarify/cleanup Selector.type (#7704) --- scrapy/selector/unified.py | 47 ++++++++++++++------------------------ 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 99b22aca9..f6334c32c 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -1,10 +1,6 @@ -""" -XPath selectors based on lxml -""" - from __future__ import annotations -from typing import Any +from typing import Any, Literal from parsel import Selector as _ParselSelector @@ -18,13 +14,10 @@ __all__ = ["Selector", "SelectorList"] _NOT_SET = object() -def _st(response: TextResponse | None, st: str | None) -> str: - if st is None: - return "xml" if isinstance(response, XmlResponse) else "html" - return st +SelectorType = Literal["html", "xml", "json", "text"] -def _response_from_text(text: str | bytes, st: str | None) -> TextResponse: +def _response_from_text(text: str | bytes, st: SelectorType | None) -> TextResponse: rt: type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8")) @@ -49,23 +42,16 @@ class Selector(_ParselSelector, object_ref): ``response`` isn't available. Using ``text`` and ``response`` together is undefined behavior. - ``type`` defines the selector type, it can be ``"html"``, ``"xml"``, ``"json"`` - or ``None`` (default). + ``type`` defines the selector type, it can be ``"html"``, ``"xml"``, + ``"json"``, ``"text"`` or ``None`` (default). It's passed to + :class:`parsel.Selector` and its meaning is defined there. However, when + ``type`` is ``None``, it is set to ``"xml"`` for an + :class:`~scrapy.http.XmlResponse` and to ``"html"`` otherwise before + passing it to :class:`parsel.Selector`. - If ``type`` is ``None``, the selector automatically chooses the best type - based on ``response`` type (see below), or defaults to ``"html"`` in case it - is used together with ``text``. - - If ``type`` is ``None`` and a ``response`` is passed, the selector type is - inferred from the response type as follows: - - * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type - * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type - * ``"json"`` for :class:`~scrapy.http.TextResponse` type - * ``"html"`` for anything else - - Otherwise, if ``type`` is set, the selector type will be forced and no - detection will occur. + .. note:: JSON selector support requires ``parsel`` 1.8.0 or higher. With + older versions setting ``type`` to ``"json"`` or ``"text"`` is not + supported. """ __slots__ = ["response"] @@ -75,7 +61,7 @@ class Selector(_ParselSelector, object_ref): self, response: TextResponse | None = None, text: str | None = None, - type: str | None = None, # noqa: A002 + type: SelectorType | None = None, # noqa: A002 root: Any | None = _NOT_SET, **kwargs: Any, ): @@ -84,10 +70,11 @@ class Selector(_ParselSelector, object_ref): f"{self.__class__.__name__}.__init__() received both response and text" ) - st = _st(response, type) + if type is None: + type = "xml" if isinstance(response, XmlResponse) else "html" # noqa: A001 if text is not None: - response = _response_from_text(text, st) + response = _response_from_text(text, type) if response is not None: text = response.text @@ -98,4 +85,4 @@ class Selector(_ParselSelector, object_ref): if root is not _NOT_SET: kwargs["root"] = root - super().__init__(text=text, type=st, **kwargs) + super().__init__(text=text, type=type, **kwargs) From 361f689df785959a59cf939b9efaefc72079f037 Mon Sep 17 00:00:00 2001 From: Adrian Date: Wed, 1 Jul 2026 08:53:07 +0200 Subject: [PATCH 05/65] Improve test coverage for crawler.py (#7682) * Improve test coverage for crawler.py * Silence mypy warnings * Improve test coverage for crawler.py --- ...yncio_enabled_reactor_same_loop_default.py | 31 ++++ .../dns_resolver_deprecated.py | 31 ++++ .../reactorless_sleeping.py | 2 +- tests/AsyncCrawlerProcess/sleeping.py | 2 +- .../CrawlerProcess/dns_resolver_deprecated.py | 31 ++++ tests/CrawlerProcess/sleeping.py | 2 +- tests/test_crawler.py | 163 +++++++++++++++++- tests/test_crawler_subprocess.py | 30 +++- 8 files changed, 286 insertions(+), 6 deletions(-) create mode 100644 tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop_default.py create mode 100644 tests/AsyncCrawlerProcess/dns_resolver_deprecated.py create mode 100644 tests/CrawlerProcess/dns_resolver_deprecated.py diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop_default.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop_default.py new file mode 100644 index 000000000..c519e123b --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop_default.py @@ -0,0 +1,31 @@ +import asyncio +import sys + +from twisted.internet import asyncioreactor + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + +if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +loop = asyncio.SelectorEventLoop() +asyncio.set_event_loop(loop) +asyncioreactor.install(loop) + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "asyncio.SelectorEventLoop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/dns_resolver_deprecated.py b/tests/AsyncCrawlerProcess/dns_resolver_deprecated.py new file mode 100644 index 000000000..8c8df96bb --- /dev/null +++ b/tests/AsyncCrawlerProcess/dns_resolver_deprecated.py @@ -0,0 +1,31 @@ +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.settings import Settings + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +settings = Settings() +# The deprecated DNS_RESOLVER setting, set above its default priority so that +# AsyncCrawlerProcess._setup_reactor() emits the deprecation warning. +settings.set("DNS_RESOLVER", "scrapy.resolver.CachingThreadedResolver", priority=10) +if len(sys.argv) > 1 and sys.argv[1] == "twisted-wins": + # TWISTED_DNS_RESOLVER at a higher priority takes precedence over the + # deprecated DNS_RESOLVER setting. + settings.set( + "TWISTED_DNS_RESOLVER", + "scrapy.resolver.CachingThreadedResolver", + priority=20, + ) + +process = AsyncCrawlerProcess(settings) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactorless_sleeping.py b/tests/AsyncCrawlerProcess/reactorless_sleeping.py index 12101d221..0d11a0996 100644 --- a/tests/AsyncCrawlerProcess/reactorless_sleeping.py +++ b/tests/AsyncCrawlerProcess/reactorless_sleeping.py @@ -17,4 +17,4 @@ class SleepingSpider(scrapy.Spider): process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False}) process.crawl(SleepingSpider) -process.start() +process.start(stop_after_crawl="--no-stop" not in sys.argv) diff --git a/tests/AsyncCrawlerProcess/sleeping.py b/tests/AsyncCrawlerProcess/sleeping.py index 88caf5032..dad6a3f20 100644 --- a/tests/AsyncCrawlerProcess/sleeping.py +++ b/tests/AsyncCrawlerProcess/sleeping.py @@ -17,4 +17,4 @@ class SleepingSpider(scrapy.Spider): process = AsyncCrawlerProcess(settings={}) process.crawl(SleepingSpider) -process.start() +process.start(stop_after_crawl="--no-stop" not in sys.argv) diff --git a/tests/CrawlerProcess/dns_resolver_deprecated.py b/tests/CrawlerProcess/dns_resolver_deprecated.py new file mode 100644 index 000000000..b8cd24325 --- /dev/null +++ b/tests/CrawlerProcess/dns_resolver_deprecated.py @@ -0,0 +1,31 @@ +import sys + +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy.settings import Settings + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +settings = Settings() +# The deprecated DNS_RESOLVER setting, set above its default priority so that +# CrawlerProcess._setup_reactor() emits the deprecation warning. +settings.set("DNS_RESOLVER", "scrapy.resolver.CachingThreadedResolver", priority=10) +if len(sys.argv) > 1 and sys.argv[1] == "twisted-wins": + # TWISTED_DNS_RESOLVER at a higher priority takes precedence over the + # deprecated DNS_RESOLVER setting. + settings.set( + "TWISTED_DNS_RESOLVER", + "scrapy.resolver.CachingThreadedResolver", + priority=20, + ) + +process = CrawlerProcess(settings) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py index cb8f869e1..a577b1909 100644 --- a/tests/CrawlerProcess/sleeping.py +++ b/tests/CrawlerProcess/sleeping.py @@ -23,4 +23,4 @@ class SleepingSpider(scrapy.Spider): process = CrawlerProcess(settings={}) process.crawl(SleepingSpider) -process.start() +process.start(stop_after_crawl="--no-stop" not in sys.argv) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0cddfd0ed..adac32df1 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -3,8 +3,11 @@ from __future__ import annotations import asyncio import logging import re +import signal +import threading from pathlib import Path -from typing import Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar +from unittest.mock import MagicMock import pytest from zope.interface.exceptions import MultipleInvalid @@ -32,6 +35,9 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from collections.abc import Callable + BASE_SETTINGS: dict[str, Any] = {} @@ -651,6 +657,145 @@ class TestAsyncCrawlerProcess(TestBaseCrawler): self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") +class TestAsyncCrawlerProcessReactorlessHelpers: + """Unit tests for the reactorless shutdown helpers of AsyncCrawlerProcess. + + These cover defensive branches that guard against shutdown races and that + are not reachable through a full process run. + """ + + @staticmethod + def _bare_process( + monkeypatch: pytest.MonkeyPatch, + ) -> tuple[AsyncCrawlerProcess, list[Any]]: + # AsyncCrawlerProcess.__init__ has global side effects (it installs a + # reactor import hook and an asyncio event loop), so build a bare + # instance and set only the attributes these helpers read. The shutdown + # handlers installed by these helpers are recorded for assertions + # instead of touching the real process-wide signal handlers. + installed_handlers: list[Any] = [] + monkeypatch.setattr( + "scrapy.crawler.install_shutdown_handlers", + lambda handler, *args, **kwargs: installed_handlers.append(handler), + ) + return AsyncCrawlerProcess.__new__(AsyncCrawlerProcess), installed_handlers + + @staticmethod + def _run_in_thread(target: Callable[[], None]) -> None: + # Run target in a dedicated thread so its event loop is not nested + # inside the event loop that may already be running the test session. + thread = threading.Thread(target=target) + thread.start() + thread.join() + + def test_signal_shutdown_reactorless_without_loop( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + process, installed_handlers = self._bare_process(monkeypatch) + process._reactorless_loop = None + # No loop to schedule the shutdown task on, so it returns early, but it + # must still escalate the handler so a second signal forces a kill. + process._signal_shutdown_reactorless(signal.SIGINT, None) + assert installed_handlers == [process._signal_kill_reactorless] + + def test_signal_kill_reactorless_without_loop( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + process, installed_handlers = self._bare_process(monkeypatch) + process._reactorless_loop = None + process._reactorless_main_task = None + # No loop to cancel the main task on, so it returns early, but it must + # still ignore any further signals. + process._signal_kill_reactorless(signal.SIGINT, None) + assert installed_handlers == [signal.SIG_IGN] + + def test_signal_kill_reactorless_without_main_task( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + process, installed_handlers = self._bare_process(monkeypatch) + loop = MagicMock() + process._reactorless_loop = loop + process._reactorless_main_task = None + # No main task to cancel, so nothing is scheduled on the loop. + process._signal_kill_reactorless(signal.SIGINT, None) + assert installed_handlers == [signal.SIG_IGN] + loop.call_soon_threadsafe.assert_not_called() + + def test_shutdown_graceful_reactorless_main_task_already_done( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + process, _ = self._bare_process(monkeypatch) + process._stop_after_crawl = False + + async def noop() -> None: + return None + + monkeypatch.setattr(process, "stop", noop) + monkeypatch.setattr(process, "join", noop) + + def run() -> None: + loop = asyncio.new_event_loop() + try: + main_task: asyncio.Future[None] = loop.create_future() + main_task.set_result(None) + process._reactorless_main_task = main_task + # The main task is already done, so it is not cancelled. + loop.run_until_complete(process._shutdown_graceful_reactorless()) + assert not main_task.cancelled() + finally: + loop.close() + + self._run_in_thread(run) + + def test_create_shutdown_task_closed_loop( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + process, _ = self._bare_process(monkeypatch) + loop = asyncio.new_event_loop() + loop.close() + process._reactorless_loop = loop + process._stop_after_crawl = True + # create_task() raises RuntimeError on a closed loop; the coroutine + # must be closed instead of leaking. + process._create_shutdown_task() + + def test_cancel_all_tasks_logs_task_exception(self) -> None: + contexts: list[dict[str, Any]] = [] + task_was_cancelled: list[bool] = [] + + def run() -> None: + loop = asyncio.new_event_loop() + loop.set_exception_handler(lambda _loop, context: contexts.append(context)) + + async def fail_on_cancel() -> None: + try: + await asyncio.sleep(10) + except asyncio.CancelledError: + raise RuntimeError("boom") + + try: + task = loop.create_task(fail_on_cancel()) + # Let the task start and suspend on the sleep so the + # cancellation is raised inside its body and turned into a + # RuntimeError rather than cancelling the task cleanly. + loop.run_until_complete(asyncio.sleep(0)) + AsyncCrawlerProcess._cancel_all_tasks(loop) + task_was_cancelled.append(task.cancelled()) + finally: + loop.close() + + self._run_in_thread(run) + + # The task raised instead of being cancelled, so its exception is + # reported to the loop exception handler. + assert task_was_cancelled == [False] + assert any( + context.get("message") + == "unhandled exception during AsyncCrawlerProcess shutdown" + for context in contexts + ) + + @pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) def test_runner_settings_applied_to_crawler_instance( runner_cls: type[CrawlerRunnerBase], @@ -687,6 +832,22 @@ def test_create_crawler_instance_consistent_with_spider_class() -> None: assert pre_built.settings["FOO"] == "runner" +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_create_crawler_rejects_spider_object( + runner_cls: type[CrawlerRunnerBase], +) -> None: + runner = runner_cls() + with pytest.raises(ValueError, match="cannot be a spider object"): + runner.create_crawler(DefaultSpider()) # type: ignore[arg-type] + + +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None: + runner = runner_cls() + with pytest.raises(ValueError, match="cannot be a spider object"): + runner.crawl(DefaultSpider()) # type: ignore[arg-type] + + class ExceptionSpider(scrapy.Spider): name = "exception" diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index e3f9ac161..018a2b31b 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -126,6 +126,16 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "TimeoutError" not in log assert "scrapy.exceptions.CannotResolveHostError" not in log + def test_dns_resolver_deprecated(self) -> None: + log = self.run_script("dns_resolver_deprecated.py") + assert "Spider closed (finished)" in log + assert "The DNS_RESOLVER setting is deprecated" in log + + def test_dns_resolver_deprecated_twisted_dns_resolver(self) -> None: + log = self.run_script("dns_resolver_deprecated.py", "twisted-wins") + assert "Spider closed (finished)" in log + assert "The DNS_RESOLVER setting is deprecated" in log + def test_twisted_reactor_asyncio(self) -> None: log = self.run_script("twisted_reactor_asyncio.py") assert "Spider closed (finished)" in log @@ -205,9 +215,11 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "Spider closed (finished)" in log assert "The value of FOO is 42" in log - def _test_shutdown_graceful(self, script: str = "sleeping.py") -> None: + def _test_shutdown_graceful( + self, script: str = "sleeping.py", *extra_args: str + ) -> None: sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK # type: ignore[attr-defined] - args = self.get_script_args(script, "3") + args = self.get_script_args(script, "3", *extra_args) p = PopenSpawn(args, timeout=5, env=get_script_run_env()) p.expect_exact("Spider opened") p.expect_exact("Crawled (200)") @@ -245,6 +257,9 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): async def test_shutdown_forced(self) -> None: await self._test_shutdown_forced() + def test_shutdown_graceful_no_stop(self) -> None: + self._test_shutdown_graceful("sleeping.py", "--no-stop") + class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): @property @@ -429,6 +444,17 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): async def test_shutdown_forced(self) -> None: await self._test_shutdown_forced("reactorless_sleeping.py") + def test_shutdown_graceful_reactorless_no_stop(self) -> None: + self._test_shutdown_graceful("reactorless_sleeping.py", "--no-stop") + + def test_asyncio_enabled_reactor_same_loop_default(self) -> None: + log = self.run_script("asyncio_enabled_reactor_same_loop_default.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): """Common tests between CrawlerRunner and AsyncCrawlerRunner, From 870803b7fb1ed56c296eb6b51d0212f0460dfdda Mon Sep 17 00:00:00 2001 From: Fat-Coder-CN Date: Wed, 1 Jul 2026 15:16:01 +0800 Subject: [PATCH 06/65] fix-utf16-response-test-on-big-endian-systems (#7508) --- tests/test_http_response_text.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py index 507fd1864..5ef89fe4a 100644 --- a/tests/test_http_response_text.py +++ b/tests/test_http_response_text.py @@ -163,12 +163,13 @@ class TestTextResponse(TestResponse): def test_utf16(self): """Test utf-16 because UnicodeDammit is known to have problems with""" + body = b"\xff\xfeh\x00i\x00" r = self.response_class( "http://www.example.com", - body=b"\xff\xfeh\x00i\x00", + body=body, encoding="utf-16", ) - self._assert_response_values(r, "utf-16", "hi") + self._assert_response_values(r, "utf-16", body) def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self): r6 = self.response_class( From dd10cb8e9a982fe3d311078d6e1207596e272717 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 5 Jul 2026 13:46:22 +0200 Subject: [PATCH 07/65] LxmlLinkExtractor: add deny_attrs and deny_tags (#7679) --- docs/topics/link-extractors.rst | 103 +--------------------- scrapy/linkextractors/lxmlhtml.py | 142 +++++++++++++++++++++++++++++- tests/test_linkextractors.py | 53 +++++++++++ 3 files changed, 194 insertions(+), 104 deletions(-) diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 613e175da..3fc896507 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -47,108 +47,7 @@ LxmlLinkExtractor :synopsis: lxml's HTMLParser-based link extractors -.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, strip=True) - - LxmlLinkExtractor is the recommended link extractor with handy filtering - options. It is implemented using lxml's robust HTMLParser. - - :param allow: a single regular expression (or list of regular expressions) - that the (absolute) urls must match in order to be extracted. If not - given (or empty), it will match all links. - :type allow: str or list - - :param deny: a single regular expression (or list of regular expressions) - that the (absolute) urls must match in order to be excluded (i.e. not - extracted). It has precedence over the ``allow`` parameter. If not - given (or empty) it won't exclude any links. - :type deny: str or list - - :param allow_domains: a single value or a list of string containing - domains which will be considered for extracting the links - :type allow_domains: str or list - - :param deny_domains: a single value or a list of strings containing - domains which won't be considered for extracting the links - :type deny_domains: str or list - - :param deny_extensions: a single value or list of strings containing - extensions that should be ignored when extracting links. - If not given, it will default to - :data:`scrapy.linkextractors.IGNORED_EXTENSIONS`. - - :type deny_extensions: list - - :param restrict_xpaths: is an XPath (or list of XPath's) which defines - regions inside the response where links should be extracted from. - If given, only the text selected by those XPath will be scanned for - links. - :type restrict_xpaths: str or list - - :param restrict_css: a CSS selector (or list of selectors) which defines - regions inside the response where links should be extracted from. - Has the same behaviour as ``restrict_xpaths``. - :type restrict_css: str or list - - :param restrict_text: a single regular expression (or list of regular expressions) - that the link's text must match in order to be extracted. If not - given (or empty), it will match all links. If a list of regular expressions is - given, the link will be extracted if it matches at least one. - :type restrict_text: str or list - - :param tags: a tag or a list of tags to consider when extracting links. - Defaults to ``('a', 'area')``. - :type tags: str or list - - :param attrs: an attribute or list of attributes which should be considered when looking - for links to extract (only for those tags specified in the ``tags`` - parameter). Defaults to ``('href',)`` - :type attrs: list - - :param canonicalize: canonicalize each extracted url (using - w3lib.url.canonicalize_url). Defaults to ``False``. - Note that canonicalize_url is meant for duplicate checking; - it can change the URL visible at server side, so the response can be - different for requests with canonicalized and raw URLs. If you're - using LinkExtractor to follow links it is more robust to - keep the default ``canonicalize=False``. - :type canonicalize: bool - - :param unique: whether duplicate filtering should be applied to extracted - links. - :type unique: bool - - :param process_value: a function which receives each value extracted from - the tag and attributes scanned and can modify the value and return a - new one, or return ``None`` to ignore the link altogether. If not - given, ``process_value`` defaults to ``lambda x: x``. - - .. highlight:: html - - For example, to extract links from this code:: - - Link text - - .. highlight:: python - - You can use the following function in ``process_value``: - - .. code-block:: python - - def process_value(value): - m = re.search(r"javascript:goToPage\('(.*?)'", value) - if m: - return m.group(1) - - :type process_value: collections.abc.Callable - - :param strip: whether to strip whitespaces from extracted attributes. - According to HTML5 standard, leading and trailing whitespaces - must be stripped from ``href`` attributes of ````, ```` - and many other elements, ``src`` attribute of ````, ``