From 0bbfca6c1d1327a9919f2c0efc0c75aadb5b0033 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 10 Mar 2025 13:15:28 +0500 Subject: [PATCH 1/9] Better fix for test_non_pickable_object on Windows. --- tests/test_squeues.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 0b6ed8e11..21bbeece2 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -130,9 +130,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin): ) as exc_info: q.push(sel) assert isinstance(exc_info.value.__context__, TypeError) - # This seems to help with https://github.com/scrapy/queuelib/issues/70. - # It will need to remain under a queuelib version check after that bug is fixed. - del exc_info + q.close() class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): From bee74fb753afa373ecdb1dbf272af9a108b14b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 11:56:44 +0100 Subject: [PATCH 2/9] Remove trailing whitespace --- .github/workflows/publish.yml | 2 +- .pre-commit-config.yaml | 4 + README.rst | 2 +- docs/_tests/quotes.html | 138 +++++++++++++++---------------- docs/_tests/quotes1.html | 138 +++++++++++++++---------------- docs/faq.rst | 2 +- docs/intro/install.rst | 4 +- docs/intro/tutorial.rst | 6 +- docs/news.rst | 8 +- docs/topics/api.rst | 2 +- docs/topics/architecture.rst | 4 +- docs/topics/exporters.rst | 2 +- docs/topics/extensions.rst | 16 ++-- docs/topics/feed-exports.rst | 10 +-- docs/topics/logging.rst | 10 +-- docs/topics/media-pipeline.rst | 14 ++-- docs/topics/practices.rst | 2 +- docs/topics/request-response.rst | 6 +- docs/topics/selectors.rst | 6 +- docs/topics/spiders.rst | 4 +- docs/topics/stats.rst | 2 +- sep/sep-004.rst | 2 +- sep/sep-007.rst | 2 +- sep/sep-008.rst | 4 +- sep/sep-014.rst | 4 +- sep/sep-018.rst | 4 +- 26 files changed, 201 insertions(+), 197 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8e01ffd88..d1589f4f7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.13" - - run: | + - run: | python -m pip install --upgrade build python -m build - name: Publish to PyPI diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 18402b908..0d1a76247 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,3 +11,7 @@ repos: - id: blacken-docs additional_dependencies: - black==24.10.0 +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace diff --git a/README.rst b/README.rst index 3f468953e..cf7c6043c 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,6 @@ .. image:: https://scrapy.org/img/scrapylogo.png :target: https://scrapy.org/ - + ====== Scrapy ====== diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html index f4002ecd1..d1cfd9020 100644 --- a/docs/_tests/quotes.html +++ b/docs/_tests/quotes.html @@ -16,13 +16,13 @@

- + Login - +

- +
@@ -34,16 +34,16 @@
Tags: - - + + change - + deep-thoughts - + thinking - + world - +
@@ -54,12 +54,12 @@
Tags: - - + + abilities - + choices - +
@@ -70,18 +70,18 @@
Tags: - - + + inspirational - + life - + live - + miracle - + miracles - +
@@ -92,16 +92,16 @@
Tags: - - + + aliteracy - + books - + classic - + humor - +
@@ -112,12 +112,12 @@
Tags: - - + + be-yourself - + inspirational - +
@@ -128,14 +128,14 @@
Tags: - - + + adulthood - + success - + value - +
@@ -146,12 +146,12 @@
Tags: - - + + life - + love - +
@@ -162,16 +162,16 @@
Tags: - - + + edison - + failure - + inspirational - + paraphrased - +
@@ -182,10 +182,10 @@
Tags: - - + + misattributed-eleanor-roosevelt - +
@@ -196,73 +196,73 @@
Tags: - - + + humor - + obvious - + simile - +
- +

Top Ten tags

- + love - + inspirational - + life - + humor - + books - + reading - + friendship - + friends - + truth - + simile - - + +
diff --git a/docs/_tests/quotes1.html b/docs/_tests/quotes1.html index f4002ecd1..d1cfd9020 100644 --- a/docs/_tests/quotes1.html +++ b/docs/_tests/quotes1.html @@ -16,13 +16,13 @@

- + Login - +

- +
@@ -34,16 +34,16 @@
Tags: - - + + change - + deep-thoughts - + thinking - + world - +
@@ -54,12 +54,12 @@
Tags: - - + + abilities - + choices - +
@@ -70,18 +70,18 @@
Tags: - - + + inspirational - + life - + live - + miracle - + miracles - +
@@ -92,16 +92,16 @@
Tags: - - + + aliteracy - + books - + classic - + humor - +
@@ -112,12 +112,12 @@
Tags: - - + + be-yourself - + inspirational - +
@@ -128,14 +128,14 @@
Tags: - - + + adulthood - + success - + value - +
@@ -146,12 +146,12 @@
Tags: - - + + life - + love - +
@@ -162,16 +162,16 @@
Tags: - - + + edison - + failure - + inspirational - + paraphrased - +
@@ -182,10 +182,10 @@
Tags: - - + + misattributed-eleanor-roosevelt - +
@@ -196,73 +196,73 @@
Tags: - - + + humor - + obvious - + simile - +
- +

Top Ten tags

- + love - + inspirational - + life - + humor - + books - + reading - + friendship - + friends - + truth - + simile - - + +
diff --git a/docs/faq.rst b/docs/faq.rst index f81ec3601..cef3e69f3 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -410,7 +410,7 @@ How can I make a blank request? ------------------------------- .. code-block:: python - + from scrapy import Request diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 82a0e18c5..488a66f36 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -111,7 +111,7 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: To install Scrapy on Windows using ``pip``: .. warning:: - This installation method requires “Microsoft Visual C++” for installing some + This installation method requires “Microsoft Visual C++” for installing some Scrapy dependencies, which demands significantly more disk space than Anaconda. #. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer. @@ -123,7 +123,7 @@ To install Scrapy on Windows using ``pip``: #. Check the installation details and make sure following packages are selected as optional components: * **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) ) - + * **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0)) #. Install the Visual Studio Build Tools. diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 6e6caebf1..5041b49ea 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -292,7 +292,7 @@ As an alternative, you could've written: >>> response.css("title::text")[0].get() 'Quotes to Scrape' -Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will +Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will raise an :exc:`IndexError` exception if there are no results: .. code-block:: pycon @@ -302,8 +302,8 @@ raise an :exc:`IndexError` exception if there are no results: ... IndexError: list index out of range -You might want to use ``.get()`` directly on the -:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` +You might want to use ``.get()`` directly on the +:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` if there are no results: .. code-block:: pycon diff --git a/docs/news.rst b/docs/news.rst index 8230c3aef..9a68f8852 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -934,10 +934,10 @@ Modified requirements Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` - instead of ``False``. In earlier Scrapy versions empty files were created - even when this setting was ``False`` (which was a bug that is now fixed), - so the new default should keep the old behavior. (:issue:`872`, +- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` + instead of ``False``. In earlier Scrapy versions empty files were created + even when this setting was ``False`` (which was a bug that is now fixed), + so the new default should keep the old behavior. (:issue:`872`, :issue:`5847`) Deprecation removals diff --git a/docs/topics/api.rst b/docs/topics/api.rst index f7cffb61b..edc625be8 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -88,7 +88,7 @@ how you :ref:`configure the downloader middlewares The execution engine, which coordinates the core crawling logic between the scheduler, downloader and spiders. - Some extension may want to access the Scrapy engine, to inspect or + Some extension may want to access the Scrapy engine, to inspect or modify the downloader and scheduler behaviour, although this is an advanced use and this API is not yet stable. diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 0370dc538..4e53b6e3d 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -87,8 +87,8 @@ of the system, and triggering events when certain actions occur. See the Scheduler --------- -The :ref:`scheduler ` receives requests from the engine and -enqueues them for feeding them later (also to the engine) when the engine +The :ref:`scheduler ` receives requests from the engine and +enqueues them for feeding them later (also to the engine) when the engine requests them. .. _component-downloader: diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 7a85c099b..5c078568b 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -224,7 +224,7 @@ BaseItemExporter .. [1] Not all exporters respect the specified field order. .. [2] When using :ref:`item objects ` that do not expose all their possible fields, exporters that do not support exporting - a different subset of fields per item will only export the fields + a different subset of fields per item will only export the fields found in the first item exported. .. attribute:: export_empty_fields diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index c47a3226a..23bbcfcb5 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -256,14 +256,14 @@ Spider state extension Manages spider state data by loading it before a crawl and saving it after. Give a value to the :setting:`JOBDIR` setting to enable this extension. -When enabled, this extension manages the :attr:`~scrapy.Spider.state` +When enabled, this extension manages the :attr:`~scrapy.Spider.state` attribute of your :class:`~scrapy.Spider` instance: - -- When your spider closes (:signal:`spider_closed`), the contents of its - :attr:`~scrapy.Spider.state` attribute are serialized into a file named + +- When your spider closes (:signal:`spider_closed`), the contents of its + :attr:`~scrapy.Spider.state` attribute are serialized into a file named ``spider.state`` in the :setting:`JOBDIR` folder. -- When your spider opens (:signal:`spider_opened`), if a previously-generated - ``spider.state`` file exists in the :setting:`JOBDIR` folder, it is loaded +- When your spider opens (:signal:`spider_opened`), if a previously-generated + ``spider.state`` file exists in the :setting:`JOBDIR` folder, it is loaded into the :attr:`~scrapy.Spider.state` attribute. @@ -291,8 +291,8 @@ settings: .. note:: - When a certain closing condition is met, requests which are - currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS` + When a certain closing condition is met, requests which are + currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS` requests) are still processed. .. setting:: CLOSESPIDER_TIMEOUT diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 07a3f3678..7f401f0c7 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -180,7 +180,7 @@ FTP supports two different connection modes: `active or passive mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. -The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. .. caution:: The value ``True`` in ``overwrite`` will cause you to lose the @@ -222,7 +222,7 @@ feeds using these settings: - :setting:`AWS_ENDPOINT_URL` - :setting:`AWS_REGION_NAME` -The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. .. caution:: The value ``True`` in ``overwrite`` will cause you to lose the @@ -255,7 +255,7 @@ You can set a *Project ID* and *Access Control List (ACL)* through the following - :setting:`FEED_STORAGE_GCS_ACL` - :setting:`GCS_PROJECT_ID` -The default value for the ``overwrite`` key in the :setting:`FEEDS` for this +The default value for the ``overwrite`` key in the :setting:`FEEDS` for this storage backend is: ``True``. .. caution:: The value ``True`` in ``overwrite`` will cause you to lose the @@ -587,8 +587,8 @@ FEED_STORE_EMPTY Default: ``True`` Whether to export empty feeds (i.e. feeds with no items). -If ``False``, and there are no items to export, no new files are created and -existing files are not modified, even if the :ref:`overwrite feed option +If ``False``, and there are no items to export, no new files are created and +existing files are not modified, even if the :ref:`overwrite feed option ` is enabled. .. setting:: FEED_STORAGES diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index fe1c4d162..a398d6c83 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -266,9 +266,9 @@ e.g. in the spider's ``__init__`` method: If you run this spider again then INFO messages from ``scrapy.spidermiddlewares.httperror`` logger will be gone. -You can also filter log records by :class:`~logging.LogRecord` data. For +You can also filter log records by :class:`~logging.LogRecord` data. For example, you can filter log records by message content using a substring or -a regular expression. Create a :class:`logging.Filter` subclass +a regular expression. Create a :class:`logging.Filter` subclass and equip it with a regular expression pattern to filter out unwanted messages: @@ -284,8 +284,8 @@ filter out unwanted messages: if match: return False -A project-level filter may be attached to the root -handler created by Scrapy, this is a wieldy way to +A project-level filter may be attached to the root +handler created by Scrapy, this is a wieldy way to filter all loggers in different parts of the project (middlewares, spider, etc.): @@ -301,7 +301,7 @@ filter all loggers in different parts of the project for handler in logging.root.handlers: handler.addFilter(ContentFilter()) -Alternatively, you may choose a specific logger +Alternatively, you may choose a specific logger and hide it without affecting other loggers: .. code-block:: python diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index f086a943e..cc1fe8703 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -414,7 +414,7 @@ class name. E.g. given pipeline class called MyPipeline you can set setting key: and pipeline class MyPipeline will have expiration time set to 180. -The last modified time from the file is used to determine the age of the file in days, +The last modified time from the file is used to determine the age of the file in days, which is then compared to the set expiration time to determine if the file is expired. .. _topics-images-thumbnails: @@ -519,7 +519,7 @@ See here the methods that you can override in your custom Files Pipeline: In addition to ``response``, this method receives the original :class:`request `, - :class:`info ` and + :class:`info ` and :class:`item ` You can override this method to customize the download path of each file. @@ -541,9 +541,9 @@ See here the methods that you can override in your custom Files Pipeline: def file_path(self, request, response=None, info=None, *, item=None): return "files/" + PurePosixPath(urlparse_cached(request).path).name - Similarly, you can use the ``item`` to determine the file path based on some item + Similarly, you can use the ``item`` to determine the file path based on some item property. - + By default the :meth:`file_path` method returns ``full/.``. @@ -677,7 +677,7 @@ See here the methods that you can override in your custom Images Pipeline: In addition to ``response``, this method receives the original :class:`request `, - :class:`info ` and + :class:`info ` and :class:`item ` You can override this method to customize the download path of each file. @@ -699,9 +699,9 @@ See here the methods that you can override in your custom Images Pipeline: def file_path(self, request, response=None, info=None, *, item=None): return "files/" + PurePosixPath(urlparse_cached(request).path).name - Similarly, you can use the ``item`` to determine the file path based on some item + Similarly, you can use the ``item`` to determine the file path based on some item property. - + By default the :meth:`file_path` method returns ``full/.``. diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 5f6798601..db91cd073 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -309,7 +309,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites: services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. * use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy - plugin `__ and additional + plugin `__ and additional features, like `AI web scraping `__ If you are still unable to prevent your bot getting banned, consider contacting diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index b187f3aaf..55e8518a5 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -1309,7 +1309,7 @@ JsonResponse objects .. class:: JsonResponse(url[, ...]) - The :class:`JsonResponse` class is a subclass of :class:`TextResponse` - that is used when the response has a `JSON MIME type - `_ in its `Content-Type` + The :class:`JsonResponse` class is a subclass of :class:`TextResponse` + that is used when the response has a `JSON MIME type + `_ in its `Content-Type` header. diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index b95e6eab3..dbef07b73 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -559,7 +559,7 @@ For example, suppose you want to extract all ``

`` elements inside ``

`` elements. First, you would get all ``
`` elements: .. code-block:: pycon - + >>> divs = response.xpath("//div") At first, you may be tempted to use the following approach, which is wrong, as @@ -610,7 +610,7 @@ As it turns out, Scrapy selectors allow you to chain selectors, so most of the t you can just select by class using CSS and then switch to XPath when needed: .. code-block:: pycon - + >>> from scrapy import Selector >>> sel = Selector( ... text='
' @@ -1032,7 +1032,7 @@ whereas the CSS lookup is translated into XPath and thus runs more efficiently, so performance-wise its uses are limited to situations that are not easily described with CSS selectors. -Parsel also simplifies adding your own XPath extensions with +Parsel also simplifies adding your own XPath extensions with :func:`~parsel.xpathfuncs.set_xpathfunc`. .. _topics-selectors-ref: diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index e1b1c5ad6..0a67240d6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -379,8 +379,8 @@ The above example can also be written as follows: def start_requests(self): yield scrapy.Request(f"http://www.example.com/categories/{self.category}") -If you are :ref:`running Scrapy from a script `, you can -specify spider arguments when calling +If you are :ref:`running Scrapy from a script `, you can +specify spider arguments when calling :class:`CrawlerProcess.crawl ` or :class:`CrawlerRunner.crawl `: diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index be8ecb7a5..9572a3785 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -86,7 +86,7 @@ Available Stats Collectors Besides the basic :class:`StatsCollector` there are other Stats Collectors available in Scrapy which extend the basic Stats Collector. You can select which Stats Collector to use through the :setting:`STATS_CLASS` setting. The -default Stats Collector used is the :class:`MemoryStatsCollector`. +default Stats Collector used is the :class:`MemoryStatsCollector`. .. currentmodule:: scrapy.statscollectors diff --git a/sep/sep-004.rst b/sep/sep-004.rst index b1cef2600..7a4ebe886 100644 --- a/sep/sep-004.rst +++ b/sep/sep-004.rst @@ -11,7 +11,7 @@ SEP-004: Library API ==================== .. note:: the library API has been implemented, but slightly different from proposed in this SEP. You can run a Scrapy crawler inside a Twisted - reactor, but not outside it. + reactor, but not outside it. Introduction ============ diff --git a/sep/sep-007.rst b/sep/sep-007.rst index 0ca2036ce..73ce0d338 100644 --- a/sep/sep-007.rst +++ b/sep/sep-007.rst @@ -96,7 +96,7 @@ specified, else utf-8 is used) and returns a new unicode object. E.g: ``clean_spaces`` ---------------- - + Converts multispaces into single spaces for the given string. E.g: :: diff --git a/sep/sep-008.rst b/sep/sep-008.rst index be5987e39..1c38b1c40 100644 --- a/sep/sep-008.rst +++ b/sep/sep-008.rst @@ -73,8 +73,8 @@ Alternative Public API Proposal - ``ItemLoader.get_stored_values()`` or ``ItemLoader.get_values()`` *(returns the ``ItemLoader values)* - ``ItemLoader.get_output_value()`` -- ``ItemLoader.get_input_processor()`` or ``ItemLoader.get_in_processor()`` *(short version)* -- ``ItemLoader.get_output_processor()`` or ``ItemLoader.get_out_processor()`` *(short version)* +- ``ItemLoader.get_input_processor()`` or ``ItemLoader.get_in_processor()`` *(short version)* +- ``ItemLoader.get_output_processor()`` or ``ItemLoader.get_out_processor()`` *(short version)* - ``ItemLoader.context`` diff --git a/sep/sep-014.rst b/sep/sep-014.rst index e03a2b0f6..0a2e6b51e 100644 --- a/sep/sep-014.rst +++ b/sep/sep-014.rst @@ -21,7 +21,7 @@ Current flaws and inconsistencies 2. Link extractors are inflexible and hard to maintain, link processing/filtering is tightly coupled. (e.g. canonicalize) 3. Isn't possible to crawl an url directly from command line because the Spider - does not know which callback use. + does not know which callback use. These flaws will be corrected by the changes proposed in this SEP. @@ -55,7 +55,7 @@ Request Extractors Request Extractors takes response object and determines which requests follow. This is an enhancement to ``LinkExtractors`` which returns urls (links), -Request Extractors return Request objects. +Request Extractors return Request objects. Request Processors ------------------ diff --git a/sep/sep-018.rst b/sep/sep-018.rst index 13ab501ed..e6d601fe1 100644 --- a/sep/sep-018.rst +++ b/sep/sep-018.rst @@ -200,7 +200,7 @@ the same spider: # extract item from response return item -The Spider Middleware that implements spider code +The Spider Middleware that implements spider code ================================================= There's gonna be one middleware that will take care of calling the proper @@ -625,7 +625,7 @@ Resolved: not the original one (think of redirections), but it does carry the ``meta`` of the original one. The original one may not be available anymore (in memory) if we're using a persistent scheduler., but in that case it would be - the deserialized request from the persistent scheduler queue. + the deserialized request from the persistent scheduler queue. - No - this would make implementation more complex and we're not sure it's really needed From 9b7db1a068895254aae1618fb684baf9cb0c2784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 12:43:50 +0100 Subject: [PATCH 3/9] Move some reference docs of Request to the code (#6721) --- docs/topics/request-response.rst | 75 +++++++++++++------------------- scrapy/http/request/__init__.py | 49 +++++++++++++++++++++ 2 files changed, 80 insertions(+), 44 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 55e8518a5..0375e0ff1 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -31,23 +31,12 @@ Request objects If the URL is invalid, a :exc:`ValueError` exception is raised. :type url: str - :param callback: the function that will be called with the response of this - request (once it's downloaded) as its first parameter. + :param callback: sets :attr:`callback`, defaults to ``None``. - In addition to a function, the following values are supported: - - - ``None`` (default), which indicates that the spider's - :meth:`~scrapy.Spider.parse` method must be used. - - - :func:`~scrapy.http.request.NO_CALLBACK` - - For more information, see - :ref:`topics-request-response-ref-request-callback-arguments`. - - .. note:: If exceptions are raised during processing, ``errback`` is - called instead. - - :type callback: collections.abc.Callable + .. versionchanged:: 2.0 + The *callback* parameter is no longer required when the *errback* + parameter is specified. + :type callback: Callable[Concatenate[Response, ...], Any] | None :param method: the HTTP method of this request. Defaults to ``'GET'``. :type method: str @@ -144,23 +133,15 @@ Request objects Negative values are allowed in order to indicate relatively low-priority. :type priority: int - :param dont_filter: indicates that this request should not be filtered by - the scheduler or some middlewares. This is used when you want to perform - an identical request multiple times, to ignore the duplicates filter. - Use it with care, or you will get into crawling loops. Default to ``False``. + :param dont_filter: sets :attr:`dont_filter`, defaults to ``False``. :type dont_filter: bool - :param errback: a function that will be called if any exception was - raised while processing the request. This includes pages that failed - with 404 HTTP errors and such. It receives a - :exc:`~twisted.python.failure.Failure` as first parameter. - For more information, - see :ref:`topics-request-response-ref-errbacks` below. + :param errback: sets :attr:`errback`, defaults to ``None``. - .. versionchanged:: 2.0 - The *callback* parameter is no longer required when the *errback* - parameter is specified. - :type errback: collections.abc.Callable + .. versionchanged:: 2.0 + The *callback* parameter is no longer required when the *errback* + parameter is specified. + :type errback: Callable[[Failure], Any] | None :param flags: Flags sent to the request, can be used for logging or similar purposes. :type flags: list @@ -194,6 +175,25 @@ Request objects This attribute is read-only. To change the body of a Request use :meth:`replace`. + .. autoattribute:: callback + + .. autoattribute:: errback + + .. attribute:: Request.cb_kwargs + + A dictionary that contains arbitrary metadata for this request. Its contents + will be passed to the Request's callback as keyword arguments. It is empty + for new Requests, which means by default callbacks only get a + :class:`~scrapy.http.Response` object as argument. + + This dict is :doc:`shallow copied ` when the request is + cloned using the ``copy()`` or ``replace()`` methods, and can also be + accessed, in your spider, from the ``response.cb_kwargs`` attribute. + + In case of a failure to process the request, this dict can be accessed as + ``failure.request.cb_kwargs`` in the request's errback. For more information, + see :ref:`errback-cb_kwargs`. + .. attribute:: Request.meta :value: {} @@ -237,20 +237,7 @@ Request objects Also mind that the :meth:`copy` and :meth:`replace` request methods :doc:`shallow-copy ` request metadata. - .. attribute:: Request.cb_kwargs - - A dictionary that contains arbitrary metadata for this request. Its contents - will be passed to the Request's callback as keyword arguments. It is empty - for new Requests, which means by default callbacks only get a - :class:`~scrapy.http.Response` object as argument. - - This dict is :doc:`shallow copied ` when the request is - cloned using the ``copy()`` or ``replace()`` methods, and can also be - accessed, in your spider, from the ``response.cb_kwargs`` attribute. - - In case of a failure to process the request, this dict can be accessed as - ``failure.request.cb_kwargs`` in the request's errback. For more information, - see :ref:`errback-cb_kwargs`. + .. autoattribute:: dont_filter .. autoattribute:: Request.attributes diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index e24f6874d..6d3b7a926 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -138,11 +138,60 @@ class Request(object_ref): ) if not (callable(errback) or errback is None): raise TypeError(f"errback must be a callable, got {type(errback).__name__}") + + #: :class:`~collections.abc.Callable` to parse the + #: :class:`~scrapy.http.Response` to this request once received. + #: + #: The callable must expect the response as its first parameter, and + #: support any additional keyword arguments set through + #: :attr:`cb_kwargs`. + #: + #: In addition to an arbitrary callable, the following values are also + #: supported: + #: + #: - ``None`` (default), which indicates that the + #: :meth:`~scrapy.Spider.parse` method of the spider must be used. + #: + #: - :func:`~scrapy.http.request.NO_CALLBACK`. + #: + #: If an unhandled exception is raised during request or response + #: processing, i.e. by a :ref:`spider middleware + #: `, :ref:`downloader middleware + #: ` or download handler + #: (:setting:`DOWNLOAD_HANDLERS`), :attr:`errback` is called instead. + #: + #: .. tip:: + #: :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware` + #: raises exceptions for non-2xx responses by default, sending them + #: to the :attr:`errback` instead. + #: + #: .. seealso:: + #: :ref:`topics-request-response-ref-request-callback-arguments` self.callback: CallbackT | None = callback + + #: :class:`~collections.abc.Callable` to handle exceptions raised + #: during request or response processing. + #: + #: The callable must expect a :exc:`~twisted.python.failure.Failure` as + #: its first parameter. + #: + #: .. seealso:: :ref:`topics-request-response-ref-errbacks` self.errback: Callable[[Failure], Any] | None = errback self.cookies: CookiesT = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) + + #: Whether this request may be filtered out by :ref:`components + #: ` that support filtering out requests (``False``, + #: default), or those components should not filter out this request + #: (``True``). + #: + #: This attribute is commonly set to ``True`` to prevent duplicate + #: requests from being filtered out. + #: + #: When defining the start URLs of a spider through + #: :attr:`~scrapy.Spider.start_urls`, this attribute is enabled by + #: default. See :meth:`~scrapy.Spider.start_requests`. self.dont_filter: bool = dont_filter self._meta: dict[str, Any] | None = dict(meta) if meta else None From 26ecc93228bc3dbb9d62daa419cb344ba6f88caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 14:12:11 +0100 Subject: [PATCH 4/9] Run CI only on the main branch, on release branches and on PRs (#6720) --- .github/workflows/checks.yml | 7 ++++++- .github/workflows/tests-macos.yml | 7 ++++++- .github/workflows/tests-ubuntu.yml | 7 ++++++- .github/workflows/tests-windows.yml | 7 ++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a064bf5b2..312af3b2e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,5 +1,10 @@ name: Checks -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index c28a99982..ce0e1a6c2 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,5 +1,10 @@ name: macOS -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 6c7842217..f74575ee1 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,5 +1,10 @@ name: Ubuntu -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 45e4ca157..21d621240 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,5 +1,10 @@ name: Windows -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} From 5a0690c89d718b33bd63c1cd724c50c9ceb809e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 14:52:48 +0100 Subject: [PATCH 5/9] Remove or post-pone the use of itemadapter.is_item, as a potentially expensive call (#6719) --- docs/faq.rst | 14 +++++++------ docs/topics/items.rst | 5 ++--- scrapy/commands/parse.py | 8 ++++---- scrapy/core/engine.py | 9 +-------- scrapy/core/scraper.py | 10 +--------- scrapy/exporters.py | 4 ++-- .../project/module/middlewares.py.tmpl | 2 +- scrapy/utils/serialize.py | 4 ++-- tests/test_crawl.py | 20 ++++++------------- 9 files changed, 27 insertions(+), 49 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index cef3e69f3..da255f29e 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -361,16 +361,18 @@ method for this purpose. For example: from copy import deepcopy - from itemadapter import is_item, ItemAdapter + from itemadapter import ItemAdapter + from scrapy import Request class MultiplyItemsMiddleware: def process_spider_output(self, response, result, spider): - for item in result: - if is_item(item): - adapter = ItemAdapter(item) - for _ in range(adapter["multiply_by"]): - yield deepcopy(item) + for item_or_request in result: + if isinstance(item_or_request, Request): + continue + adapter = ItemAdapter(item) + for _ in range(adapter["multiply_by"]): + yield deepcopy(item) Does Scrapy support IPv6 addresses? ----------------------------------- diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 7cc476863..0365c95b3 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -384,9 +384,8 @@ Supporting All Item Types In code that receives an item, such as methods of :ref:`item pipelines ` or :ref:`spider middlewares `, it is a good practice to use the -:class:`~itemadapter.ItemAdapter` class and the -:func:`~itemadapter.is_item` function to write code that works for -any supported item type. +:class:`~itemadapter.ItemAdapter` class to write code that works for any +supported item type. Other classes related to items ============================== diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 61aea3ee4..c6ed20b3b 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -6,7 +6,7 @@ import json import logging from typing import TYPE_CHECKING, Any, TypeVar, overload -from itemadapter import ItemAdapter, is_item +from itemadapter import ItemAdapter from twisted.internet.defer import Deferred, maybeDeferred from w3lib.url import is_url @@ -211,10 +211,10 @@ class Command(BaseRunSpiderCommand): ) -> tuple[list[Any], list[Request], argparse.Namespace, int, Spider, CallbackT]: items, requests = [], [] for x in spider_output: - if is_item(x): - items.append(x) - elif isinstance(x, Request): + if isinstance(x, Request): requests.append(x) + else: + items.append(x) return items, requests, opts, depth, spider, callback def run_callback( diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 61f444e31..b7a73700b 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -11,7 +11,6 @@ import logging from time import time from typing import TYPE_CHECKING, Any, TypeVar, cast -from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.task import LoopingCall from twisted.python.failure import Failure @@ -194,14 +193,8 @@ class ExecutionEngine: else: if isinstance(request_or_item, Request): self.crawl(request_or_item) - elif is_item(request_or_item): - self.scraper.start_itemproc(request_or_item, response=None) else: - logger.error( - f"Got {request_or_item!r} among start requests. Only " - f"requests and items are supported. It will be " - f"ignored." - ) + self.scraper.start_itemproc(request_or_item, response=None) if self.spider_is_idle() and self.slot.close_if_idle: self._spider_idle() diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 03301717d..b664b61f6 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -8,7 +8,6 @@ from collections import deque from collections.abc import AsyncIterable, Iterator from typing import TYPE_CHECKING, Any, TypeVar, Union, cast -from itemadapter import is_item from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure @@ -298,17 +297,10 @@ class Scraper: if isinstance(output, Request): assert self.crawler.engine is not None # typing self.crawler.engine.crawl(request=output) - elif is_item(output): - return self.start_itemproc(output, response=response) elif output is None: pass else: - typename = type(output).__name__ - logger.error( - "Spider must return request, item, or None, got %(typename)r in %(request)s", - {"request": request, "typename": typename}, - extra={"spider": spider}, - ) + return self.start_itemproc(output, response=response) return None def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[Any]: diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 46c6aa3fa..0a641752e 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -356,12 +356,12 @@ class PythonItemExporter(BaseItemExporter): def _serialize_value(self, value: Any) -> Any: if isinstance(value, Item): return self.export_item(value) + if isinstance(value, (str, bytes)): + return to_unicode(value, encoding=self.encoding) if is_item(value): return dict(self._serialize_item(value)) if is_listlike(value): return [self._serialize_value(v) for v in value] - if isinstance(value, (str, bytes)): - return to_unicode(value, encoding=self.encoding) return value def _serialize_item(self, item: Any) -> Iterable[tuple[str | bytes, Any]]: diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 8c9a86dce..dcb2d63de 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -6,7 +6,7 @@ from scrapy import signals # useful for handling different item types with a single interface -from itemadapter import is_item, ItemAdapter +from itemadapter import ItemAdapter class ${ProjectName}SpiderMiddleware: diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index 308e351c6..bcfae0c00 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -28,12 +28,12 @@ class ScrapyJSONEncoder(json.JSONEncoder): return str(o) if isinstance(o, defer.Deferred): return str(o) - if is_item(o): - return ItemAdapter(o).asdict() if isinstance(o, Request): return f"<{type(o).__name__} {o.method} {o.url}>" if isinstance(o, Response): return f"<{type(o).__name__} {o.status} {o.url}>" + if is_item(o): + return ItemAdapter(o).asdict() return super().default(o) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 5766f9313..6f4045fc8 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,6 +1,5 @@ import json import logging -import re import unittest from ipaddress import IPv4Address from socket import gethostbyname @@ -195,23 +194,16 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_requests_unsupported_output(self): + """Anything that is not a request is assumed to be an item, avoiding a + potentially expensive call to itemadapter.is_item, and letting instead + things fail when ItemAdapter is actually used on the corresponding + non-item object.""" + with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(StartRequestsGoodAndBadOutput) yield crawler.crawl(mockserver=self.mockserver) - assert len(log.records) == 2 - assert log.records[0].msg == ( - "Got 'data:,b' among start requests. Only requests and items " - "are supported. It will be ignored." - ) - assert re.match( - ( - r"^Got among start " - r"requests\. Only requests and items are supported\. It " - r"will be ignored\.$" - ), - log.records[1].msg, - ) + assert len(log.records) == 0 @defer.inlineCallbacks def test_start_requests_laziness(self): From ba28d96d3ef2488e9c5c86d4a5c28a1aac269a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 14:53:04 +0100 Subject: [PATCH 6/9] Centralize from_crawler docs (and somewhat related changes) (#6723) --- docs/topics/addons.rst | 17 +----- docs/topics/api.rst | 9 +-- docs/topics/components.rst | 80 +++++++++++++++++++++++++-- docs/topics/downloader-middleware.rst | 19 +------ docs/topics/email.rst | 14 +---- docs/topics/extensions.rst | 80 ++++++--------------------- docs/topics/item-pipeline.rst | 18 ++---- docs/topics/request-response.rst | 24 +------- docs/topics/settings.rst | 39 ++++--------- docs/topics/spider-middleware.rst | 69 +++++++++-------------- 10 files changed, 150 insertions(+), 219 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 8ec7b0295..17e3c177a 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -32,7 +32,8 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are Python classes that include one or both of the following methods: +Add-ons are :ref:`components ` that include one or both of +the following methods: .. method:: update_settings(settings) @@ -54,20 +55,6 @@ Add-ons are Python classes that include one or both of the following methods: :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.BaseSettings` -They can also have the following method: - -.. classmethod:: from_crawler(cls, crawler) - :noindex: - - If present, this class method is called to create an add-on instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the add-on. The crawler object provides access to all Scrapy core - components like settings and signals; it is a way for the add-on to access - them and hook its functionality into Scrapy. - - :param crawler: The crawler that uses this add-on - :type crawler: :class:`~scrapy.crawler.Crawler` - The settings set by the add-on should use the ``addon`` priority (see :ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: diff --git a/docs/topics/api.rst b/docs/topics/api.rst index edc625be8..5a00fd570 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -12,10 +12,11 @@ extensions and middlewares. Crawler API =========== -The main entry point to Scrapy API is the :class:`~scrapy.crawler.Crawler` -object, passed to extensions through the ``from_crawler`` class method. This -object provides access to all Scrapy core components, and it's the only way for -extensions to access them and hook their functionality into Scrapy. +The main entry point to the Scrapy API is the :class:`~scrapy.crawler.Crawler` +object, which :ref:`components ` can :ref:`get for +initialization `. It provides access to all Scrapy core +components, and it is the only way for components to access them and hook their +functionality into Scrapy. .. module:: scrapy.crawler :synopsis: The Scrapy crawler diff --git a/docs/topics/components.rst b/docs/topics/components.rst index d34b3884b..3a7644379 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -9,6 +9,8 @@ A Scrapy component is any class whose objects are built using That includes the classes that you may assign to the following settings: +- :setting:`ADDONS` + - :setting:`DNS_RESOLVER` - :setting:`DOWNLOAD_HANDLERS` @@ -41,10 +43,80 @@ Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to modify their behavior. +.. _from-crawler: + +Initializing from the crawler +============================= + +Any Scrapy component may optionally define the following class method: + +.. classmethod:: from_crawler(cls, crawler: scrapy.crawler.Crawler, *args, **kwargs) + + Return an instance of the component based on *crawler*. + + *args* and *kwargs* are component-specific arguments that some components + receive. However, most components do not get any arguments, and instead + :ref:`use settings `. + + If a component class defines this method, this class method is called to + create any instance of the component. + + The *crawler* object provides access to all Scrapy core components like + :ref:`settings ` and :ref:`signals `, + allowing the component to access them and hook its functionality into + Scrapy. + +.. _component-settings: + +Settings +======== + +Components can be configured through :ref:`settings `. + +Components can read any setting from the +:attr:`~scrapy.crawler.Crawler.settings` attribute of the +:class:`~scrapy.crawler.Crawler` object they can :ref:`get for initialization +`. That includes both built-in and custom settings. + +For example: + +.. code-block:: python + + class MyExtension: + @classmethod + def from_crawler(cls, crawler): + settings = crawler.settings + return cls(settings.getbool("LOG_ENABLED")) + + def __init__(self, log_is_enabled=False): + if log_is_enabled: + print("log is enabled!") + +Components do not need to declare their custom settings programmatically. +However, they should document them, so that users know they exist and how to +use them. + +It is a good practice to prefix custom settings with the name of the component, +to avoid collisions with custom settings of other existing (or future) +components. For example, an extension called ``WarcCaching`` could prefix its +custom settings with ``WARC_CACHING_``. + +Another good practice, mainly for components meant for :ref:`component priority +dictionaries `, is to provide a boolean setting +called ``_ENABLED`` (e.g. ``WARC_CACHING_ENABLED``) to allow toggling +that component on and off without changing the component priority dictionary +setting. You can usually check the value of such a setting during +initialization, and if ``False``, raise +:exc:`~scrapy.exceptions.NotConfigured`. + +When choosing a name for a custom setting, it is also a good idea to have a +look at the names of :ref:`built-in settings `, to try to +maintain consistency with them. + .. _enforce-component-requirements: -Enforcing component requirements -================================ +Enforcing requirements +====================== Sometimes, your components may only be intended to work under certain conditions. For example, they may require a minimum version of Scrapy to work as @@ -58,8 +130,8 @@ In the case of :ref:`downloader middlewares `, :ref:`extensions `, :ref:`item pipelines `, and :ref:`spider middlewares `, you should raise -:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a -parameter to the exception so that it is printed in the logs, for the user to +:exc:`~scrapy.exceptions.NotConfigured`, passing a description of the issue as +a parameter to the exception so that it is printed in the logs, for the user to see. For other components, feel free to raise whatever other exception feels right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy version mismatch, while :exc:`ValueError` may be better if the issue is the diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index ab7e6a0ec..60b6aab78 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -61,12 +61,8 @@ particular setting. See each middleware documentation for more info. Writing your own downloader middleware ====================================== -Each downloader middleware is a Python class that defines one or more of the -methods defined below. - -The main entry point is the ``from_crawler`` class method, which receives a -:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` -object gives you access, for example, to the :ref:`settings `. +Each downloader middleware is a :ref:`component ` that +defines one or more of these methods: .. module:: scrapy.downloadermiddlewares @@ -167,17 +163,6 @@ object gives you access, for example, to the :ref:`settings `. :param spider: the spider for which this request is intended :type spider: :class:`~scrapy.Spider` object - .. method:: from_crawler(cls, crawler) - - If present, this classmethod is called to create a middleware instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the middleware. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for middleware to - access them and hook its functionality into Scrapy. - - :param crawler: crawler that uses this middleware - :type crawler: :class:`~scrapy.crawler.Crawler` object - .. _topics-downloader-middleware-ref: Built-in downloader middleware reference diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 8f7a2357a..1d7bad787 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -50,9 +50,9 @@ And here is how to use it to send an e-mail (without attachments): MailSender class reference ========================== -MailSender is the preferred class to use for sending emails from Scrapy, as it -uses :doc:`Twisted non-blocking IO `, like the -rest of the framework. +The MailSender :ref:`components ` is the preferred class to +use for sending emails from Scrapy, as it uses :doc:`Twisted non-blocking IO +`, like the rest of the framework. .. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) @@ -81,14 +81,6 @@ rest of the framework. :param smtpssl: enforce using a secure SSL connection :type smtpssl: bool - .. classmethod:: from_crawler(crawler) - - Instantiate using a :class:`scrapy.Crawler` instance, which will - respect :ref:`these Scrapy settings `. - - :param crawler: the crawler - :type settings: :class:`scrapy.Crawler` object - .. method:: send(to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None) Send email to the given recipients. diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 23bbcfcb5..e1e3dd6b4 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -4,34 +4,21 @@ Extensions ========== -The extensions framework provides a mechanism for inserting your own -custom functionality into Scrapy. +Extensions are :ref:`components ` that allow inserting your +own custom functionality into Scrapy. -Extensions are just regular classes. +Unlike other components, extensions do not have a specific role in Scrapy. They +are “wildcard” components that can be used for anything that does not fit the +role of any other type of component. -Extension settings -================== +Loading and activating extensions +================================= -Extensions use the :ref:`Scrapy settings ` to manage their -settings, just like any other Scrapy code. +Extensions are loaded at startup by creating a single instance of the extension +class per spider being run. -It is customary for extensions to prefix their settings with their own name, to -avoid collision with existing (and future) extensions. For example, a -hypothetical extension to handle `Google Sitemaps`_ would use settings like -``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on. - -.. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps - -Loading & activating extensions -=============================== - -Extensions are loaded and activated at startup by instantiating a single -instance of the extension class per spider being run. All the extension -initialization code must be performed in the class ``__init__`` method. - -To make an extension available, add it to the :setting:`EXTENSIONS` setting in -your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented -by a string: the full Python path to the extension's class name. For example: +To enable an extension, add it to the :setting:`EXTENSIONS` setting. For +example: .. code-block:: python @@ -40,55 +27,24 @@ by a string: the full Python path to the extension's class name. For example: "scrapy.extensions.telnet.TelnetConsole": 500, } - -As you can see, the :setting:`EXTENSIONS` setting is a dict where the keys are -the extension paths, and their values are the orders, which define the -extension *loading* order. The :setting:`EXTENSIONS` setting is merged with the -:setting:`EXTENSIONS_BASE` setting defined in Scrapy (and not meant to be -overridden) and then sorted by order to get the final sorted list of enabled -extensions. +:setting:`EXTENSIONS` is merged with :setting:`EXTENSIONS_BASE` (not meant to +be overridden), and the priorities in the resulting value determine the +*loading* order. As extensions typically do not depend on each other, their loading order is irrelevant in most cases. This is why the :setting:`EXTENSIONS_BASE` setting -defines all extensions with the same order (``0``). However, this feature can -be exploited if you need to add an extension which depends on other extensions -already loaded. - -Available, enabled and disabled extensions -========================================== - -Not all available extensions will be enabled. Some of them usually depend on a -particular setting. For example, the HTTP Cache extension is available by default -but disabled unless the :setting:`HTTPCACHE_ENABLED` setting is set. - -Disabling an extension -====================== - -In order to disable an extension that comes enabled by default (i.e. those -included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to -``None``. For example: - -.. code-block:: python - - EXTENSIONS = { - "scrapy.extensions.corestats.CoreStats": None, - } +defines all extensions with the same order (``0``). However, you may need to +carefully use priorities if you add an extension that depends on other +extensions being already loaded. Writing your own extension ========================== -Each extension is a Python class. The main entry point for a Scrapy extension -(this also includes middlewares and pipelines) is the ``from_crawler`` -class method which receives a ``Crawler`` instance. Through the Crawler object -you can access settings, signals, stats, and also control the crawling behaviour. +Each extension is a :ref:`component `. Typically, extensions connect to :ref:`signals ` and perform tasks triggered by them. -Finally, if the ``from_crawler`` method raises the -:exc:`~scrapy.exceptions.NotConfigured` exception, the extension will be -disabled. Otherwise, the extension will be enabled. - Sample extension ---------------- diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 310f153e8..dc27ce6ca 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -23,7 +23,8 @@ Typical uses of item pipelines are: Writing your own item pipeline ============================== -Each item pipeline component is a Python class that must implement the following method: +Each item pipeline is a :ref:`component ` that must +implement the following method: .. method:: process_item(self, item, spider) @@ -60,17 +61,6 @@ Additionally, they may also implement the following methods: :param spider: the spider which was closed :type spider: :class:`~scrapy.Spider` object -.. classmethod:: from_crawler(cls, crawler) - - If present, this class method is called to create a pipeline instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the pipeline. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for pipeline to - access them and hook its functionality into Scrapy. - - :param crawler: crawler that uses this pipeline - :type crawler: :class:`~scrapy.crawler.Crawler` object - Item pipeline example ===================== @@ -139,8 +129,8 @@ In this example we'll write items to MongoDB_ using pymongo_. MongoDB address and database name are specified in Scrapy settings; MongoDB collection is named after item class. -The main point of this example is to show how to use :meth:`from_crawler` -method and how to clean up the resources properly. +The main point of this example is to show how to :ref:`get the crawler +` and how to clean up the resources properly. .. skip: next .. code-block:: python diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 0375e0ff1..77837378e 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -463,35 +463,17 @@ import path. Writing your own request fingerprinter ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A request fingerprinter is a class that must implement the following method: +A request fingerprinter is a :ref:`component ` that must +implement the following method: .. currentmodule:: None -.. method:: fingerprint(self, request) +.. method:: fingerprint(self, request: scrapy.Request) Return a :class:`bytes` object that uniquely identifies *request*. See also :ref:`request-fingerprint-restrictions`. - :param request: request to fingerprint - :type request: scrapy.Request - -Additionally, it may also implement the following method: - -.. classmethod:: from_crawler(cls, crawler) - :noindex: - - If present, this class method is called to create a request fingerprinter - instance from a :class:`~scrapy.crawler.Crawler` object. It must return a - new instance of the request fingerprinter. - - *crawler* provides access to all Scrapy core components like settings and - signals; it is a way for the request fingerprinter to access them and hook - its functionality into Scrapy. - - :param crawler: crawler that uses this request fingerprinter - :type crawler: :class:`~scrapy.crawler.Crawler` object - .. currentmodule:: scrapy.http The :meth:`fingerprint` method of the default request fingerprinter, diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 7646aca4f..b2bb7148f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -204,7 +204,7 @@ How to access settings .. highlight:: python -In a spider, the settings are available through ``self.settings``: +In a spider, settings are available through ``self.settings``: .. code-block:: python @@ -217,37 +217,17 @@ In a spider, the settings are available through ``self.settings``: .. note:: The ``settings`` attribute is set in the base Spider class after the spider - is initialized. If you want to use the settings before the initialization + is initialized. If you want to use settings before the initialization (e.g., in your spider's ``__init__()`` method), you'll need to override the :meth:`~scrapy.Spider.from_crawler` method. -Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings` -attribute of the Crawler that is passed to ``from_crawler`` method in -extensions, middlewares and item pipelines: +:ref:`Components ` can also :ref:`access settings +`. -.. code-block:: python - - class MyExtension: - def __init__(self, log_is_enabled=False): - if log_is_enabled: - print("log is enabled!") - - @classmethod - def from_crawler(cls, crawler): - settings = crawler.settings - return cls(settings.getbool("LOG_ENABLED")) - -The settings object can be used like a dict (e.g., -``settings['LOG_ENABLED']``), but it's usually preferred to extract the setting -in the format you need it to avoid type errors, using one of the methods -provided by the :class:`~scrapy.settings.Settings` API. - -Rationale for setting names -=========================== - -Setting names are usually prefixed with the component that they configure. For -example, proper setting names for a fictional robots.txt extension would be -``ROBOTSTXT_ENABLED``, ``ROBOTSTXT_OBEY``, ``ROBOTSTXT_CACHEDIR``, etc. +The ``settings`` object can be used like a :class:`dict` (e.g. +``settings["LOG_ENABLED"]``). However, to support non-string setting values, +which may be passed from the command line as strings, it is recommended to use +one of the methods provided by the :class:`~scrapy.settings.Settings` API. .. _component-priority-dictionaries: @@ -1211,7 +1191,8 @@ EXTENSIONS Default:: ``{}`` -A dict containing the extensions enabled in your project, and their orders. +:ref:`Component priority dictionary ` of +enabled extensions. See :ref:`topics-extensions`. .. setting:: EXTENSIONS_BASE diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2b59cabe1..567a875b6 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -63,17 +63,38 @@ particular setting. See each middleware documentation for more info. Writing your own spider middleware ================================== -Each spider middleware is a Python class that defines one or more of the -methods defined below. - -The main entry point is the ``from_crawler`` class method, which receives a -:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` -object gives you access, for example, to the :ref:`settings `. +Each spider middleware is a :ref:`component ` that defines +one or more of these methods: .. module:: scrapy.spidermiddlewares .. class:: SpiderMiddleware + .. method:: process_start_requests(start_requests, spider) + + This method is called with the start requests of the spider, and works + similarly to the :meth:`process_spider_output` method, except that it + doesn't have a response associated and must return only requests (not + items). + + It receives an iterable (in the ``start_requests`` parameter) and must + return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects `. + + .. note:: When implementing this method in your spider middleware, you + should always return an iterable (that follows the input one) and + not consume all ``start_requests`` iterator because it can be very + large (or even unbounded) and cause a memory overflow. The Scrapy + engine is designed to pull start requests while it has capacity to + process them, so the start requests iterator can be effectively + endless where there is some other condition for stopping the spider + (like a time limit or item/page count). + + :param start_requests: the start requests + :type start_requests: an iterable of :class:`~scrapy.Request` + + :param spider: the spider to whom the start requests belong + :type spider: :class:`~scrapy.Spider` object + .. method:: process_spider_input(response, spider) This method is called for each response that goes through the spider @@ -168,42 +189,6 @@ object gives you access, for example, to the :ref:`settings `. :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object - .. method:: process_start_requests(start_requests, spider) - - This method is called with the start requests of the spider, and works - similarly to the :meth:`process_spider_output` method, except that it - doesn't have a response associated and must return only requests (not - items). - - It receives an iterable (in the ``start_requests`` parameter) and must - return another iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects `. - - .. note:: When implementing this method in your spider middleware, you - should always return an iterable (that follows the input one) and - not consume all ``start_requests`` iterator because it can be very - large (or even unbounded) and cause a memory overflow. The Scrapy - engine is designed to pull start requests while it has capacity to - process them, so the start requests iterator can be effectively - endless where there is some other condition for stopping the spider - (like a time limit or item/page count). - - :param start_requests: the start requests - :type start_requests: an iterable of :class:`~scrapy.Request` - - :param spider: the spider to whom the start requests belong - :type spider: :class:`~scrapy.Spider` object - - .. method:: from_crawler(cls, crawler) - - If present, this classmethod is called to create a middleware instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the middleware. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for middleware to - access them and hook its functionality into Scrapy. - - :param crawler: crawler that uses this middleware - :type crawler: :class:`~scrapy.crawler.Crawler` object - .. _topics-spider-middleware-ref: Built-in spider middleware reference From 803b4f258d85ab4f85c7c230e6c025f54e836269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Mar 2025 14:53:42 +0100 Subject: [PATCH 7/9] tox: move to posargs pytest parameters that can be too noisy when running specific tests (#6724) --- tox.ini | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 041fcffca..70c841603 100644 --- a/tox.ini +++ b/tox.ini @@ -39,7 +39,7 @@ passenv = #allow tox virtualenv to upgrade pip/wheel/setuptools download = true commands = - pytest --cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml {posargs:--durations=10 docs scrapy tests} --doctest-modules + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --durations=10 docs scrapy tests --doctest-modules} install_command = python -I -m pip install -ctests/upper-constraints.txt {opts} {packages} @@ -58,7 +58,7 @@ deps = pytest >= 8.2.0 w3lib >= 2.2.0 commands = - mypy {posargs: scrapy tests} + mypy {posargs:scrapy tests} [testenv:typing-tests] basepython = python3.9 @@ -67,7 +67,7 @@ deps = {[testenv:typing]deps} pytest-mypy-testing==0.1.3 commands = - pytest {posargs: tests_typing} + pytest {posargs:tests_typing} [testenv:pre-commit] basepython = python3 @@ -119,7 +119,7 @@ install_command = python -I -m pip install {opts} {packages} commands = ; tests for docs fail with parsel < 1.8.0 - pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:--durations=10 scrapy tests} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --durations=10 scrapy tests} [testenv:pinned] basepython = {[pinned]basepython} @@ -266,7 +266,7 @@ deps = {[testenv]deps} botocore>=1.4.87 commands = - pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests -m requires_botocore} [testenv:botocore-pinned] basepython = {[pinned]basepython} @@ -277,4 +277,4 @@ install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = - pytest --cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -m requires_botocore} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests -m requires_botocore} From eb654aa1a8d2ef6433957fcc1361420b6141094e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 11 Mar 2025 21:00:36 +0400 Subject: [PATCH 8/9] Convert remaining unittest assert* calls, use the tmp_path fixture. (#6725) --- pyproject.toml | 4 -- scrapy/utils/test.py | 2 +- tests/test_downloadermiddleware_robotstxt.py | 13 ++++- tests/test_engine.py | 8 ++- tests/test_pipeline_files.py | 58 ++++++++------------ tests/test_pipeline_images.py | 44 +++++++-------- tests/test_utils_template.py | 16 +----- 7 files changed, 62 insertions(+), 83 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 82d8056f6..84bf41a94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -374,10 +374,6 @@ ignore = [ "B904", # Use capitalized environment variable "SIM112", - - # Temporarily silenced PT rules - # Use a regular `assert` instead of unittest-style `assertEqual` - "PT009", ] [tool.ruff.lint.per-file-ignores] diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index db1f5c419..b69f43438 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -156,7 +156,7 @@ def assert_samelines( category=ScrapyDeprecationWarning, stacklevel=2, ) - testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) + testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) # noqa: PT009 def get_from_asyncio_queue(value: _T) -> Awaitable[_T]: diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 38f0333bb..ad335f852 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,3 +1,4 @@ +from typing import Any from unittest import mock import pytest @@ -171,7 +172,11 @@ Disallow: /some/randome/page.html middleware = RobotsTxtMiddleware(self.crawler) middleware._logerror = mock.MagicMock(side_effect=middleware._logerror) deferred = middleware.process_request(Request("http://site.local"), None) - deferred.addCallback(lambda _: self.assertTrue(middleware._logerror.called)) + + def check_called(_: Any) -> None: + assert middleware._logerror.called + + deferred.addCallback(check_called) return deferred def test_robotstxt_immediate_error(self): @@ -202,7 +207,11 @@ Disallow: /some/randome/page.html mw_module_logger.error = mock.MagicMock() d = self.assertNotIgnored(Request("http://site.local/allowed"), middleware) - d.addCallback(lambda _: self.assertFalse(mw_module_logger.error.called)) + + def check_not_called(_: Any) -> None: + assert not mw_module_logger.error.called # type: ignore[attr-defined] + + d.addCallback(check_not_called) return d def test_robotstxt_user_agent_setting(self): diff --git a/tests/test_engine.py b/tests/test_engine.py index 4bac8d273..ba4c6dc40 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -433,10 +433,12 @@ class TestEngine(TestEngineBase): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.open_spider(MySpider(), []) e.start() + + def cb(exc: BaseException) -> None: + assert str(exc), "Engine already running" + try: - yield self.assertFailure(e.start(), RuntimeError).addBoth( - lambda exc: self.assertEqual(str(exc), "Engine already running") - ) + yield self.assertFailure(e.start(), RuntimeError).addBoth(cb) finally: yield e.stop() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index e515c16a0..9a582e4b7 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -266,17 +266,11 @@ class TestFilesPipeline(unittest.TestCase): class FilesPipelineTestCaseFieldsMixin: - def setup_method(self): - self.tempdir = mkdtemp() - - def teardown_method(self): - rmtree(self.tempdir) - - def test_item_fields_default(self): + def test_item_fields_default(self, tmp_path): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", file_urls=[url]) pipeline = FilesPipeline.from_crawler( - get_crawler(None, {"FILES_STORE": self.tempdir}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) requests = list(pipeline.get_media_requests(item, None)) assert requests[0].url == url @@ -286,14 +280,14 @@ class FilesPipelineTestCaseFieldsMixin: assert files == [results[0][1]] assert isinstance(item, self.item_class) - def test_item_fields_override_settings(self): + def test_item_fields_override_settings(self, tmp_path): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", custom_file_urls=[url]) pipeline = FilesPipeline.from_crawler( get_crawler( None, { - "FILES_STORE": self.tempdir, + "FILES_STORE": tmp_path, "FILES_URLS_FIELD": "custom_file_urls", "FILES_RESULT_FIELD": "custom_files", }, @@ -368,13 +362,7 @@ class TestFilesPipelineCustomSettings: ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"), } - def setup_method(self): - self.tempdir = mkdtemp() - - def teardown_method(self): - rmtree(self.tempdir) - - def _generate_fake_settings(self, prefix=None): + def _generate_fake_settings(self, tmp_path, prefix=None): def random_string(): return "".join([chr(random.randint(97, 123)) for _ in range(10)]) @@ -382,7 +370,7 @@ class TestFilesPipelineCustomSettings: "FILES_EXPIRES": random.randint(100, 1000), "FILES_URLS_FIELD": random_string(), "FILES_RESULT_FIELD": random_string(), - "FILES_STORE": self.tempdir, + "FILES_STORE": tmp_path, } if not prefix: return settings @@ -400,16 +388,16 @@ class TestFilesPipelineCustomSettings: return UserDefinedFilePipeline - def test_different_settings_for_different_instances(self): + def test_different_settings_for_different_instances(self, tmp_path): """ If there are different instances with different settings they should keep different settings. """ - custom_settings = self._generate_fake_settings() + custom_settings = self._generate_fake_settings(tmp_path) another_pipeline = FilesPipeline.from_crawler( get_crawler(None, custom_settings) ) - one_pipeline = FilesPipeline(self.tempdir, crawler=get_crawler(None)) + one_pipeline = FilesPipeline(tmp_path, crawler=get_crawler(None)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: default_value = self.default_cls_settings[pipe_attr] assert getattr(one_pipeline, pipe_attr) == default_value @@ -417,24 +405,24 @@ class TestFilesPipelineCustomSettings: assert default_value != custom_value assert getattr(another_pipeline, pipe_ins_attr) == custom_value - def test_subclass_attributes_preserved_if_no_settings(self): + def test_subclass_attributes_preserved_if_no_settings(self, tmp_path): """ If subclasses override class attributes and there are no special settings those values should be kept. """ pipe_cls = self._generate_fake_pipeline() - pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": self.tempdir})) + pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": tmp_path})) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: custom_value = getattr(pipe, pipe_ins_attr) assert custom_value != self.default_cls_settings[pipe_attr] assert getattr(pipe, pipe_ins_attr) == getattr(pipe, pipe_attr) - def test_subclass_attrs_preserved_custom_settings(self): + def test_subclass_attrs_preserved_custom_settings(self, tmp_path): """ If file settings are defined but they are not defined for subclass settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline() - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: value = getattr(pipeline, pipe_ins_attr) @@ -442,7 +430,7 @@ class TestFilesPipelineCustomSettings: assert value != self.default_cls_settings[pipe_attr] assert value == setting_value - def test_no_custom_settings_for_subclasses(self): + def test_no_custom_settings_for_subclasses(self, tmp_path): """ If there are no settings for subclass and no subclass attributes, pipeline should use attributes of base class. @@ -452,14 +440,14 @@ class TestFilesPipelineCustomSettings: pass user_pipeline = UserDefinedFilesPipeline.from_crawler( - get_crawler(None, {"FILES_STORE": self.tempdir}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_cls_settings.get(pipe_attr.upper()) assert getattr(user_pipeline, pipe_ins_attr) == custom_value - def test_custom_settings_for_subclasses(self): + def test_custom_settings_for_subclasses(self, tmp_path): """ If there are custom settings for subclass and NO class attributes, pipeline should use custom settings. @@ -469,7 +457,7 @@ class TestFilesPipelineCustomSettings: pass prefix = UserDefinedFilesPipeline.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = UserDefinedFilesPipeline.from_crawler( get_crawler(None, settings) ) @@ -479,14 +467,14 @@ class TestFilesPipelineCustomSettings: assert custom_value != self.default_cls_settings[pipe_attr] assert getattr(user_pipeline, pipe_inst_attr) == custom_value - def test_custom_settings_and_class_attrs_for_subclasses(self): + def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path): """ If there are custom settings for subclass AND class attributes setting keys are preferred and override attributes. """ pipeline_cls = self._generate_fake_pipeline() prefix = pipeline_cls.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for ( pipe_cls_attr, @@ -497,13 +485,13 @@ class TestFilesPipelineCustomSettings: assert custom_value != self.default_cls_settings[pipe_cls_attr] assert getattr(user_pipeline, pipe_inst_attr) == custom_value - def test_cls_attrs_with_DEFAULT_prefix(self): + def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path): class UserDefinedFilesPipeline(FilesPipeline): DEFAULT_FILES_RESULT_FIELD = "this" DEFAULT_FILES_URLS_FIELD = "that" pipeline = UserDefinedFilesPipeline.from_crawler( - get_crawler(None, {"FILES_STORE": self.tempdir}) + get_crawler(None, {"FILES_STORE": tmp_path}) ) assert ( pipeline.files_result_field @@ -514,12 +502,12 @@ class TestFilesPipelineCustomSettings: == UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD ) - def test_user_defined_subclass_default_key_names(self): + def test_user_defined_subclass_default_key_names(self, tmp_path): """Test situation when user defines subclass of FilesPipeline, but uses attribute names for default pipeline (without prefixing them with pipeline class name). """ - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) class UserPipe(FilesPipeline): pass diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index fef6bbbe9..f2ee18bd9 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -314,13 +314,7 @@ class TestImagesPipelineCustomSettings: "IMAGES_RESULT_FIELD": "images", } - def setup_method(self): - self.tempdir = mkdtemp() - - def teardown_method(self): - rmtree(self.tempdir) - - def _generate_fake_settings(self, prefix=None): + def _generate_fake_settings(self, tmp_path, prefix=None): """ :param prefix: string for setting keys :return: dictionary of image pipeline settings @@ -331,7 +325,7 @@ class TestImagesPipelineCustomSettings: settings = { "IMAGES_EXPIRES": random.randint(100, 1000), - "IMAGES_STORE": self.tempdir, + "IMAGES_STORE": tmp_path, "IMAGES_RESULT_FIELD": random_string(), "IMAGES_URLS_FIELD": random_string(), "IMAGES_MIN_WIDTH": random.randint(1, 1000), @@ -368,13 +362,13 @@ class TestImagesPipelineCustomSettings: return UserDefinedImagePipeline - def test_different_settings_for_different_instances(self): + def test_different_settings_for_different_instances(self, tmp_path): """ If there are two instances of ImagesPipeline class with different settings, they should have different settings. """ - custom_settings = self._generate_fake_settings() - default_sts_pipe = ImagesPipeline(self.tempdir, crawler=get_crawler(None)) + custom_settings = self._generate_fake_settings(tmp_path) + default_sts_pipe = ImagesPipeline(tmp_path, crawler=get_crawler(None)) user_sts_pipe = ImagesPipeline.from_crawler(get_crawler(None, custom_settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: expected_default_value = self.default_pipeline_settings.get(pipe_attr) @@ -385,14 +379,14 @@ class TestImagesPipelineCustomSettings: ) assert getattr(user_sts_pipe, pipe_attr.lower()) == custom_value - def test_subclass_attrs_preserved_default_settings(self): + def test_subclass_attrs_preserved_default_settings(self, tmp_path): """ If image settings are not defined at all subclass of ImagePipeline takes values from class attributes. """ pipeline_cls = self._generate_fake_pipeline_subclass() pipeline = pipeline_cls.from_crawler( - get_crawler(None, {"IMAGES_STORE": self.tempdir}) + get_crawler(None, {"IMAGES_STORE": tmp_path}) ) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Instance attribute (lowercase) must be equal to class attribute (uppercase). @@ -400,13 +394,13 @@ class TestImagesPipelineCustomSettings: assert attr_value != self.default_pipeline_settings[pipe_attr] assert attr_value == getattr(pipeline, pipe_attr) - def test_subclass_attrs_preserved_custom_settings(self): + def test_subclass_attrs_preserved_custom_settings(self, tmp_path): """ If image settings are defined but they are not defined for subclass default values taken from settings should be preserved. """ pipeline_cls = self._generate_fake_pipeline_subclass() - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Instance attribute (lowercase) must be equal to @@ -416,7 +410,7 @@ class TestImagesPipelineCustomSettings: setings_value = settings.get(settings_attr) assert value == setings_value - def test_no_custom_settings_for_subclasses(self): + def test_no_custom_settings_for_subclasses(self, tmp_path): """ If there are no settings for subclass and no subclass attributes, pipeline should use attributes of base class. @@ -426,14 +420,14 @@ class TestImagesPipelineCustomSettings: pass user_pipeline = UserDefinedImagePipeline.from_crawler( - get_crawler(None, {"IMAGES_STORE": self.tempdir}) + get_crawler(None, {"IMAGES_STORE": tmp_path}) ) for pipe_attr, settings_attr in self.img_cls_attribute_names: # Values from settings for custom pipeline should be set on pipeline instance. custom_value = self.default_pipeline_settings.get(pipe_attr.upper()) assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_custom_settings_for_subclasses(self): + def test_custom_settings_for_subclasses(self, tmp_path): """ If there are custom settings for subclass and NO class attributes, pipeline should use custom settings. @@ -443,7 +437,7 @@ class TestImagesPipelineCustomSettings: pass prefix = UserDefinedImagePipeline.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = UserDefinedImagePipeline.from_crawler( get_crawler(None, settings) ) @@ -453,27 +447,27 @@ class TestImagesPipelineCustomSettings: assert custom_value != self.default_pipeline_settings[pipe_attr] assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_custom_settings_and_class_attrs_for_subclasses(self): + def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path): """ If there are custom settings for subclass AND class attributes setting keys are preferred and override attributes. """ pipeline_cls = self._generate_fake_pipeline_subclass() prefix = pipeline_cls.__name__.upper() - settings = self._generate_fake_settings(prefix=prefix) + settings = self._generate_fake_settings(tmp_path, prefix=prefix) user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings)) for pipe_attr, settings_attr in self.img_cls_attribute_names: custom_value = settings.get(prefix + "_" + settings_attr) assert custom_value != self.default_pipeline_settings[pipe_attr] assert getattr(user_pipeline, pipe_attr.lower()) == custom_value - def test_cls_attrs_with_DEFAULT_prefix(self): + def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path): class UserDefinedImagePipeline(ImagesPipeline): DEFAULT_IMAGES_URLS_FIELD = "something" DEFAULT_IMAGES_RESULT_FIELD = "something_else" pipeline = UserDefinedImagePipeline.from_crawler( - get_crawler(None, {"IMAGES_STORE": self.tempdir}) + get_crawler(None, {"IMAGES_STORE": tmp_path}) ) assert ( pipeline.images_result_field @@ -484,12 +478,12 @@ class TestImagesPipelineCustomSettings: == UserDefinedImagePipeline.DEFAULT_IMAGES_URLS_FIELD ) - def test_user_defined_subclass_default_key_names(self): + def test_user_defined_subclass_default_key_names(self, tmp_path): """Test situation when user defines subclass of ImagePipeline, but uses attribute names for default pipeline (without prefixing them with pipeline class name). """ - settings = self._generate_fake_settings() + settings = self._generate_fake_settings(tmp_path) class UserPipe(ImagesPipeline): pass diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py index 0b845fdb0..41d9b8933 100644 --- a/tests/test_utils_template.py +++ b/tests/test_utils_template.py @@ -1,24 +1,14 @@ -from pathlib import Path -from shutil import rmtree -from tempfile import mkdtemp - from scrapy.utils.template import render_templatefile class TestUtilsRenderTemplateFile: - def setup_method(self): - self.tmp_path = mkdtemp() - - def teardown_method(self): - rmtree(self.tmp_path) - - def test_simple_render(self): + def test_simple_render(self, tmp_path): context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"} template = "from ${project_name}.spiders.${name} import ${classname}" rendered = "from proj.spiders.spi import TheSpider" - template_path = Path(self.tmp_path, "templ.py.tmpl") - render_path = Path(self.tmp_path, "templ.py") + template_path = tmp_path / "templ.py.tmpl" + render_path = tmp_path / "templ.py" template_path.write_text(template, encoding="utf8") assert template_path.is_file() # Failure of test itself From d0dabbc09706b082e2250790cd7a00c033ad8021 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 12 Mar 2025 00:18:30 +0400 Subject: [PATCH 9/9] Enable AsyncioSelectorReactor by default. (#6713) * Enable AsyncioSelectorReactor by default. * Improve get_crawler(), switch more tests to it. * Fix the remaining default-reactor test failures. * Address documentation feedback. * Make pinned envs more consistent. --- .github/workflows/tests-ubuntu.yml | 14 ++-- .github/workflows/tests-windows.yml | 16 ++++- conftest.py | 12 ++-- docs/topics/asyncio.rst | 23 +++++-- docs/topics/media-pipeline.rst | 2 +- docs/topics/settings.rst | 14 ++-- scrapy/pipelines/images.py | 2 +- scrapy/settings/default_settings.py | 2 +- .../templates/project/module/settings.py.tmpl | 1 - scrapy/utils/log.py | 4 +- scrapy/utils/test.py | 23 ++++++- .../CrawlerProcess/asyncio_enabled_reactor.py | 14 ++-- tests/CrawlerProcess/reactor_default.py | 4 +- tests/CrawlerProcess/reactor_select.py | 4 +- tests/CrawlerRunner/ip_address.py | 6 ++ tests/test_addons.py | 5 +- tests/test_crawl.py | 4 +- tests/test_crawler.py | 46 +++++++------ tests/test_dependencies.py | 2 +- tests/test_downloader_handlers.py | 6 +- tests/test_downloaderslotssettings.py | 2 +- tests/test_extension_periodic_log.py | 19 +++--- tests/test_pipeline_crawl.py | 31 +++++---- tests/test_spider.py | 8 ++- tests/test_utils_asyncio.py | 5 +- tox.ini | 68 ++++++++----------- 26 files changed, 197 insertions(+), 140 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f74575ee1..444aa3557 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -34,25 +34,25 @@ jobs: TOXENV: py - python-version: "3.13" env: - TOXENV: asyncio + TOXENV: default-reactor - python-version: pypy3.10 env: TOXENV: pypy3 # pinned deps - - python-version: 3.9.19 + - python-version: "3.9.21" env: TOXENV: pinned - - python-version: 3.9.19 + - python-version: "3.9.21" env: - TOXENV: asyncio-pinned + TOXENV: default-reactor-pinned - python-version: pypy3.10 env: TOXENV: pypy3-pinned - - python-version: 3.9.19 + - python-version: "3.9.21" env: TOXENV: extra-deps-pinned - - python-version: 3.9.19 + - python-version: "3.9.21" env: TOXENV: botocore-pinned @@ -78,7 +78,7 @@ jobs: if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update - sudo apt-get install libxml2-dev libxslt-dev libjpeg-dev + sudo apt-get install libxml2-dev libxslt-dev - name: Run tests env: ${{ matrix.env }} diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 21d621240..537a01e29 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -19,7 +19,7 @@ jobs: include: - python-version: "3.9" env: - TOXENV: windows-pinned + TOXENV: py - python-version: "3.10" env: TOXENV: py @@ -34,7 +34,19 @@ jobs: TOXENV: py - python-version: "3.13" env: - TOXENV: asyncio + TOXENV: default-reactor + + # pinned deps + - python-version: "3.9.13" + env: + TOXENV: pinned + - python-version: "3.9.13" + env: + TOXENV: extra-deps-pinned + + - python-version: "3.13" + env: + TOXENV: extra-deps steps: - uses: actions/checkout@v4 diff --git a/conftest.py b/conftest.py index f33ffb1a4..9999e41d2 100644 --- a/conftest.py +++ b/conftest.py @@ -51,7 +51,7 @@ def chdir(tmpdir): def pytest_addoption(parser): parser.addoption( "--reactor", - default="default", + default="asyncio", choices=["default", "asyncio"], ) @@ -67,17 +67,17 @@ def reactor_pytest(request): @pytest.fixture(autouse=True) def only_asyncio(request, reactor_pytest): - if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio": - pytest.skip("This test is only run with --reactor=asyncio") + if request.node.get_closest_marker("only_asyncio") and reactor_pytest == "default": + pytest.skip("This test is only run without --reactor=default") @pytest.fixture(autouse=True) def only_not_asyncio(request, reactor_pytest): if ( request.node.get_closest_marker("only_not_asyncio") - and reactor_pytest == "asyncio" + and reactor_pytest != "default" ): - pytest.skip("This test is only run without --reactor=asyncio") + pytest.skip("This test is only run with --reactor=default") @pytest.fixture(autouse=True) @@ -117,7 +117,7 @@ def requires_boto3(request): def pytest_configure(config): - if config.getoption("--reactor") == "asyncio": + if config.getoption("--reactor") != "default": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 07baea071..35afdc11b 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -16,15 +16,19 @@ asyncio reactor `, you may use :mod:`asyncio` and Installing the asyncio reactor ============================== -To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to -``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``. +To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs +to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, +which is the default value. If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` reactor manually. You can do that using -:func:`~scrapy.utils.reactor.install_reactor`:: +:func:`~scrapy.utils.reactor.install_reactor`: - install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') +.. skip: next +.. code-block:: python + + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") .. _asyncio-preinstalled-reactor: @@ -144,3 +148,14 @@ Using custom asyncio loops You can also use custom asyncio event loops with the asyncio reactor. Set the :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to use it instead of the default asyncio event loop. + + +.. _disable-asyncio: + +Switching to a non-asyncio reactor +================================== + +If for some reason your code doesn't work with the asyncio reactor, you can use +a different reactor by setting the :setting:`TWISTED_REACTOR` setting to its +import path (e.g. ``'twisted.internet.epollreactor.EPollReactor'``) or to +``None``, which will use the default reactor for your platform. diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index cc1fe8703..01da53342 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you can configure some extra functions like generating thumbnails and filtering the images based on their size. -The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for +The Images Pipeline requires Pillow_ 8.0.0 or greater. It is used for thumbnailing and normalizing images to JPEG/RGB format. .. _Pillow: https://github.com/python-pillow/Pillow diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index b2bb7148f..ca0af569f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1911,7 +1911,7 @@ TWISTED_REACTOR .. versionadded:: 2.0 -Default: ``None`` +Default: ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"`` Import path of a given :mod:`~twisted.internet.reactor`. @@ -1996,17 +1996,19 @@ which raises :exc:`Exception`, becomes: self.crawler.engine.close_spider(self, "timeout") -The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which -means that Scrapy will use the existing reactor if one is already installed, or -install the default reactor defined by Twisted for the current platform. This -is to maintain backward compatibility and avoid possible problems caused by -using a non-default reactor. +If this setting is set ``None``, Scrapy will use the existing reactor if one is +already installed, or install the default reactor defined by Twisted for the +current platform. .. versionchanged:: 2.7 The :command:`startproject` command now sets this setting to ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated ``settings.py`` file. +.. versionchanged:: VERSION + The default value was changed from ``None`` to + ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``. + For additional information, see :doc:`core/howto/choosing-reactor`. diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 29dc13f0a..63c6908dc 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -68,7 +68,7 @@ class ImagesPipeline(FilesPipeline): self._Image = Image except ImportError: raise NotConfigured( - "ImagesPipeline requires installing Pillow 4.0.0 or later" + "ImagesPipeline requires installing Pillow 8.0.0 or later" ) super().__init__( diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c473b369c..645e50301 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -341,7 +341,7 @@ TELNETCONSOLE_HOST = "127.0.0.1" TELNETCONSOLE_USERNAME = "scrapy" TELNETCONSOLE_PASSWORD = None -TWISTED_REACTOR = None +TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 0bb31ffaa..db7400af8 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -90,5 +90,4 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" # Set settings whose default value is deprecated to a future-proof value -TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index b865cf48d..24e17ecb6 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -182,11 +182,9 @@ def log_scrapy_info(settings: Settings) -> None: def log_reactor_info() -> None: - from twisted.internet import reactor + from twisted.internet import asyncioreactor, reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) - from twisted.internet import asyncioreactor - if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor): logger.debug( "Using asyncio event loop: %s.%s", diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index b69f43438..2da526cd8 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -18,6 +18,7 @@ from twisted.trial.unittest import SkipTest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available from scrapy.utils.deprecate import create_deprecated_class +from scrapy.utils.reactor import is_asyncio_reactor_installed from scrapy.utils.spider import DefaultSpider if TYPE_CHECKING: @@ -109,6 +110,19 @@ def get_ftp_content_and_delete( TestSpider = create_deprecated_class("TestSpider", DefaultSpider) +def get_reactor_settings() -> dict[str, Any]: + """Return a settings dict that works with the installed reactor. + + ``Crawler._apply_settings()`` checks that the installed reactor matches the + settings, so tests that run the crawler in the current process may need to + pass a correct ``"TWISTED_REACTOR"`` setting value when creating it. + """ + settings: dict[str, Any] = {} + if not is_asyncio_reactor_installed(): + settings["TWISTED_REACTOR"] = None + return settings + + def get_crawler( spidercls: type[Spider] | None = None, settings_dict: dict[str, Any] | None = None, @@ -120,9 +134,12 @@ def get_crawler( """ from scrapy.crawler import CrawlerRunner - # Set by default settings that prevent deprecation warnings. - settings: dict[str, Any] = {} - settings.update(settings_dict or {}) + # When needed, useful settings can be added here, e.g. ones that prevent + # deprecation warnings. + settings: dict[str, Any] = { + **get_reactor_settings(), + **(settings_dict or {}), + } runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or DefaultSpider) crawler._apply_settings() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index f013eed27..0c380610d 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,14 +1,8 @@ -import asyncio -import sys +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy.utils.reactor import install_reactor -from twisted.internet import asyncioreactor - -if sys.platform == "win32": - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncioreactor.install(asyncio.get_event_loop()) - -import scrapy # noqa: E402 -from scrapy.crawler import CrawlerProcess # noqa: E402 +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 078cb72cb..e2933338b 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -1,4 +1,5 @@ from twisted.internet import reactor # noqa: F401 +from twisted.python import log import scrapy from scrapy.crawler import CrawlerProcess @@ -13,5 +14,6 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess(settings={}) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index 814a2a46d..b61e52625 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -1,4 +1,5 @@ from twisted.internet import selectreactor +from twisted.python import log import scrapy from scrapy.crawler import CrawlerProcess @@ -15,5 +16,6 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess(settings={}) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 2f1bb7713..b1b297777 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -1,3 +1,9 @@ +# ruff: noqa: E402 + +from scrapy.utils.reactor import install_reactor + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + from urllib.parse import urlparse from twisted.internet import reactor diff --git a/tests/test_addons.py b/tests/test_addons.py index 686bf9952..b4294c815 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -9,7 +9,7 @@ from scrapy import Spider from scrapy.crawler import Crawler, CrawlerRunner from scrapy.exceptions import NotConfigured from scrapy.settings import BaseSettings, Settings -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings class SimpleAddon: @@ -105,6 +105,7 @@ class TestAddonManager(unittest.TestCase): } settings_dict = { "ADDONS": {get_addon_cls(config): 1}, + **get_reactor_settings(), } crawler = get_crawler(settings_dict=settings_dict) assert crawler.settings.getint("KEY") == 15 @@ -119,6 +120,7 @@ class TestAddonManager(unittest.TestCase): settings_dict = { "KEY": 20, # priority=project "ADDONS": {get_addon_cls(config): 1}, + **get_reactor_settings(), } settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") @@ -196,6 +198,7 @@ class TestAddonManager(unittest.TestCase): return spider settings = Settings() + settings.setdict(get_reactor_settings()) settings.set("KEY", "default", priority="default") runner = CrawlerRunner(settings) crawler = runner.create_crawler(MySpider) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 6f4045fc8..f49deac1f 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -18,7 +18,7 @@ from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import to_unicode -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import MockServer from tests.spiders import ( @@ -412,7 +412,7 @@ with multiples lines @defer.inlineCallbacks def test_crawl_multiple(self): - runner = CrawlerRunner() + runner = CrawlerRunner(get_reactor_settings()) runner.crawl( SimpleSpider, self.mockserver.url("/status?n=200"), diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0bbcc0843..98352b66e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -25,7 +25,7 @@ from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings from tests.mockserver import MockServer, get_mockserver_env BASE_SETTINGS: dict[str, Any] = {} @@ -35,6 +35,7 @@ def get_raw_crawler(spidercls=None, settings_dict=None): """get_crawler alternative that only calls the __init__ method of the crawler.""" settings = Settings() + settings.setdict(get_reactor_settings()) settings.setdict(settings_dict or {}) return Crawler(spidercls or DefaultSpider, settings) @@ -48,7 +49,12 @@ class TestBaseCrawler(unittest.TestCase): class TestCrawler(TestBaseCrawler): def test_populate_spidercls_settings(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"} + project_settings = { + **BASE_SETTINGS, + "TEST1": "project", + "TEST3": "project", + **get_reactor_settings(), + } class CustomSettingsSpider(DefaultSpider): custom_settings = spider_settings @@ -581,7 +587,7 @@ class NoRequestsSpider(scrapy.Spider): @pytest.mark.usefixtures("reactor_pytest") class TestCrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): - return CrawlerRunner() + return CrawlerRunner(get_reactor_settings()) @inlineCallbacks def test_crawler_runner_bootstrap_successful(self): @@ -626,13 +632,7 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): @inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): - if self.reactor_pytest == "asyncio": - CrawlerRunner( - settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - } - ) - else: + if self.reactor_pytest == "default": runner = CrawlerRunner( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", @@ -643,6 +643,12 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", ): yield runner.crawl(NoRequestsSpider) + else: + CrawlerRunner( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + ) class ScriptRunnerMixin: @@ -672,7 +678,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): assert "Spider closed (finished)" in log assert ( "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log + in log ) def test_multi(self): @@ -680,18 +686,17 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): assert "Spider closed (finished)" in log assert ( "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log + in log ) assert "ReactorAlreadyInstalledError" not in log def test_reactor_default(self): log = self.run_script("reactor_default.py") - assert "Spider closed (finished)" in log + assert "Spider closed (finished)" not in log assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log - ) - assert "ReactorAlreadyInstalledError" not in log + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log def test_reactor_default_twisted_reactor_select(self): log = self.run_script("reactor_default_twisted_reactor_select.py") @@ -716,8 +721,11 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_reactor_select(self): log = self.run_script("reactor_select.py") - assert "Spider closed (finished)" in log - assert "ReactorAlreadyInstalledError" not in log + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log def test_reactor_select_twisted_reactor_select(self): log = self.run_script("reactor_select_twisted_reactor_select.py") diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 162747581..c2df67c66 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -33,7 +33,7 @@ class TestScrapyUtils: tox_config_file_path = Path(__file__).parent / ".." / "tox.ini" config_parser = ConfigParser() config_parser.read(tox_config_file_path) - pattern = r"Twisted\[http2\]==([\d.]+)" + pattern = r"Twisted==([\d.]+)" match = re.search(pattern, config_parser["pinned"]["deps"]) pinned_twisted_version_string = match[1] diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 19bd02498..bc18e76e1 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -307,7 +307,7 @@ class TestHttp(unittest.TestCase, ABC): @defer.inlineCallbacks def test_timeout_download_from_spider_nodata_rcvd(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": + if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 raise unittest.SkipTest( "This test produces DirtyReactorAggregateError on Windows with asyncio" @@ -322,7 +322,7 @@ class TestHttp(unittest.TestCase, ABC): @defer.inlineCallbacks def test_timeout_download_from_spider_server_hangs(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": + if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 raise unittest.SkipTest( "This test produces DirtyReactorAggregateError on Windows with asyncio" @@ -1136,7 +1136,7 @@ class TestFTPBase(unittest.TestCase): class TestFTP(TestFTPBase): def test_invalid_credentials(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": + if self.reactor_pytest != "default" and sys.platform == "win32": raise unittest.SkipTest( "This test produces DirtyReactorAggregateError on Windows with asyncio" ) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 15b3ad5af..4fca9eefb 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -64,7 +64,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_delay(self): - crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider) + crawler = get_crawler(DownloaderSlotsSettingsTestSpider) yield crawler.crawl(mockserver=self.mockserver) slots = crawler.engine.downloader.slots times = crawler.spider.times diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index ca5ffdc26..85bd42857 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,9 +1,11 @@ -import datetime -import typing -import unittest +from __future__ import annotations + +import datetime +import unittest +from typing import Any, Callable -from scrapy.crawler import Crawler from scrapy.extensions.periodic_log import PeriodicLog +from scrapy.utils.test import get_crawler from .spiders import MetaSpider @@ -59,9 +61,8 @@ class CustomPeriodicLog(PeriodicLog): self.stats._stats = stats_dump_2 -def extension(settings=None): - crawler = Crawler(MetaSpider, settings=settings) - crawler._apply_settings() +def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog: + crawler = get_crawler(MetaSpider, settings) return CustomPeriodicLog.from_crawler(crawler) @@ -94,7 +95,7 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b - def check(settings: dict, condition: typing.Callable): + def check(settings: dict[str, Any], condition: Callable) -> None: ext, a, b = emulate(settings) assert list(a["delta"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) @@ -151,7 +152,7 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b - def check(settings: dict, condition: typing.Callable): + def check(settings: dict[str, Any], condition: Callable) -> None: ext, a, b = emulate(settings) assert list(a["stats"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 162dfdaf4..c5f1b6321 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -3,18 +3,22 @@ from __future__ import annotations import shutil from pathlib import Path from tempfile import mkdtemp +from typing import TYPE_CHECKING, Any from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase from w3lib.url import add_or_replace_parameter -from scrapy import signals -from scrapy.crawler import CrawlerRunner +from scrapy import Spider, signals from scrapy.utils.misc import load_object +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from tests.spiders import SimpleSpider +if TYPE_CHECKING: + from scrapy.crawler import Crawler + class MediaDownloadSpider(SimpleSpider): name = "mediadownload" @@ -80,7 +84,6 @@ class TestFileDownloadCrawl(TestCase): "ITEM_PIPELINES": {self.pipeline_class: 1}, self.store_setting_key: str(self.tmpmediastore), } - self.runner = CrawlerRunner(self.settings) self.items = [] def tearDown(self): @@ -90,10 +93,12 @@ class TestFileDownloadCrawl(TestCase): def _on_item_scraped(self, item): self.items.append(item) - def _create_crawler(self, spider_class, runner=None, **kwargs): - if runner is None: - runner = self.runner - crawler = runner.create_crawler(spider_class, **kwargs) + def _create_crawler( + self, spider_class: type[Spider], settings: dict[str, Any] | None = None + ) -> Crawler: + if settings is None: + settings = self.settings + crawler = get_crawler(spider_class, settings) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) return crawler @@ -175,10 +180,11 @@ class TestFileDownloadCrawl(TestCase): @defer.inlineCallbacks def test_download_media_redirected_allowed(self): - settings = dict(self.settings) - settings.update({"MEDIA_ALLOW_REDIRECTS": True}) - runner = CrawlerRunner(settings) - crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner) + settings = { + **self.settings, + "MEDIA_ALLOW_REDIRECTS": True, + } + crawler = self._create_crawler(RedirectedMediaDownloadSpider, settings) with LogCapture() as log: yield crawler.crawl( self.mockserver.url("/files/images/"), @@ -201,8 +207,7 @@ class TestFileDownloadCrawl(TestCase): **self.settings, "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1}, } - runner = CrawlerRunner(settings) - crawler = self._create_crawler(MediaDownloadSpider, runner=runner) + crawler = self._create_crawler(MediaDownloadSpider, settings) with LogCapture() as log: yield crawler.crawl( self.mockserver.url("/files/images/"), diff --git a/tests/test_spider.py b/tests/test_spider.py index 4e8330c06..aaf72390d 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -27,7 +27,7 @@ from scrapy.spiders import ( XMLFeedSpider, ) from scrapy.spiders.init import InitSpider -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata, tests_datadir @@ -108,7 +108,11 @@ class TestSpider(unittest.TestCase): @inlineCallbacks def test_settings_in_from_crawler(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {"TEST1": "project", "TEST3": "project"} + project_settings = { + "TEST1": "project", + "TEST3": "project", + **get_reactor_settings(), + } class TestSpider(self.spider_class): name = "test" diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index a65a36219..901e03d59 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -2,6 +2,7 @@ import asyncio import warnings import pytest +from twisted.trial.unittest import TestCase from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( @@ -12,10 +13,10 @@ from scrapy.utils.reactor import ( @pytest.mark.usefixtures("reactor_pytest") -class TestAsyncio: +class TestAsyncio(TestCase): def test_is_asyncio_reactor_installed(self): # the result should depend only on the pytest --reactor argument - assert is_asyncio_reactor_installed() == (self.reactor_pytest == "asyncio") + assert is_asyncio_reactor_installed() == (self.reactor_pytest != "default") def test_install_asyncio_reactor(self): from twisted.internet import reactor as original_reactor diff --git a/tox.ini b/tox.ini index 70c841603..eb084f0f5 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ deps = {[test-requirements]deps} # mitmproxy does not support PyPy - mitmproxy; implementation_name != 'pypy' + mitmproxy; implementation_name != "pypy" setenv = COVERAGE_CORE=sysmon passenv = @@ -96,19 +96,18 @@ commands = [pinned] basepython = python3.9 deps = + Protego==0.1.15 + Twisted==21.7.0 cryptography==37.0.0 cssselect==0.9.1 - h2==3.0 itemadapter==0.1.0 + lxml==4.6.0 parsel==1.5.0 - Protego==0.1.15 pyOpenSSL==22.0.0 queuelib==1.4.2 service_identity==18.1.0 - Twisted[http2]==21.7.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.6.0 {[test-requirements]deps} # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies @@ -131,60 +130,50 @@ setenv = {[pinned]setenv} commands = {[pinned]commands} -[testenv:windows-pinned] -basepython = {[pinned]basepython} -deps = - {[pinned]deps} - PyDispatcher==2.0.5 -install_command = {[pinned]install_command} -setenv = - {[pinned]setenv} -commands = {[pinned]commands} - [testenv:extra-deps] basepython = python3 deps = {[testenv]deps} - boto3 - google-cloud-storage - robotexclusionrulesparser Pillow Twisted[http2] - uvloop; platform_system != "Windows" + boto3 bpython # optional for shell wrapper tests - brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests - brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests - zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests + brotli; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests + brotlicffi; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests + google-cloud-storage ipython + robotexclusionrulesparser + uvloop; platform_system != "Windows" + zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests [testenv:extra-deps-pinned] basepython = {[pinned]basepython} deps = {[pinned]deps} + Pillow==8.0.0 boto3==1.20.0 - google-cloud-storage==1.29.0 - Pillow==7.1.0 - robotexclusionrulesparser==1.6.2 - brotlipy - uvloop==0.14.0; platform_system != "Windows" bpython==0.7.1 - zstandard==0.1; implementation_name != 'pypy' + brotli==0.5.2; implementation_name != "pypy" + brotlicffi==0.8.0; implementation_name == "pypy" + brotlipy + google-cloud-storage==1.29.0 ipython==2.0.0 - brotli==0.5.2; implementation_name != 'pypy' - brotlicffi==0.8.0; implementation_name == 'pypy' + robotexclusionrulesparser==1.6.2 + uvloop==0.14.0; platform_system != "Windows" + zstandard==0.1; implementation_name != "pypy" install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = {[pinned]commands} -[testenv:asyncio] +[testenv:default-reactor] commands = - {[testenv]commands} --reactor=asyncio + {[testenv]commands} --reactor=default -[testenv:asyncio-pinned] +[testenv:default-reactor-pinned] basepython = {[pinned]basepython} deps = {[testenv:pinned]deps} -commands = {[pinned]commands} --reactor=asyncio +commands = {[pinned]commands} --reactor=default install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -204,21 +193,20 @@ commands = {[testenv:pypy3]commands} [testenv:pypy3-pinned] basepython = pypy3.10 deps = + PyPyDispatcher==2.1.0 + {[test-requirements]deps} + Protego==0.1.15 + Twisted==21.7.0 cryptography==41.0.5 cssselect==0.9.1 - h2==3.1 itemadapter==0.1.0 + lxml==4.6.0 parsel==1.5.0 - Protego==0.1.15 pyOpenSSL==23.3.0 queuelib==1.4.2 service_identity==18.1.0 - Twisted[http2]==21.7.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.6.0 - {[test-requirements]deps} - PyPyDispatcher==2.1.0 commands = ; disabling both coverage and docs tests pytest {posargs:--durations=10 scrapy tests}